Flag in _field_caps to return only fields with values in index

docs/changelog/103651.yaml

@@ -0,0 +1,5 @@
+pr: 103651
+summary: Flag in `_field_caps` to return only fields with values in index
+area: Search
+type: enhancement
+issues: []

docs/reference/search/field-caps.asciidoc

@@ -77,6 +77,10 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailab
   (Optional, Boolean) If `true`, unmapped fields that are mapped in one index but not in another are included in the response. Fields that don't have any mapping are never included.
   Defaults to `false`.
 
+`include_empty_fields`::
+  (Optional, Boolean) If `false`, fields that never had a value in any shards are not included in the response. Fields that are not empty are always included. This flag does not consider deletions and updates.  If a field was non-empty and all the documents containing that field were deleted or the field was removed by updates,  it will still be returned even if the flag is `false`.
+  Defaults to `true`.
+
 `filters`::
 (Optional, string) Comma-separated list of filters to apply to the response.
 +

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java

@@ -9,6 +9,8 @@
 package org.elasticsearch.index.mapper.extras;
 
 import org.apache.lucene.document.FeatureField;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
@@ -38,6 +40,7 @@
  */
 public class RankFeatureFieldMapper extends FieldMapper {
 
+    public static final String NAME = "_feature";
     public static final String CONTENT_TYPE = "rank_feature";
 
     private static RankFeatureFieldType ft(FieldMapper in) {
@@ -128,7 +131,17 @@ public boolean positiveScoreImpact() {
 
         @Override
         public Query existsQuery(SearchExecutionContext context) {
-            return new TermQuery(new Term("_feature", name()));
+            return new TermQuery(new Term(NAME, name()));
+        }
+
+        @Override
+        public boolean fieldHasValue(FieldInfos fieldInfos) {
+            for (FieldInfo fieldInfo : fieldInfos) {
+                if (fieldInfo.getName().equals(NAME)) {
+                    return true;
+                }
+            }
+            return false;
         }
 
         @Override
@@ -208,7 +221,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
             value = 1 / value;
         }
 
-        context.doc().addWithKey(name(), new FeatureField("_feature", name(), value));
+        context.doc().addWithKey(name(), new FeatureField(NAME, name(), value));
     }
 
     private static Float objectToFloat(Object value) {

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapper.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.index.mapper.extras;
 
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
@@ -35,7 +37,8 @@ public static final class RankFeatureMetaFieldType extends MappedFieldType {
 
         public static final RankFeatureMetaFieldType INSTANCE = new RankFeatureMetaFieldType();
 
-        private RankFeatureMetaFieldType() {
+        // made visible for tests
+        RankFeatureMetaFieldType() {
             super(NAME, false, false, false, TextSearchInfo.NONE, Collections.emptyMap());
         }
 
@@ -54,6 +57,16 @@ public Query existsQuery(SearchExecutionContext context) {
             throw new UnsupportedOperationException("Cannot run exists query on [_feature]");
         }
 
+        @Override
+        public boolean fieldHasValue(FieldInfos fieldInfos) {
+            for (FieldInfo fieldInfo : fieldInfos) {
+                if (fieldInfo.getName().equals(NAME)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+
         @Override
         public Query termQuery(Object value, SearchExecutionContext context) {
             throw new UnsupportedOperationException("The [_feature] field may not be queried directly");

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/FieldCapsRankFeatureIT.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper.extras;
+
+import org.elasticsearch.action.fieldcaps.FieldCapabilities;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
+public class FieldCapsRankFeatureIT extends ESIntegTestCase {
+    private final String INDEX = "index-1";
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        var plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(MapperExtrasPlugin.class);
+        return plugins;
+    }
+
+    @Before
+    public void setUpIndices() {
+        assertAcked(
+            prepareCreate(INDEX).setWaitForActiveShards(ActiveShardCount.ALL)
+                .setSettings(indexSettings())
+                .setMapping("fooRank", "type=rank_feature", "barRank", "type=rank_feature")
+        );
+    }
+
+    public void testRankFeatureInIndex() {
+        prepareIndex(INDEX).setSource("fooRank", 8).setSource("barRank", 8).get();
+        refresh(INDEX);
+
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(INDEX).setFields("*").setincludeEmptyFields(false).get();
+        assertEquals(1, response.getIndices().length);
+        assertEquals(response.getIndices()[0], INDEX);
+        assertThat(response.get(), Matchers.hasKey("fooRank"));
+        // Check the capabilities for the 'fooRank' field.
+        Map<String, FieldCapabilities> fooRankField = response.getField("fooRank");
+        assertEquals(1, fooRankField.size());
+        assertThat(fooRankField, Matchers.hasKey("rank_feature"));
+        assertEquals(
+            new FieldCapabilities("fooRank", "rank_feature", false, true, false, null, null, null, Collections.emptyMap()),
+            fooRankField.get("rank_feature")
+        );
+    }
+
+    public void testRankFeatureInIndexAfterRestart() throws Exception {
+        prepareIndex(INDEX).setSource("fooRank", 8).get();
+        internalCluster().fullRestart();
+        ensureGreen(INDEX);
+
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(INDEX).setFields("*").setincludeEmptyFields(false).get();
+
+        assertEquals(1, response.getIndices().length);
+        assertEquals(response.getIndices()[0], INDEX);
+        assertThat(response.get(), Matchers.hasKey("fooRank"));
+        // Check the capabilities for the 'fooRank' field.
+        Map<String, FieldCapabilities> fooRankField = response.getField("fooRank");
+        assertEquals(1, fooRankField.size());
+        assertThat(fooRankField, Matchers.hasKey("rank_feature"));
+        assertEquals(
+            new FieldCapabilities("fooRank", "rank_feature", false, true, false, null, null, null, Collections.emptyMap()),
+            fooRankField.get("rank_feature")
+        );
+    }
+
+    public void testAllRankFeatureReturnedIfOneIsPresent() throws Exception {
+        prepareIndex(INDEX).setSource("fooRank", 8).get();
+        refresh(INDEX);
+
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(INDEX).setFields("*").setincludeEmptyFields(false).get();
+
+        assertEquals(1, response.getIndices().length);
+        assertEquals(response.getIndices()[0], INDEX);
+        assertThat(response.get(), Matchers.hasKey("fooRank"));
+        // Check the capabilities for the 'fooRank' field.
+        Map<String, FieldCapabilities> fooRankField = response.getField("fooRank");
+        assertEquals(1, fooRankField.size());
+        assertThat(fooRankField, Matchers.hasKey("rank_feature"));
+        assertEquals(
+            new FieldCapabilities("fooRank", "rank_feature", false, true, false, null, null, null, Collections.emptyMap()),
+            fooRankField.get("rank_feature")
+        );
+        assertThat(response.get(), Matchers.hasKey("barRank"));
+        // Check the capabilities for the 'barRank' field.
+        Map<String, FieldCapabilities> barRankField = response.getField("barRank");
+        assertEquals(1, barRankField.size());
+        assertThat(barRankField, Matchers.hasKey("rank_feature"));
+        assertEquals(
+            new FieldCapabilities("barRank", "rank_feature", false, true, false, null, null, null, Collections.emptyMap()),
+            barRankField.get("rank_feature")
+        );
+    }
+}

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldTypeTests.java

@@ -8,18 +8,25 @@
 
 package org.elasticsearch.index.mapper.extras;
 
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 
 public class RankFeatureFieldTypeTests extends FieldTypeTestCase {
 
     public void testIsNotAggregatable() {
-        MappedFieldType fieldType = new RankFeatureFieldMapper.RankFeatureFieldType("field", Collections.emptyMap(), true, null);
+        MappedFieldType fieldType = getRankFeatureFieldType();
         assertFalse(fieldType.isAggregatable());
     }
 
@@ -32,4 +39,47 @@ public void testFetchSourceValue() throws IOException {
         assertEquals(List.of(42.9f), fetchSourceValue(mapper, "42.9"));
         assertEquals(List.of(2.0f), fetchSourceValue(mapper, null));
     }
+
+    public void testFieldHasValueIf_featureIsPresentInFieldInfos() {
+        MappedFieldType fieldType = getRankFeatureFieldType();
+        FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] { getFieldInfoWithName("_feature") });
+        assertTrue(fieldType.fieldHasValue(fieldInfos));
+    }
+
+    public void testFieldEmptyIfNameIsPresentInFieldInfos() {
+        MappedFieldType fieldType = getRankFeatureFieldType();
+        FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] { getFieldInfoWithName("field") });
+        assertFalse(fieldType.fieldHasValue(fieldInfos));
+    }
+
+    public void testFieldEmptyIfEmptyFieldInfos() {
+        MappedFieldType fieldType = getRankFeatureFieldType();
+        FieldInfos fieldInfos = FieldInfos.EMPTY;
+        assertFalse(fieldType.fieldHasValue(fieldInfos));
+    }
+
+    private RankFeatureFieldMapper.RankFeatureFieldType getRankFeatureFieldType() {
+        return new RankFeatureFieldMapper.RankFeatureFieldType("field", Collections.emptyMap(), true, null);
+    }
+
+    private FieldInfo getFieldInfoWithName(String name) {
+        return new FieldInfo(
+            name,
+            1,
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean(),
+            IndexOptions.NONE,
+            DocValuesType.NONE,
+            -1,
+            new HashMap<>(),
+            1,
+            1,
+            1,
+            1,
+            VectorEncoding.BYTE,
+            VectorSimilarityFunction.COSINE,
+            randomBoolean()
+        );
+    }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapperTests.java

@@ -8,11 +8,18 @@
 
 package org.elasticsearch.index.mapper.extras;
 
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentParsingException;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperServiceTestCase;
 import org.elasticsearch.index.mapper.Mapping;
@@ -24,6 +31,7 @@
 
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 
 public class RankFeatureMetaFieldMapperTests extends MapperServiceTestCase {
 
@@ -73,4 +81,43 @@ public void testDocumentParsingFailsOnMetaField() throws Exception {
             CoreMatchers.containsString("Field [" + rfMetaField + "] is a metadata field and cannot be added inside a document.")
         );
     }
+
+    public void testFieldHasValueIf_featureIsPresentInFieldInfos() {
+        MappedFieldType fieldType = new RankFeatureMetaFieldMapper.RankFeatureMetaFieldType();
+        FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] { getFieldInfoWithName("_feature") });
+        assertTrue(fieldType.fieldHasValue(fieldInfos));
+    }
+
+    public void testFieldEmptyIfNameIsPresentInFieldInfos() {
+        MappedFieldType fieldType = new RankFeatureMetaFieldMapper.RankFeatureMetaFieldType();
+        FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] { getFieldInfoWithName("field") });
+        assertFalse(fieldType.fieldHasValue(fieldInfos));
+    }
+
+    public void testFieldEmptyIfEmptyFieldInfos() {
+        MappedFieldType fieldType = new RankFeatureMetaFieldMapper.RankFeatureMetaFieldType();
+        FieldInfos fieldInfos = FieldInfos.EMPTY;
+        assertFalse(fieldType.fieldHasValue(fieldInfos));
+    }
+
+    private FieldInfo getFieldInfoWithName(String name) {
+        return new FieldInfo(
+            name,
+            1,
+            randomBoolean(),
+            randomBoolean(),
+            randomBoolean(),
+            IndexOptions.NONE,
+            DocValuesType.NONE,
+            -1,
+            new HashMap<>(),
+            1,
+            1,
+            1,
+            1,
+            VectorEncoding.BYTE,
+            VectorSimilarityFunction.COSINE,
+            randomBoolean()
+        );
+    }
 }