Flag in _field_caps to return only fields with values in index (#103651)

We are adding a query parameter to the field_caps api in order to filter out 
fields with no values. The parameter is called `include_empty_fields`  and 
defaults to true, and if set to false it will filter out from the field_caps 
response all the fields that has no value in the index.
We keep track of FieldInfos during refresh in order to know which field has 
value in an index. We added also a system property 
`es.field_caps_empty_fields_filter` in order to disable this feature if needed.

---------

Co-authored-by: Matthias Wilhelm <[email protected]>

Author: piergm

docs/changelog/103651.yaml

@@ -0,0 +1,5 @@
+pr: 103651
+summary: Flag in `_field_caps` to return only fields with values in index
+area: Search
+type: enhancement
+issues: []

docs/reference/search/field-caps.asciidoc

@@ -77,6 +77,10 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailab
   (Optional, Boolean) If `true`, unmapped fields that are mapped in one index but not in another are included in the response. Fields that don't have any mapping are never included.
   Defaults to `false`.
 
+`include_empty_fields`::
+  (Optional, Boolean) If `false`, fields that never had a value in any shards are not included in the response. Fields that are not empty are always included. This flag does not consider deletions and updates.  If a field was non-empty and all the documents containing that field were deleted or the field was removed by updates,  it will still be returned even if the flag is `false`.
+  Defaults to `true`.
+
 `filters`::
 (Optional, string) Comma-separated list of filters to apply to the response.
 +

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java

@@ -9,6 +9,8 @@
 package org.elasticsearch.index.mapper.extras;
 
 import org.apache.lucene.document.FeatureField;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
@@ -38,6 +40,7 @@
  */
 public class RankFeatureFieldMapper extends FieldMapper {
 
+    public static final String NAME = "_feature";
     public static final String CONTENT_TYPE = "rank_feature";
 
     private static RankFeatureFieldType ft(FieldMapper in) {
@@ -128,7 +131,17 @@ public boolean positiveScoreImpact() {
 
         @Override
         public Query existsQuery(SearchExecutionContext context) {
-            return new TermQuery(new Term("_feature", name()));
+            return new TermQuery(new Term(NAME, name()));
+        }
+
+        @Override
+        public boolean fieldHasValue(FieldInfos fieldInfos) {
+            for (FieldInfo fieldInfo : fieldInfos) {
+                if (fieldInfo.getName().equals(NAME)) {
+                    return true;
+                }
+            }
+            return false;
         }
 
         @Override
@@ -208,7 +221,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
             value = 1 / value;
         }
 
-        context.doc().addWithKey(name(), new FeatureField("_feature", name(), value));
+        context.doc().addWithKey(name(), new FeatureField(NAME, name(), value));
     }
 
     private static Float objectToFloat(Object value) {

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapper.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.index.mapper.extras;
 
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
@@ -35,7 +37,8 @@ public static final class RankFeatureMetaFieldType extends MappedFieldType {
 
         public static final RankFeatureMetaFieldType INSTANCE = new RankFeatureMetaFieldType();
 
-        private RankFeatureMetaFieldType() {
+        // made visible for tests
+        RankFeatureMetaFieldType() {
             super(NAME, false, false, false, TextSearchInfo.NONE, Collections.emptyMap());
         }
 
@@ -54,6 +57,16 @@ public Query existsQuery(SearchExecutionContext context) {
             throw new UnsupportedOperationException("Cannot run exists query on [_feature]");
         }
 
+        @Override
+        public boolean fieldHasValue(FieldInfos fieldInfos) {
+            for (FieldInfo fieldInfo : fieldInfos) {
+                if (fieldInfo.getName().equals(NAME)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+
         @Override
         public Query termQuery(Object value, SearchExecutionContext context) {
             throw new UnsupportedOperationException("The [_feature] field may not be queried directly");

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/FieldCapsRankFeatureTests.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper.extras;
+
+import org.elasticsearch.action.fieldcaps.FieldCapabilities;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
+public class FieldCapsRankFeatureTests extends ESIntegTestCase {
+    private final String INDEX = "index-1";
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        var plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(MapperExtrasPlugin.class);
+        return plugins;
+    }
+
+    @Before
+    public void setUpIndices() {
+        assertAcked(
+            prepareCreate(INDEX).setWaitForActiveShards(ActiveShardCount.ALL)
+                .setSettings(indexSettings())
+                .setMapping("fooRank", "type=rank_feature", "barRank", "type=rank_feature")
+        );
+    }
+
+    public void testRankFeatureInIndex() {
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(INDEX).setFields("*").setincludeEmptyFields(false).get();
+        assertFalse(response.get().containsKey("fooRank"));
+        assertFalse(response.get().containsKey("barRank"));
+        prepareIndex(INDEX).setSource("fooRank", 8).setSource("barRank", 8).get();
+        refresh(INDEX);
+
+        response = client().prepareFieldCaps(INDEX).setFields("*").setincludeEmptyFields(false).get();
+        assertEquals(1, response.getIndices().length);
+        assertEquals(response.getIndices()[0], INDEX);
+        assertThat(response.get(), Matchers.hasKey("fooRank"));
+        // Check the capabilities for the 'fooRank' field.
+        Map<String, FieldCapabilities> fooRankField = response.getField("fooRank");
+        assertEquals(1, fooRankField.size());
+        assertThat(fooRankField, Matchers.hasKey("rank_feature"));
+        assertEquals(
+            new FieldCapabilities("fooRank", "rank_feature", false, true, false, null, null, null, Collections.emptyMap()),
+            fooRankField.get("rank_feature")
+        );
+    }
+
+    public void testRankFeatureInIndexAfterRestart() throws Exception {
+        prepareIndex(INDEX).setSource("fooRank", 8).get();
+        internalCluster().fullRestart();
+        ensureGreen(INDEX);
+
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(INDEX).setFields("*").setincludeEmptyFields(false).get();
+
+        assertEquals(1, response.getIndices().length);
+        assertEquals(response.getIndices()[0], INDEX);
+        assertThat(response.get(), Matchers.hasKey("fooRank"));
+        // Check the capabilities for the 'fooRank' field.
+        Map<String, FieldCapabilities> fooRankField = response.getField("fooRank");
+        assertEquals(1, fooRankField.size());
+        assertThat(fooRankField, Matchers.hasKey("rank_feature"));
+        assertEquals(
+            new FieldCapabilities("fooRank", "rank_feature", false, true, false, null, null, null, Collections.emptyMap()),
+            fooRankField.get("rank_feature")
+        );
+    }
+
+    public void testAllRankFeatureReturnedIfOneIsPresent() {
+        prepareIndex(INDEX).setSource("fooRank", 8).get();
+        refresh(INDEX);
+
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(INDEX).setFields("*").setincludeEmptyFields(false).get();
+
+        assertEquals(1, response.getIndices().length);
+        assertEquals(response.getIndices()[0], INDEX);
+        assertThat(response.get(), Matchers.hasKey("fooRank"));
+        // Check the capabilities for the 'fooRank' field.
+        Map<String, FieldCapabilities> fooRankField = response.getField("fooRank");
+        assertEquals(1, fooRankField.size());
+        assertThat(fooRankField, Matchers.hasKey("rank_feature"));
+        assertEquals(
+            new FieldCapabilities("fooRank", "rank_feature", false, true, false, null, null, null, Collections.emptyMap()),
+            fooRankField.get("rank_feature")
+        );
+        assertThat(response.get(), Matchers.hasKey("barRank"));
+        // Check the capabilities for the 'barRank' field.
+        Map<String, FieldCapabilities> barRankField = response.getField("barRank");
+        assertEquals(1, barRankField.size());
+        assertThat(barRankField, Matchers.hasKey("rank_feature"));
+        assertEquals(
+            new FieldCapabilities("barRank", "rank_feature", false, true, false, null, null, null, Collections.emptyMap()),
+            barRankField.get("rank_feature")
+        );
+    }
+}

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldTypeTests.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.index.mapper.extras;
 
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
@@ -19,7 +21,7 @@
 public class RankFeatureFieldTypeTests extends FieldTypeTestCase {
 
     public void testIsNotAggregatable() {
-        MappedFieldType fieldType = new RankFeatureFieldMapper.RankFeatureFieldType("field", Collections.emptyMap(), true, null);
+        MappedFieldType fieldType = getMappedFieldType();
         assertFalse(fieldType.isAggregatable());
     }
 
@@ -32,4 +34,28 @@ public void testFetchSourceValue() throws IOException {
         assertEquals(List.of(42.9f), fetchSourceValue(mapper, "42.9"));
         assertEquals(List.of(2.0f), fetchSourceValue(mapper, null));
     }
+
+    @Override
+    public void testFieldHasValue() {
+        MappedFieldType fieldType = getMappedFieldType();
+        FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] { getFieldInfoWithName("_feature") });
+        assertTrue(fieldType.fieldHasValue(fieldInfos));
+    }
+
+    @Override
+    public void testFieldHasValueWithEmptyFieldInfos() {
+        MappedFieldType fieldType = getMappedFieldType();
+        assertFalse(fieldType.fieldHasValue(FieldInfos.EMPTY));
+    }
+
+    public void testFieldEmptyIfNameIsPresentInFieldInfos() {
+        MappedFieldType fieldType = getMappedFieldType();
+        FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] { getFieldInfoWithName("field") });
+        assertFalse(fieldType.fieldHasValue(fieldInfos));
+    }
+
+    @Override
+    public MappedFieldType getMappedFieldType() {
+        return new RankFeatureFieldMapper.RankFeatureFieldType("field", Collections.emptyMap(), true, null);
+    }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapperTests.java

@@ -8,11 +8,14 @@
 
 package org.elasticsearch.index.mapper.extras;
 
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentParsingException;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperServiceTestCase;
 import org.elasticsearch.index.mapper.Mapping;
@@ -73,4 +76,19 @@ public void testDocumentParsingFailsOnMetaField() throws Exception {
             CoreMatchers.containsString("Field [" + rfMetaField + "] is a metadata field and cannot be added inside a document.")
         );
     }
+
+    @Override
+    public void testFieldHasValue() {
+        assertTrue(getMappedFieldType().fieldHasValue(new FieldInfos(new FieldInfo[] { getFieldInfoWithName("_feature") })));
+    }
+
+    @Override
+    public void testFieldHasValueWithEmptyFieldInfos() {
+        assertFalse(getMappedFieldType().fieldHasValue(FieldInfos.EMPTY));
+    }
+
+    @Override
+    public MappedFieldType getMappedFieldType() {
+        return new RankFeatureMetaFieldMapper.RankFeatureMetaFieldType();
+    }
 }

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml

@@ -65,7 +65,6 @@
       field_caps:
         index: 'my_remote_cluster:some_index_that_doesnt_exist'
         fields: [number]
-        
   - match: { error.type: "index_not_found_exception" }
   - match: { error.reason: "no such index [some_index_that_doesnt_exist]" }
 
@@ -156,3 +155,38 @@
   - length: {fields.number:                             1}
   - match: {fields.number.long.searchable:              true}
   - match: {fields.number.long.aggregatable:            true}
+
+---
+"Field caps with with include_empty_fields false":
+  - skip:
+      version: " - 8.12.99"
+      reason: include_empty_fields has been added in 8.13.0
+  - do:
+        indices.create:
+          index: field_caps_index_5
+          body:
+            mappings:
+              properties:
+                number:
+                  type: double
+                empty-baz:
+                  type: text
+
+  - do:
+      index:
+        index:  field_caps_index_5
+        body:   { number: "42", unmapped-bar: "bar" }
+
+  - do:
+      indices.refresh:
+        index: [field_caps_index_5]
+  - do:
+      field_caps:
+        include_empty_fields: false
+        index: 'field_caps_index_5,my_remote_cluster:field_*'
+        fields: '*'
+
+  - is_true: fields.number
+  - is_false: fields.empty-baz
+  - is_true: fields.unmapped-bar
+  - is_true: fields._index

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml

@@ -82,6 +82,10 @@
                       nested2:
                         type: keyword
                         doc_values: false
+  - do:
+        index:
+          index:  field_caps_index_1
+          body:   { number: "42", unmapped-bar: "bar" }
   - do:
       indices.create:
         index: test_index

rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json

@@ -71,6 +71,11 @@
       "types": {
         "type": "list",
         "description":"Only return results for fields that have one of the types in the list"
+      },
+      "include_empty_fields": {
+        "type":"boolean",
+        "default": true,
+        "description":"Include empty fields in result"
       }
     },
     "body":{