Flag in _field_caps to return only fields with values in index

docs/changelog/103651.yaml

@@ -0,0 +1,5 @@
+pr: 103651
+summary: Flag in `_field_caps` to return only fields with values in index
+area: Search
+type: enhancement
+issues: []

docs/reference/search/field-caps.asciidoc

@@ -77,6 +77,10 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailab
   (Optional, Boolean) If `true`, unmapped fields that are mapped in one index but not in another are included in the response. Fields that don't have any mapping are never included.
   Defaults to `false`.
 
+`include_empty_fields`::
+  (Optional, Boolean) If `false`, fields that never had a value in any shards are not included in the response. Fields that are not empty are always included. This flag does not consider deletions and updates.  If a field was non-empty and all the documents containing that field were deleted or the field was removed by updates,  it will still be returned even if the flag is `false`.
+  Defaults to `true`.
+
 `filters`::
 (Optional, string) Comma-separated list of filters to apply to the response.
 +

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java

@@ -9,6 +9,8 @@
 package org.elasticsearch.index.mapper.extras;
 
 import org.apache.lucene.document.FeatureField;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
@@ -38,6 +40,7 @@
  */
 public class RankFeatureFieldMapper extends FieldMapper {
 
+    public static final String NAME = "_feature";
     public static final String CONTENT_TYPE = "rank_feature";
 
     private static RankFeatureFieldType ft(FieldMapper in) {
@@ -128,7 +131,17 @@ public boolean positiveScoreImpact() {
 
         @Override
         public Query existsQuery(SearchExecutionContext context) {
-            return new TermQuery(new Term("_feature", name()));
+            return new TermQuery(new Term(NAME, name()));
+        }
+
+        @Override
+        public boolean fieldHasValue(FieldInfos fieldInfos) {
+            for (FieldInfo fieldInfo : fieldInfos) {
+                if (fieldInfo.getName().equals(NAME)) {
+                    return true;
+                }
+            }
+            return false;
         }
 
         @Override
@@ -208,7 +221,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
             value = 1 / value;
         }
 
-        context.doc().addWithKey(name(), new FeatureField("_feature", name(), value));
+        context.doc().addWithKey(name(), new FeatureField(NAME, name(), value));
     }
 
     private static Float objectToFloat(Object value) {

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapper.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.index.mapper.extras;
 
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
@@ -35,7 +37,8 @@ public static final class RankFeatureMetaFieldType extends MappedFieldType {
 
         public static final RankFeatureMetaFieldType INSTANCE = new RankFeatureMetaFieldType();
 
-        private RankFeatureMetaFieldType() {
+        // made visible for tests
+        RankFeatureMetaFieldType() {
             super(NAME, false, false, false, TextSearchInfo.NONE, Collections.emptyMap());
         }
 
@@ -54,6 +57,16 @@ public Query existsQuery(SearchExecutionContext context) {
             throw new UnsupportedOperationException("Cannot run exists query on [_feature]");
         }
 
+        @Override
+        public boolean fieldHasValue(FieldInfos fieldInfos) {
+            for (FieldInfo fieldInfo : fieldInfos) {
+                if (fieldInfo.getName().equals(NAME)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+
         @Override
         public Query termQuery(Object value, SearchExecutionContext context) {
             throw new UnsupportedOperationException("The [_feature] field may not be queried directly");

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/FieldCapsRankFeatureTests.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper.extras;
+
+import org.elasticsearch.action.fieldcaps.FieldCapabilities;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.hamcrest.Matchers;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
+public class FieldCapsRankFeatureTests extends ESIntegTestCase {
+    private final String INDEX = "index-1";
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        var plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(MapperExtrasPlugin.class);
+        return plugins;
+    }
+
+    @Before
+    public void setUpIndices() {
+        assertAcked(
+            prepareCreate(INDEX).setWaitForActiveShards(ActiveShardCount.ALL)
+                .setSettings(indexSettings())
+                .setMapping("fooRank", "type=rank_feature", "barRank", "type=rank_feature")
+        );
+    }
+
+    public void testRankFeatureInIndex() {
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(INDEX).setFields("*").setincludeEmptyFields(false).get();
+        assertFalse(response.get().containsKey("fooRank"));
+        assertFalse(response.get().containsKey("barRank"));
+        prepareIndex(INDEX).setSource("fooRank", 8).setSource("barRank", 8).get();
+        refresh(INDEX);
+
+        response = client().prepareFieldCaps(INDEX).setFields("*").setincludeEmptyFields(false).get();
+        assertEquals(1, response.getIndices().length);
+        assertEquals(response.getIndices()[0], INDEX);
+        assertThat(response.get(), Matchers.hasKey("fooRank"));
+        // Check the capabilities for the 'fooRank' field.
+        Map<String, FieldCapabilities> fooRankField = response.getField("fooRank");
+        assertEquals(1, fooRankField.size());
+        assertThat(fooRankField, Matchers.hasKey("rank_feature"));
+        assertEquals(
+            new FieldCapabilities("fooRank", "rank_feature", false, true, false, null, null, null, Collections.emptyMap()),
+            fooRankField.get("rank_feature")
+        );
+    }
+
+    public void testRankFeatureInIndexAfterRestart() throws Exception {
+        prepareIndex(INDEX).setSource("fooRank", 8).get();
+        internalCluster().fullRestart();
+        ensureGreen(INDEX);
+
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(INDEX).setFields("*").setincludeEmptyFields(false).get();
+
+        assertEquals(1, response.getIndices().length);
+        assertEquals(response.getIndices()[0], INDEX);
+        assertThat(response.get(), Matchers.hasKey("fooRank"));
+        // Check the capabilities for the 'fooRank' field.
+        Map<String, FieldCapabilities> fooRankField = response.getField("fooRank");
+        assertEquals(1, fooRankField.size());
+        assertThat(fooRankField, Matchers.hasKey("rank_feature"));
+        assertEquals(
+            new FieldCapabilities("fooRank", "rank_feature", false, true, false, null, null, null, Collections.emptyMap()),
+            fooRankField.get("rank_feature")
+        );
+    }
+
+    public void testAllRankFeatureReturnedIfOneIsPresent() {
+        prepareIndex(INDEX).setSource("fooRank", 8).get();
+        refresh(INDEX);
+
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(INDEX).setFields("*").setincludeEmptyFields(false).get();
+
+        assertEquals(1, response.getIndices().length);
+        assertEquals(response.getIndices()[0], INDEX);
+        assertThat(response.get(), Matchers.hasKey("fooRank"));
+        // Check the capabilities for the 'fooRank' field.
+        Map<String, FieldCapabilities> fooRankField = response.getField("fooRank");
+        assertEquals(1, fooRankField.size());
+        assertThat(fooRankField, Matchers.hasKey("rank_feature"));
+        assertEquals(
+            new FieldCapabilities("fooRank", "rank_feature", false, true, false, null, null, null, Collections.emptyMap()),
+            fooRankField.get("rank_feature")
+        );
+        assertThat(response.get(), Matchers.hasKey("barRank"));
+        // Check the capabilities for the 'barRank' field.
+        Map<String, FieldCapabilities> barRankField = response.getField("barRank");
+        assertEquals(1, barRankField.size());
+        assertThat(barRankField, Matchers.hasKey("rank_feature"));
+        assertEquals(
+            new FieldCapabilities("barRank", "rank_feature", false, true, false, null, null, null, Collections.emptyMap()),
+            barRankField.get("rank_feature")
+        );
+    }
+}

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldTypeTests.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.index.mapper.extras;
 
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
@@ -19,7 +21,7 @@
 public class RankFeatureFieldTypeTests extends FieldTypeTestCase {
 
     public void testIsNotAggregatable() {
-        MappedFieldType fieldType = new RankFeatureFieldMapper.RankFeatureFieldType("field", Collections.emptyMap(), true, null);
+        MappedFieldType fieldType = getMappedFieldType();
         assertFalse(fieldType.isAggregatable());
     }
 
@@ -32,4 +34,28 @@ public void testFetchSourceValue() throws IOException {
         assertEquals(List.of(42.9f), fetchSourceValue(mapper, "42.9"));
         assertEquals(List.of(2.0f), fetchSourceValue(mapper, null));
     }
+
+    @Override
+    public void testFieldHasValue() {
+        MappedFieldType fieldType = getMappedFieldType();
+        FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] { getFieldInfoWithName("_feature") });
+        assertTrue(fieldType.fieldHasValue(fieldInfos));
+    }
+
+    @Override
+    public void testFieldHasValueWithEmptyFieldInfos() {
+        MappedFieldType fieldType = getMappedFieldType();
+        assertFalse(fieldType.fieldHasValue(FieldInfos.EMPTY));
+    }
+
+    public void testFieldEmptyIfNameIsPresentInFieldInfos() {
+        MappedFieldType fieldType = getMappedFieldType();
+        FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] { getFieldInfoWithName("field") });
+        assertFalse(fieldType.fieldHasValue(fieldInfos));
+    }
+
+    @Override
+    public MappedFieldType getMappedFieldType() {
+        return new RankFeatureFieldMapper.RankFeatureFieldType("field", Collections.emptyMap(), true, null);
+    }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapperTests.java

@@ -8,11 +8,14 @@
 
 package org.elasticsearch.index.mapper.extras;
 
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentParsingException;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperServiceTestCase;
 import org.elasticsearch.index.mapper.Mapping;
@@ -73,4 +76,19 @@ public void testDocumentParsingFailsOnMetaField() throws Exception {
             CoreMatchers.containsString("Field [" + rfMetaField + "] is a metadata field and cannot be added inside a document.")
         );
     }
+
+    @Override
+    public void testFieldHasValue() {
+        assertTrue(getMappedFieldType().fieldHasValue(new FieldInfos(new FieldInfo[] { getFieldInfoWithName("_feature") })));
+    }
+
+    @Override
+    public void testFieldHasValueWithEmptyFieldInfos() {
+        assertFalse(getMappedFieldType().fieldHasValue(FieldInfos.EMPTY));
+    }
+
+    @Override
+    public MappedFieldType getMappedFieldType() {
+        return new RankFeatureMetaFieldMapper.RankFeatureMetaFieldType();
+    }
 }

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml

@@ -65,7 +65,6 @@
       field_caps:
         index: 'my_remote_cluster:some_index_that_doesnt_exist'
         fields: [number]
-
   - match: { error.type: "index_not_found_exception" }
   - match: { error.reason: "no such index [some_index_that_doesnt_exist]" }
 
@@ -156,3 +155,38 @@
   - length: {fields.number:                             1}
   - match: {fields.number.long.searchable:              true}
   - match: {fields.number.long.aggregatable:            true}
+
+---
+"Field caps with with include_empty_fields false":
+  - skip:
+      version: " - 8.12.99"
+      reason: include_empty_fields has been added in 8.13.0
+  - do:
+        indices.create:
+          index: field_caps_index_5
+          body:
+            mappings:
+              properties:
+                number:
+                  type: double
+                empty-baz:
+                  type: text
+
+  - do:
+      index:
+        index:  field_caps_index_5
+        body:   { number: "42", unmapped-bar: "bar" }
+
+  - do:
+      indices.refresh:
+        index: [field_caps_index_5]
+  - do:
+      field_caps:
+        include_empty_fields: false
+        index: 'field_caps_index_5,my_remote_cluster:field_*'
+        fields: '*'
+
+  - is_true: fields.number
+  - is_false: fields.empty-baz
+  - is_true: fields.unmapped-bar
+  - is_true: fields._index

qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml

@@ -82,6 +82,10 @@
                       nested2:
                         type: keyword
                         doc_values: false
+  - do:
+        index:
+          index:  field_caps_index_1
+          body:   { number: "42", unmapped-bar: "bar" }
   - do:
       indices.create:
         index: test_index

rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json

@@ -71,6 +71,11 @@
       "types": {
         "type": "list",
         "description":"Only return results for fields that have one of the types in the list"
+      },
+      "include_empty_fields": {
+        "type":"boolean",
+        "default": true,
+        "description":"Include empty fields in result"
       }
     },
     "body":{