Flag in _field_caps to return only fields with values in index

docs/changelog/103651.yaml

@@ -0,0 +1,5 @@
+pr: 103651
+summary: Flag in `_field_caps` to return only fields with values in index
+area: Search
+type: enhancement
+issues: []

docs/reference/search/field-caps.asciidoc

@@ -77,6 +77,10 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailab
   (Optional, Boolean) If `true`, unmapped fields that are mapped in one index but not in another are included in the response. Fields that don't have any mapping are never included.
   Defaults to `false`.
 
+`include_fields_with_no_value`::
+  (Optional, Boolean) If `false`, fields that never had a value in any shards are not included in the response. Fields that are not empty are always included. This flag does not consider deletions, if a field was non-empty and all the documents containing that field were deleted the field is returned even if the flag is `false`.
+  Defaults to `true`.
+
 `filters`::
 (Optional, string) Comma-separated list of filters to apply to the response.
 +

server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java

@@ -33,6 +33,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.core.IOUtils;
@@ -628,7 +629,8 @@ public static final IndexShard newIndexShard(
             cbs,
             IndexModule.DEFAULT_SNAPSHOT_COMMIT_SUPPLIER,
             System::nanoTime,
-            null
+            null,
+            ConcurrentCollections.newConcurrentSet()
         );
     }
 

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -183,6 +183,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion HOT_THREADS_AS_BYTES = def(8_571_00_0);
     public static final TransportVersion ML_INFERENCE_REQUEST_INPUT_TYPE_ADDED = def(8_572_00_0);
     public static final TransportVersion ESQL_ENRICH_POLICY_CCQ_MODE = def(8_573_00_0);
+    public static final TransportVersion FIELD_CAPS_FIELD_HAS_VALUE = def(8_574_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.mapper.ConstantFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.RuntimeField;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
@@ -39,10 +40,12 @@
  */
 class FieldCapabilitiesFetcher {
     private final IndicesService indicesService;
+    private final boolean includeFieldsWithNoValue;
     private final Map<String, Map<String, IndexFieldCapabilities>> indexMappingHashToResponses = new HashMap<>();
 
-    FieldCapabilitiesFetcher(IndicesService indicesService) {
+    FieldCapabilitiesFetcher(IndicesService indicesService, boolean includeFieldsWithNoValue) {
         this.indicesService = indicesService;
+        this.includeFieldsWithNoValue = includeFieldsWithNoValue;
     }
 
     FieldCapabilitiesIndexResponse fetch(
@@ -100,7 +103,19 @@ private FieldCapabilitiesIndexResponse doFetch(
         }
 
         final MappingMetadata mapping = indexService.getMetadata().mapping();
-        final String indexMappingHash = mapping != null ? mapping.getSha256() : null;
+        String indexMappingHash;
+        if (includeFieldsWithNoValue) {
+            indexMappingHash = mapping != null ? mapping.getSha256() : null;
+        } else {
+            // even if the mapping is the same if we return only fields with values we need
+            // to make sure that we consider all the shard-mappings pair, that is why we
+            // calculate a different hash for this particular case.
+            StringBuilder sb = new StringBuilder(indexService.getShard(shardId.getId()).getShardUuid());
+            if (mapping != null) {
+                sb.append(mapping.getSha256());
+            }
+            indexMappingHash = sb.toString();
+        }
         if (indexMappingHash != null) {
             final Map<String, IndexFieldCapabilities> existing = indexMappingHashToResponses.get(indexMappingHash);
             if (existing != null) {
@@ -114,7 +129,9 @@ private FieldCapabilitiesIndexResponse doFetch(
             fieldNameFilter,
             filters,
             fieldTypes,
-            fieldPredicate
+            fieldPredicate,
+            indicesService.getShardOrNull(shardId),
+            includeFieldsWithNoValue
         );
         if (indexMappingHash != null) {
             indexMappingHashToResponses.put(indexMappingHash, responseMap);
@@ -127,7 +144,9 @@ static Map<String, IndexFieldCapabilities> retrieveFieldCaps(
         Predicate<String> fieldNameFilter,
         String[] filters,
         String[] types,
-        Predicate<String> indexFieldfilter
+        Predicate<String> indexFieldfilter,
+        IndexShard indexShard,
+        boolean includeFieldsWithNoValue
     ) {
         boolean includeParentObjects = checkIncludeParents(filters);
 
@@ -139,7 +158,8 @@ static Map<String, IndexFieldCapabilities> retrieveFieldCaps(
                 continue;
             }
             MappedFieldType ft = context.getFieldType(field);
-            if (filter.test(ft)) {
+            boolean includeField = includeFieldsWithNoValue || indexShard.fieldHasValue(ft.name()) || ft instanceof ConstantFieldType;
+            if (includeField && filter.test(ft)) {
                 IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(
                     field,
                     ft.familyTypeName(),

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesNodeRequest.java

@@ -39,6 +39,7 @@ class FieldCapabilitiesNodeRequest extends ActionRequest implements IndicesReque
     private final QueryBuilder indexFilter;
     private final long nowInMillis;
     private final Map<String, Object> runtimeFields;
+    private final boolean includeFieldsWithNoValue;
 
     FieldCapabilitiesNodeRequest(StreamInput in) throws IOException {
         super(in);
@@ -55,6 +56,11 @@ class FieldCapabilitiesNodeRequest extends ActionRequest implements IndicesReque
         indexFilter = in.readOptionalNamedWriteable(QueryBuilder.class);
         nowInMillis = in.readLong();
         runtimeFields = in.readGenericMap();
+        if (in.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_FIELD_HAS_VALUE)) {
+            includeFieldsWithNoValue = in.readBoolean();
+        } else {
+            includeFieldsWithNoValue = true;
+        }
     }
 
     FieldCapabilitiesNodeRequest(
@@ -65,7 +71,8 @@ class FieldCapabilitiesNodeRequest extends ActionRequest implements IndicesReque
         OriginalIndices originalIndices,
         QueryBuilder indexFilter,
         long nowInMillis,
-        Map<String, Object> runtimeFields
+        Map<String, Object> runtimeFields,
+        boolean includeFieldsWithNoValue
     ) {
         this.shardIds = Objects.requireNonNull(shardIds);
         this.fields = fields;
@@ -75,6 +82,7 @@ class FieldCapabilitiesNodeRequest extends ActionRequest implements IndicesReque
         this.indexFilter = indexFilter;
         this.nowInMillis = nowInMillis;
         this.runtimeFields = runtimeFields;
+        this.includeFieldsWithNoValue = includeFieldsWithNoValue;
     }
 
     public String[] fields() {
@@ -119,6 +127,10 @@ public long nowInMillis() {
         return nowInMillis;
     }
 
+    public boolean includeFieldsWithNoValue() {
+        return includeFieldsWithNoValue;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
@@ -132,6 +144,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalNamedWriteable(indexFilter);
         out.writeLong(nowInMillis);
         out.writeGenericMap(runtimeFields);
+        if (out.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_FIELD_HAS_VALUE)) {
+            out.writeBoolean(includeFieldsWithNoValue);
+        }
     }
 
     @Override
@@ -143,16 +158,24 @@ public ActionRequestValidationException validate() {
     public String getDescription() {
         final StringBuilder stringBuilder = new StringBuilder("shards[");
         Strings.collectionToDelimitedStringWithLimit(shardIds, ",", "", "", 1024, stringBuilder);
-        return completeDescription(stringBuilder, fields, filters, allowedTypes);
+        return completeDescription(stringBuilder, fields, filters, allowedTypes, includeFieldsWithNoValue);
     }
 
-    static String completeDescription(StringBuilder stringBuilder, String[] fields, String[] filters, String[] allowedTypes) {
+    static String completeDescription(
+        StringBuilder stringBuilder,
+        String[] fields,
+        String[] filters,
+        String[] allowedTypes,
+        boolean includeFieldsWithNoValue
+    ) {
         stringBuilder.append("], fields[");
         Strings.collectionToDelimitedStringWithLimit(Arrays.asList(fields), ",", "", "", 1024, stringBuilder);
         stringBuilder.append("], filters[");
         Strings.collectionToDelimitedString(Arrays.asList(filters), ",", "", "", stringBuilder);
         stringBuilder.append("], types[");
         Strings.collectionToDelimitedString(Arrays.asList(allowedTypes), ",", "", "", stringBuilder);
+        stringBuilder.append("], includeFieldsWithNoValue[");
+        stringBuilder.append(includeFieldsWithNoValue);
         stringBuilder.append("]");
         return stringBuilder.toString();
     }
@@ -179,12 +202,13 @@ public boolean equals(Object o) {
             && Arrays.equals(allowedTypes, that.allowedTypes)
             && Objects.equals(originalIndices, that.originalIndices)
             && Objects.equals(indexFilter, that.indexFilter)
-            && Objects.equals(runtimeFields, that.runtimeFields);
+            && Objects.equals(runtimeFields, that.runtimeFields)
+            && includeFieldsWithNoValue == that.includeFieldsWithNoValue;
     }
 
     @Override
     public int hashCode() {
-        int result = Objects.hash(originalIndices, indexFilter, nowInMillis, runtimeFields);
+        int result = Objects.hash(originalIndices, indexFilter, nowInMillis, runtimeFields, includeFieldsWithNoValue);
         result = 31 * result + shardIds.hashCode();
         result = 31 * result + Arrays.hashCode(fields);
         result = 31 * result + Arrays.hashCode(filters);

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java

@@ -42,6 +42,7 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
     private String[] filters = Strings.EMPTY_ARRAY;
     private String[] types = Strings.EMPTY_ARRAY;
     private boolean includeUnmapped = false;
+    private boolean includeFieldsWithNoValue = true;
     // pkg private API mainly for cross cluster search to signal that we do multiple reductions ie. the results should not be merged
     private boolean mergeResults = true;
     private QueryBuilder indexFilter;
@@ -62,6 +63,9 @@ public FieldCapabilitiesRequest(StreamInput in) throws IOException {
             filters = in.readStringArray();
             types = in.readStringArray();
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_FIELD_HAS_VALUE)) {
+            includeFieldsWithNoValue = in.readBoolean();
+        }
     }
 
     public FieldCapabilitiesRequest() {}
@@ -100,6 +104,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeStringArray(filters);
             out.writeStringArray(types);
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_FIELD_HAS_VALUE)) {
+            out.writeBoolean(includeFieldsWithNoValue);
+        }
     }
 
     @Override
@@ -168,6 +175,11 @@ public FieldCapabilitiesRequest includeUnmapped(boolean includeUnmapped) {
         return this;
     }
 
+    public FieldCapabilitiesRequest includeFieldsWithNoValue(boolean includeFieldsWithNoValue) {
+        this.includeFieldsWithNoValue = includeFieldsWithNoValue;
+        return this;
+    }
+
     @Override
     public String[] indices() {
         return indices;
@@ -192,6 +204,10 @@ public boolean includeUnmapped() {
         return includeUnmapped;
     }
 
+    public boolean includeFieldsWithNoValue() {
+        return includeFieldsWithNoValue;
+    }
+
     /**
      * Allows to filter indices if the provided {@link QueryBuilder} rewrites to `match_none` on every shard.
      */
@@ -247,12 +263,21 @@ public boolean equals(Object o) {
             && Objects.equals(nowInMillis, that.nowInMillis)
             && Arrays.equals(filters, that.filters)
             && Arrays.equals(types, that.types)
-            && Objects.equals(runtimeFields, that.runtimeFields);
+            && Objects.equals(runtimeFields, that.runtimeFields)
+            && includeFieldsWithNoValue == that.includeFieldsWithNoValue;
     }
 
     @Override
     public int hashCode() {
-        int result = Objects.hash(indicesOptions, includeUnmapped, mergeResults, indexFilter, nowInMillis, runtimeFields);
+        int result = Objects.hash(
+            indicesOptions,
+            includeUnmapped,
+            mergeResults,
+            indexFilter,
+            nowInMillis,
+            runtimeFields,
+            includeFieldsWithNoValue
+        );
         result = 31 * result + Arrays.hashCode(indices);
         result = 31 * result + Arrays.hashCode(fields);
         result = 31 * result + Arrays.hashCode(filters);
@@ -264,7 +289,7 @@ public int hashCode() {
     public String getDescription() {
         final StringBuilder stringBuilder = new StringBuilder("indices[");
         Strings.collectionToDelimitedStringWithLimit(Arrays.asList(indices), ",", "", "", 1024, stringBuilder);
-        return FieldCapabilitiesNodeRequest.completeDescription(stringBuilder, fields, filters, types);
+        return FieldCapabilitiesNodeRequest.completeDescription(stringBuilder, fields, filters, types, includeFieldsWithNoValue);
     }
 
     @Override

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java

@@ -32,6 +32,11 @@ public FieldCapabilitiesRequestBuilder setIncludeUnmapped(boolean includeUnmappe
         return this;
     }
 
+    public FieldCapabilitiesRequestBuilder setIncludeFieldsWithNoValue(boolean includeFieldsWithNoValue) {
+        request().includeFieldsWithNoValue(includeFieldsWithNoValue);
+        return this;
+    }
+
     public FieldCapabilitiesRequestBuilder setIndexFilter(QueryBuilder indexFilter) {
         request().indexFilter(indexFilter);
         return this;

server/src/main/java/org/elasticsearch/action/fieldcaps/RequestDispatcher.java

@@ -179,7 +179,8 @@ private void sendRequestToNode(String nodeId, List<ShardId> shardIds) {
             originalIndices,
             fieldCapsRequest.indexFilter(),
             nowInMillis,
-            fieldCapsRequest.runtimeFields()
+            fieldCapsRequest.runtimeFields(),
+            fieldCapsRequest.includeFieldsWithNoValue()
         );
         transportService.sendChildRequest(
             node,
@@ -203,7 +204,10 @@ private void afterRequestsCompleted(int numRequests) {
     private void onRequestResponse(List<ShardId> shardIds, FieldCapabilitiesNodeResponse nodeResponse) {
         for (FieldCapabilitiesIndexResponse indexResponse : nodeResponse.getIndexResponses()) {
             if (indexResponse.canMatch()) {
-                if (indexSelectors.remove(indexResponse.getIndexName()) != null) {
+                if (fieldCapsRequest.includeFieldsWithNoValue() == false) {
+                    // we accept all the responses because they may vary from node to node if we exclude empty fields
+                    onIndexResponse.accept(indexResponse);
+                } else if (indexSelectors.remove(indexResponse.getIndexName()) != null) {
                     onIndexResponse.accept(indexResponse);
                 }
             }

server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java

@@ -163,6 +163,15 @@ private void doExecuteForked(Task task, FieldCapabilitiesRequest request, final
                     resp = new FieldCapabilitiesIndexResponse(resp.getIndexName(), curr.getIndexMappingHash(), curr.get(), true);
                 }
             }
+            indexResponses.merge(resp.getIndexName(), resp, (a, b) -> {
+                if (request.includeFieldsWithNoValue() || a.get().equals(b.get())) {
+                    return a;
+                }
+                Map<String, IndexFieldCapabilities> mergedCaps = new HashMap<>(a.get());
+                mergedCaps.putAll(b.get());
+                return new FieldCapabilitiesIndexResponse(a.getIndexName(), a.getIndexMappingHash(), mergedCaps, a.canMatch());
+
+            });
             indexResponses.putIfAbsent(resp.getIndexName(), resp);
             if (fieldCapTask.isCancelled()) {
                 releaseResourcesOnCancel.run();
@@ -511,7 +520,7 @@ public void messageReceived(FieldCapabilitiesNodeRequest request, TransportChann
                 final Map<String, List<ShardId>> groupedShardIds = request.shardIds()
                     .stream()
                     .collect(Collectors.groupingBy(ShardId::getIndexName));
-                final FieldCapabilitiesFetcher fetcher = new FieldCapabilitiesFetcher(indicesService);
+                final FieldCapabilitiesFetcher fetcher = new FieldCapabilitiesFetcher(indicesService, request.includeFieldsWithNoValue());
                 final Predicate<String> fieldNameFilter = Regex.simpleMatcher(request.fields());
                 for (List<ShardId> shardIds : groupedShardIds.values()) {
                     final Map<ShardId, Exception> failures = new HashMap<>();
@@ -532,7 +541,9 @@ public void messageReceived(FieldCapabilitiesNodeRequest request, TransportChann
                                 unmatched.clear();
                                 failures.clear();
                                 allResponses.add(response);
-                                break;
+                                if (request.includeFieldsWithNoValue()) {
+                                    break;
+                                }
                             } else {
                                 unmatched.add(shardId);
                             }