Skip to content

Commit d7ea7ae

Browse files
elasticsearchmachineelasticsearchmachine
committed
Merge remote-tracking branch 'origin/main' into lucene_snapshot
2 parents b7476f5 + 8927a1a commit d7ea7ae

File tree

83 files changed

+967
-755
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

83 files changed

+967
-755
lines changed

‎distribution/docker/build.gradle

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -396,7 +396,7 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
396396

397397
dockerContext.fileProvider(transformTask.map { Sync task -> task.getDestinationDir() })
398398

399-
noCache = BuildParams.isCi
399+
noCache = BuildParams.isCi()
400400
tags = generateTags(base, architecture)
401401
platforms.add(architecture.dockerPlatform)
402402

@@ -465,7 +465,7 @@ void addBuildEssDockerImageTask(Architecture architecture) {
465465

466466
dockerContext.fileProvider(buildContextTask.map { it.getDestinationDir() })
467467

468-
noCache = BuildParams.isCi
468+
noCache = BuildParams.isCi()
469469
baseImages = []
470470
tags = generateTags(base, architecture)
471471
platforms.add(architecture.dockerPlatform)

‎docs/changelog/104573.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 104573
2+
summary: Fix logger Strings.format calls
3+
area: Distributed
4+
type: bug
5+
issues: []

‎docs/changelog/104721.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 104721
2+
summary: Add default rollover conditions to ILM explain API response
3+
area: ILM+SLM
4+
type: enhancement
5+
issues:
6+
- 103395

‎docs/changelog/104802.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 104802
2+
summary: "[Connectors API] Fix bug when triggering a sync job via API"
3+
area: Application
4+
type: bug
5+
issues: []

‎docs/reference/ilm/apis/explain.asciidoc

Lines changed: 25 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ GET /_cluster/health?wait_for_status=green&timeout=10s
9898

9999
[source,console]
100100
--------------------------------------------------
101-
GET my-index-000001/_ilm/explain
101+
GET my-index-000001/_ilm/explain?human
102102
--------------------------------------------------
103103
// TEST[continued]
104104

@@ -111,18 +111,23 @@ that the index is managed and in the `new` phase:
111111
"indices": {
112112
"my-index-000001": {
113113
"index": "my-index-000001",
114-
"index_creation_date_millis": 1538475653281, <1>
115-
"time_since_index_creation": "15s", <2>
116-
"managed": true, <3>
117-
"policy": "my_policy", <4>
118-
"lifecycle_date_millis": 1538475653281, <5>
119-
"age": "15s", <6>
114+
"index_creation_date_millis": 1538475653281, <1>
115+
"index_creation_date": "2018-10-15T13:45:21.981Z",
116+
"time_since_index_creation": "15s", <2>
117+
"managed": true, <3>
118+
"policy": "my_policy", <4>
119+
"lifecycle_date_millis": 1538475653281, <5>
120+
"lifecycle_date": "2018-10-15T13:45:21.981Z",
121+
"age": "15s", <6>
120122
"phase": "new",
121-
"phase_time_millis": 1538475653317, <7>
123+
"phase_time_millis": 1538475653317, <7>
124+
"phase_time": "2018-10-15T13:45:22.577Z",
122125
"action": "complete"
123-
"action_time_millis": 1538475653317, <8>
126+
"action_time_millis": 1538475653317, <8>
127+
"action_time": "2018-10-15T13:45:22.577Z",
124128
"step": "complete",
125-
"step_time_millis": 1538475653317 <9>
129+
"step_time_millis": 1538475653317, <9>
130+
"step_time": "2018-10-15T13:45:22.577Z"
126131
}
127132
}
128133
}
@@ -175,13 +180,15 @@ phase completes.
175180
"min_age": "0ms",
176181
"actions": {
177182
"rollover": {
178-
"max_age": "30s"
183+
"max_age": "30s",
184+
"max_primary_shard_docs": 200000000, <2>
185+
"min_docs": 1
179186
}
180187
}
181188
},
182-
"version": 3, <2>
183-
"modified_date": "2018-10-15T13:21:41.576Z", <3>
184-
"modified_date_in_millis": 1539609701576 <4>
189+
"version": 3, <3>
190+
"modified_date": "2018-10-15T13:21:41.576Z", <4>
191+
"modified_date_in_millis": 1539609701576 <5>
185192
}
186193
}
187194
}
@@ -191,9 +198,10 @@ phase completes.
191198

192199
<1> The JSON phase definition loaded from the specified policy when the index
193200
entered this phase
194-
<2> The version of the policy that was loaded
195-
<3> The date the loaded policy was last modified
196-
<4> The epoch time when the loaded policy was last modified
201+
<2> The rollover action includes the default `max_primary_shard_docs` and `min_docs` conditions. See <<ilm-rollover-options,ILM Rollover Options>> for more information.
202+
<3> The version of the policy that was loaded
203+
<4> The date the loaded policy was last modified
204+
<5> The epoch time when the loaded policy was last modified
197205

198206
If {ilm-init} is waiting for a step to complete, the response includes status
199207
information for the step that's being performed on the index.

‎docs/reference/ml/ml-shared.asciidoc

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -430,16 +430,17 @@ end::daily-model-snapshot-retention-after-days[]
430430

431431
tag::data-description[]
432432
The data description defines the format of the input data when you send data to
433-
the job by using the <<ml-post-data,post data>> API. Note that when configure
434-
a {dfeed}, these properties are automatically set. When data is received via
435-
the <<ml-post-data,post data>> API, it is not stored in {es}. Only the results
436-
for {anomaly-detect} are retained.
433+
the job by using the <<ml-post-data,post data>> API. Note that when using a
434+
{dfeed}, only the `time_field` needs to be set, the rest of the properties are
435+
automatically set. When data is received via the <<ml-post-data,post data>> API,
436+
it is not stored in {es}. Only the results for {anomaly-detect} are retained.
437437
+
438438
.Properties of `data_description`
439439
[%collapsible%open]
440440
====
441441
`format`:::
442-
(string) Only `JSON` format is supported at this time.
442+
(string) Only `xcontent` format is supported at this time, and this is the
443+
default value.
443444

444445
`time_field`:::
445446
(string) The name of the field that contains the timestamp.

‎modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import io.netty.util.concurrent.Future;
2626
import io.netty.util.concurrent.PromiseCombiner;
2727

28+
import org.apache.logging.log4j.LogManager;
2829
import org.apache.logging.log4j.Logger;
2930
import org.elasticsearch.ExceptionsHelper;
3031
import org.elasticsearch.common.bytes.ReleasableBytesReference;
@@ -52,7 +53,7 @@
5253
*/
5354
public class Netty4HttpPipeliningHandler extends ChannelDuplexHandler {
5455

55-
private final Logger logger;
56+
private static final Logger logger = LogManager.getLogger(Netty4HttpPipeliningHandler.class);
5657

5758
private final int maxEventsHeld;
5859
private final PriorityQueue<Tuple<? extends Netty4HttpResponse, ChannelPromise>> outboundHoldingQueue;
@@ -86,12 +87,10 @@ private record ChunkedWrite(PromiseCombiner combiner, ChannelPromise onDone, Chu
8687
/**
8788
* Construct a new pipelining handler; this handler should be used downstream of HTTP decoding/aggregation.
8889
*
89-
* @param logger for logging unexpected errors
9090
* @param maxEventsHeld the maximum number of channel events that will be retained prior to aborting the channel connection; this is
9191
* required as events cannot queue up indefinitely
9292
*/
93-
public Netty4HttpPipeliningHandler(Logger logger, final int maxEventsHeld, final Netty4HttpServerTransport serverTransport) {
94-
this.logger = logger;
93+
public Netty4HttpPipeliningHandler(final int maxEventsHeld, final Netty4HttpServerTransport serverTransport) {
9594
this.maxEventsHeld = maxEventsHeld;
9695
this.outboundHoldingQueue = new PriorityQueue<>(1, Comparator.comparingInt(t -> t.v1().getSequence()));
9796
this.serverTransport = serverTransport;

‎modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -430,7 +430,7 @@ protected boolean isContentAlwaysEmpty(HttpResponse msg) {
430430
if (handlingSettings.compression()) {
431431
ch.pipeline().addLast("encoder_compress", new HttpContentCompressor(handlingSettings.compressionLevel()));
432432
}
433-
ch.pipeline().addLast("pipelining", new Netty4HttpPipeliningHandler(logger, transport.pipeliningMaxEvents, transport));
433+
ch.pipeline().addLast("pipelining", new Netty4HttpPipeliningHandler(transport.pipeliningMaxEvents, transport));
434434
transport.serverAcceptedChannel(nettyHttpChannel);
435435
}
436436

‎modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandlerTests.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ public void testThatPipeliningWorksWithFastSerializedRequests() throws Interrupt
119119
}
120120

121121
private EmbeddedChannel makeEmbeddedChannelWithSimulatedWork(int numberOfRequests) {
122-
return new EmbeddedChannel(new Netty4HttpPipeliningHandler(logger, numberOfRequests, null) {
122+
return new EmbeddedChannel(new Netty4HttpPipeliningHandler(numberOfRequests, null) {
123123
@Override
124124
protected void handlePipelinedRequest(ChannelHandlerContext ctx, Netty4HttpRequest pipelinedRequest) {
125125
ctx.fireChannelRead(pipelinedRequest);
@@ -185,7 +185,7 @@ public void testThatPipeliningClosesConnectionWithTooManyEvents() throws Interru
185185

186186
public void testPipeliningRequestsAreReleased() {
187187
final int numberOfRequests = 10;
188-
final EmbeddedChannel embeddedChannel = new EmbeddedChannel(new Netty4HttpPipeliningHandler(logger, numberOfRequests + 1, null));
188+
final EmbeddedChannel embeddedChannel = new EmbeddedChannel(new Netty4HttpPipeliningHandler(numberOfRequests + 1, null));
189189

190190
for (int i = 0; i < numberOfRequests; i++) {
191191
embeddedChannel.writeInbound(createHttpRequest("/" + i));
@@ -493,7 +493,7 @@ private static void assertDoneWithClosedChannel(ChannelPromise chunkedWritePromi
493493
}
494494

495495
private Netty4HttpPipeliningHandler getTestHttpHandler() {
496-
return new Netty4HttpPipeliningHandler(logger, Integer.MAX_VALUE, mock(Netty4HttpServerTransport.class)) {
496+
return new Netty4HttpPipeliningHandler(Integer.MAX_VALUE, mock(Netty4HttpServerTransport.class)) {
497497
@Override
498498
protected void handlePipelinedRequest(ChannelHandlerContext ctx, Netty4HttpRequest pipelinedRequest) {
499499
ctx.fireChannelRead(pipelinedRequest);

‎server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/PutMappingClusterStateUpdateRequest.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
public class PutMappingClusterStateUpdateRequest extends IndicesClusterStateUpdateRequest<PutMappingClusterStateUpdateRequest> {
2020

2121
private final CompressedXContent source;
22+
private boolean autoUpdate;
2223

2324
public PutMappingClusterStateUpdateRequest(String source) throws IOException {
2425
this.source = CompressedXContent.fromJSON(source);
@@ -28,4 +29,12 @@ public CompressedXContent source() {
2829
return source;
2930
}
3031

32+
public PutMappingClusterStateUpdateRequest autoUpdate(boolean autoUpdate) {
33+
this.autoUpdate = autoUpdate;
34+
return this;
35+
}
36+
37+
public boolean autoUpdate() {
38+
return autoUpdate;
39+
}
3140
}

‎server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ protected void masterOperation(
9393
return;
9494
}
9595

96-
performMappingUpdate(concreteIndices, request, listener, metadataMappingService);
96+
performMappingUpdate(concreteIndices, request, listener, metadataMappingService, true);
9797
}
9898

9999
}

‎server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ protected void masterOperation(
112112
return;
113113
}
114114

115-
performMappingUpdate(concreteIndices, request, listener, metadataMappingService);
115+
performMappingUpdate(concreteIndices, request, listener, metadataMappingService, false);
116116
} catch (IndexNotFoundException ex) {
117117
logger.debug(() -> "failed to put mappings on indices [" + Arrays.asList(request.indices() + "]"), ex);
118118
throw ex;
@@ -147,7 +147,8 @@ static void performMappingUpdate(
147147
Index[] concreteIndices,
148148
PutMappingRequest request,
149149
ActionListener<AcknowledgedResponse> listener,
150-
MetadataMappingService metadataMappingService
150+
MetadataMappingService metadataMappingService,
151+
boolean autoUpdate
151152
) {
152153
final ActionListener<AcknowledgedResponse> wrappedListener = listener.delegateResponse((l, e) -> {
153154
logger.debug(() -> "failed to put mappings on indices [" + Arrays.asList(concreteIndices) + "]", e);
@@ -157,7 +158,8 @@ static void performMappingUpdate(
157158
try {
158159
updateRequest = new PutMappingClusterStateUpdateRequest(request.source()).indices(concreteIndices)
159160
.ackTimeout(request.timeout())
160-
.masterNodeTimeout(request.masterNodeTimeout());
161+
.masterNodeTimeout(request.masterNodeTimeout())
162+
.autoUpdate(autoUpdate);
161163
} catch (IOException e) {
162164
wrappedListener.onFailure(e);
163165
return;

‎server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,7 @@ static boolean executeBulkItemRequest(
377377
.merge(
378378
MapperService.SINGLE_MAPPING_NAME,
379379
new CompressedXContent(result.getRequiredMappingUpdate()),
380-
MapperService.MergeReason.MAPPING_UPDATE_PREFLIGHT
380+
MapperService.MergeReason.MAPPING_AUTO_UPDATE_PREFLIGHT
381381
)
382382
).map(DocumentMapper::mappingSource);
383383
Optional<CompressedXContent> previousSource = Optional.ofNullable(primary.mapperService().documentMapper())

‎server/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ private static <T> void addAllSynchronized(List<T> allResults, Collection<T> res
134134

135135
@Override
136136
protected void onItemFailure(String nodeId, Exception e) {
137-
logger.debug(() -> Strings.format("failed to execute on node [{}]", nodeId), e);
137+
logger.debug(() -> Strings.format("failed to execute on node [%s]", nodeId), e);
138138
synchronized (failedNodeExceptions) {
139139
failedNodeExceptions.add(new FailedNodeException(nodeId, "Failed node [" + nodeId + "]", e));
140140
}

‎server/src/main/java/org/elasticsearch/cluster/coordination/LeaderChecker.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
import org.elasticsearch.ExceptionsHelper;
1414
import org.elasticsearch.cluster.node.DiscoveryNode;
1515
import org.elasticsearch.cluster.node.DiscoveryNodes;
16-
import org.elasticsearch.common.Strings;
1716
import org.elasticsearch.common.io.stream.StreamInput;
1817
import org.elasticsearch.common.io.stream.StreamOutput;
1918
import org.elasticsearch.common.settings.Setting;
@@ -163,7 +162,7 @@ void updateLeader(@Nullable final DiscoveryNode leader) {
163162
*/
164163
void setCurrentNodes(DiscoveryNodes discoveryNodes) {
165164
// Sorting the nodes for deterministic logging until https://github.com/elastic/elasticsearch/issues/94946 is fixed
166-
logger.trace(() -> Strings.format("setCurrentNodes: {}", discoveryNodes.mastersFirstStream().toList()));
165+
logger.trace(() -> format("setCurrentNodes: %s", discoveryNodes.mastersFirstStream().toList()));
167166
this.discoveryNodes = discoveryNodes;
168167
}
169168

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,11 @@ private static ClusterState applyRequest(
148148
// try and parse it (no need to add it here) so we can bail early in case of parsing exception
149149
// first, simulate: just call merge and ignore the result
150150
Mapping mapping = mapperService.parseMapping(MapperService.SINGLE_MAPPING_NAME, mappingUpdateSource);
151-
MapperService.mergeMappings(mapperService.documentMapper(), mapping, MergeReason.MAPPING_UPDATE);
151+
MapperService.mergeMappings(
152+
mapperService.documentMapper(),
153+
mapping,
154+
request.autoUpdate() ? MergeReason.MAPPING_AUTO_UPDATE : MergeReason.MAPPING_UPDATE
155+
);
152156
}
153157
Metadata.Builder builder = Metadata.builder(metadata);
154158
boolean updated = false;
@@ -167,7 +171,7 @@ private static ClusterState applyRequest(
167171
DocumentMapper mergedMapper = mapperService.merge(
168172
MapperService.SINGLE_MAPPING_NAME,
169173
mappingUpdateSource,
170-
MergeReason.MAPPING_UPDATE
174+
request.autoUpdate() ? MergeReason.MAPPING_AUTO_UPDATE : MergeReason.MAPPING_UPDATE
171175
);
172176
CompressedXContent updatedSource = mergedMapper.mappingSource();
173177

‎server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,23 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
5656
*/
5757
public enum MergeReason {
5858
/**
59-
* Pre-flight check before sending a mapping update to the master
59+
* Pre-flight check before sending a dynamic mapping update to the master
6060
*/
61-
MAPPING_UPDATE_PREFLIGHT,
61+
MAPPING_AUTO_UPDATE_PREFLIGHT {
62+
@Override
63+
public boolean isAutoUpdate() {
64+
return true;
65+
}
66+
},
67+
/**
68+
* Dynamic mapping updates
69+
*/
70+
MAPPING_AUTO_UPDATE {
71+
@Override
72+
public boolean isAutoUpdate() {
73+
return true;
74+
}
75+
},
6276
/**
6377
* Create or update a mapping.
6478
*/
@@ -72,7 +86,11 @@ public enum MergeReason {
7286
* if a shard was moved to a different node or for administrative
7387
* purposes.
7488
*/
75-
MAPPING_RECOVERY
89+
MAPPING_RECOVERY;
90+
91+
public boolean isAutoUpdate() {
92+
return false;
93+
}
7694
}
7795

7896
public static final String SINGLE_MAPPING_NAME = "_doc";
@@ -364,7 +382,7 @@ boolean assertNoUpdateRequired(final IndexMetadata newIndexMetadata) {
364382
}
365383

366384
public void merge(IndexMetadata indexMetadata, MergeReason reason) {
367-
assert reason != MergeReason.MAPPING_UPDATE_PREFLIGHT;
385+
assert reason != MergeReason.MAPPING_AUTO_UPDATE_PREFLIGHT;
368386
MappingMetadata mappingMetadata = indexMetadata.mapping();
369387
if (mappingMetadata != null) {
370388
merge(mappingMetadata.type(), mappingMetadata.source(), reason);
@@ -521,7 +539,7 @@ private synchronized DocumentMapper doMerge(String type, MergeReason reason, Map
521539
// TODO: In many cases the source here is equal to mappingSource so we need not serialize again.
522540
// We should identify these cases reliably and save expensive serialization here
523541
DocumentMapper newMapper = newDocumentMapper(mapping, reason, mapping.toCompressedXContent());
524-
if (reason == MergeReason.MAPPING_UPDATE_PREFLIGHT) {
542+
if (reason == MergeReason.MAPPING_AUTO_UPDATE_PREFLIGHT) {
525543
return newMapper;
526544
}
527545
this.mapper = newMapper;

‎server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ public void addDynamicMappingsUpdate(Mapping update) {
160160
if (dynamicMappingsUpdate == null) {
161161
dynamicMappingsUpdate = update;
162162
} else {
163-
dynamicMappingsUpdate = dynamicMappingsUpdate.merge(update, MergeReason.MAPPING_UPDATE, Long.MAX_VALUE);
163+
dynamicMappingsUpdate = dynamicMappingsUpdate.merge(update, MergeReason.MAPPING_AUTO_UPDATE, Long.MAX_VALUE);
164164
}
165165
}
166166

0 commit comments

Comments
 (0)