Skip to content

Commit fca3fc8

Browse files
costincostin
authored
ESQL: Grammar - FROM METADATA no longer require [] (#105221)
Remove usage of [ ] through-out the grammar, in this case inside FROM METADATA.
1 parent cda94ac commit fca3fc8

File tree

25 files changed

+1059
-827
lines changed

25 files changed

+1059
-827
lines changed

‎docs/changelog/105221.yaml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
pr: 105221
2+
summary: "ESQL: Grammar - FROM METADATA no longer requires []"
3+
area: ES|QL
4+
type: breaking
5+
issues: []
6+
breaking:
7+
title: "ESQL: Grammar - FROM METADATA no longer requires []"
8+
area: REST API
9+
details: "Remove [ ] for METADATA option inside FROM command statements"
10+
impact: "Previously to return metadata fields, one had to use square brackets:\
11+
\ (eg. 'FROM index [METADATA _index]').\
12+
\ This is no longer needed: the [ ] are dropped and do not have to be specified,\
13+
\ thus simplifying the command above to:'FROM index METADATA _index'."
14+
notable: false

‎docs/reference/esql/metadata-fields.asciidoc

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@
55
<titleabbrev>Metadata fields</titleabbrev>
66
++++
77

8-
{esql} can access <<mapping-fields, metadata fields>>. The following are currently
9-
supported:
8+
{esql} can access <<mapping-fields, metadata fields>>. The currently
9+
supported ones are:
1010

11-
* <<mapping-index-field,`_index`>>: the index the document belongs to.
11+
* <<mapping-index-field,`_index`>>: the index to which the document belongs.
1212
The field is of the type <<keyword, keyword>>.
1313

1414
* <<mapping-id-field,`_id`>>: the source document's ID. The field is of the
@@ -17,18 +17,12 @@ supported:
1717
* `_version`: the source document's version. The field is of the type
1818
<<number,long>>.
1919

20-
* <<mapping-source-field,`_source`>>: the original JSON document body
21-
that was passed at index time (or a reconstructed version if
22-
<<synthetic-source, synthetic `_source_`>> is enabled). The field is
23-
loaded as a special `_source` type. This field is not supported by
24-
functions.
25-
26-
To enable access to these fields, the <<esql-from,`FROM`>> source command
27-
requires a dedicated directive:
20+
To enable the access to these fields, the <<esql-from,`FROM`>> source command needs
21+
to be provided with a dedicated directive:
2822

2923
[source,esql]
3024
----
31-
FROM index [METADATA _index, _id, _source]
25+
FROM index METADATA _index, _id
3226
----
3327

3428
Metadata fields are only available if the source of the data is an index.

‎docs/reference/esql/source-commands/from.asciidoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,9 @@ or aliases:
6666
FROM employees-00001,other-employees-*
6767
----
6868

69-
Use the `METADATA` directive to enable <<esql-metadata-fields,metadata fields>>:
69+
Use the optional `METADATA` directive to enable <<esql-metadata-fields,metadata fields>>:
7070

7171
[source,esql]
7272
----
73-
FROM employees [METADATA _id]
73+
FROM employees METADATA _id
7474
----

‎x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -161,16 +161,18 @@ static CsvSpecReader.CsvTestCase convertToRemoteIndices(CsvSpecReader.CsvTestCas
161161
String query = testCase.query;
162162
String[] commands = query.split("\\|");
163163
String first = commands[0].trim();
164+
164165
if (commands[0].toLowerCase(Locale.ROOT).startsWith("from")) {
165-
String[] parts = commands[0].split("\\[");
166+
String[] parts = commands[0].split("(?i)metadata");
166167
assert parts.length >= 1 : parts;
167168
String fromStatement = parts[0];
169+
168170
String[] localIndices = fromStatement.substring("FROM ".length()).split(",");
169171
String remoteIndices = Arrays.stream(localIndices)
170172
.map(index -> "*:" + index.trim() + "," + index.trim())
171173
.collect(Collectors.joining(","));
172-
var newFrom = "FROM " + remoteIndices + commands[0].substring(fromStatement.length());
173-
testCase.query = newFrom + " " + query.substring(first.length());
174+
var newFrom = "FROM " + remoteIndices + " " + commands[0].substring(fromStatement.length());
175+
testCase.query = newFrom + query.substring(first.length());
174176
}
175177
int offset = testCase.query.length() - query.length();
176178
if (offset != 0) {
@@ -195,7 +197,7 @@ static CsvSpecReader.CsvTestCase convertToRemoteIndices(CsvSpecReader.CsvTestCas
195197
static boolean hasIndexMetadata(String query) {
196198
String[] commands = query.split("\\|");
197199
if (commands[0].trim().toLowerCase(Locale.ROOT).startsWith("from")) {
198-
String[] parts = commands[0].split("\\[");
200+
String[] parts = commands[0].split("(?i)metadata");
199201
return parts.length > 1 && parts[1].contains("_index");
200202
}
201203
return false;

‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -441,7 +441,7 @@ public void testMetadataFieldsOnMultipleIndices() throws IOException {
441441
request.setJsonEntity("{\"a\": 3}");
442442
assertEquals(201, client().performRequest(request).getStatusLine().getStatusCode());
443443

444-
var query = fromIndex() + "* [metadata _index, _version, _id] | sort _version";
444+
var query = fromIndex() + "* metadata _index, _version, _id | sort _version";
445445
Map<String, Object> result = runEsql(new RequestObjectBuilder().query(query));
446446
var columns = List.of(
447447
Map.of("name", "a", "type", "long"),

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/id.csv-spec

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
//
2-
// Tests for _id fields
32
//
43

5-
selectAll
6-
FROM apps [metadata _id];
4+
selectAll#[skip:-8.12.99]
5+
FROM apps metadata _id;
76
ignoreOrder:true
87

98
id:integer |name:keyword |version:version | _id:keyword
@@ -23,22 +22,23 @@ id:integer |name:keyword |version:version | _id:keyword
2322
14 |mmmmm |5.2.9 | 14
2423
;
2524

26-
filterById
27-
FROM apps [metadata _id]| WHERE _id == "4";
25+
filterById#[skip:-8.12.99]
26+
FROM apps metadata _id | WHERE _id == "4";
2827

2928
id:i |name:k |version:v | _id:k
3029
4 |ddddd |2.12.0 | 4
3130
;
3231

33-
keepId
34-
FROM apps [metadata _id] | WHERE id == 3 | KEEP _id;
32+
keepId#[skip:-8.12.99]
33+
FROM apps metadata _id | WHERE id == 3 | KEEP _id;
34+
3535

3636
_id:k
3737
3
3838
;
3939

40-
idRangeAndSort
41-
FROM apps [metadata _id] | WHERE _id >= "2" AND _id <= "7" | SORT _id | keep id, name, _id;
40+
idRangeAndSort#[skip:-8.12.99]
41+
FROM apps metadata _id | WHERE _id >= "2" AND _id <= "7" | SORT _id | keep id, name, _id;
4242

4343
id:i |name:k | _id:k
4444
2 |bbbbb | 2
@@ -49,8 +49,8 @@ id:i |name:k | _id:k
4949
7 |ggggg | 7
5050
;
5151

52-
orderById
53-
FROM apps [metadata _id] | KEEP _id, name | SORT _id;
52+
orderById#[skip:-8.12.99]
53+
FROM apps metadata _id | KEEP _id, name | SORT _id;
5454

5555
_id:k | name:s
5656
1 | aaaaa
@@ -69,8 +69,8 @@ _id:k | name:s
6969
9 | iiiii
7070
;
7171

72-
orderByIdDesc
73-
FROM apps [metadata _id] | KEEP _id, name | SORT _id DESC;
72+
orderByIdDesc#[skip:-8.12.99]
73+
FROM apps metadata _id | KEEP _id, name | SORT _id DESC;
7474

7575
_id:k | name:s
7676

@@ -90,8 +90,8 @@ _id:k | name:s
9090
1 | aaaaa
9191
;
9292

93-
concatId
94-
FROM apps [metadata _id] | eval c = concat(_id, name) | SORT _id | KEEP c;
93+
concatId#[skip:-8.12.99]
94+
FROM apps metadata _id | eval c = concat(_id, name) | SORT _id | KEEP c;
9595

9696
c:k
9797
1aaaaa
@@ -110,16 +110,16 @@ c:k
110110
9iiiii
111111
;
112112

113-
statsOnId
114-
FROM apps [metadata _id] | stats c = count(_id), d = count_distinct(_id);
113+
statsOnId#[skip:-8.12.99]
114+
FROM apps metadata _id | stats c = count(_id), d = count_distinct(_id);
115115

116116
c:l | d:l
117117
14 | 14
118118
;
119119

120120

121-
statsOnIdByGroup
122-
FROM apps [metadata _id] | stats c = count(_id) by name | sort c desc, name | limit 5;
121+
statsOnIdByGroup#[skip:-8.12.99]
122+
FROM apps metadata _id | stats c = count(_id) by name | sort c desc, name | limit 5;
123123

124124
c:l | name:k
125125
2 | aaaaa

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/metadata-IT_tests_only.csv-spec

Lines changed: 32 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,32 @@
11

2-
simpleKeep
3-
from employees [metadata _index, _version] | sort emp_no | limit 2 | keep emp_no, _index, _version;
2+
simpleKeep#[skip:-8.12.99]
3+
from employees metadata _index, _version | sort emp_no | limit 2 | keep emp_no, _index, _version;
44

55
emp_no:integer |_index:keyword |_version:long
66
10001 |employees |1
77
10002 |employees |1
88
;
99

10-
aliasWithSameName
11-
from employees [metadata _index, _version] | sort emp_no | limit 2 | eval _index = _index, _version = _version | keep emp_no, _index, _version;
10+
aliasWithSameName#[skip:-8.12.99]
11+
from employees metadata _index, _version | sort emp_no | limit 2 | eval _index = _index, _version = _version | keep emp_no, _index, _version;
1212

1313
emp_no:integer |_index:keyword |_version:long
1414
10001 |employees |1
1515
10002 |employees |1
1616
;
1717

18-
inComparison
19-
from employees [metadata _index, _version] | sort emp_no | where _index == "employees" | where _version == 1 | keep emp_no | limit 2;
18+
inComparison#[skip:-8.12.99]
19+
from employees metadata _index, _version | sort emp_no | where _index == "employees" | where _version == 1 | keep emp_no | limit 2;
20+
2021

2122
emp_no:integer
2223
10001
2324
10002
2425
;
2526

26-
metaIndexInAggs
27+
metaIndexInAggs#[skip:-8.12.99]
2728
// tag::metaIndexInAggs[]
28-
FROM employees [METADATA _index, _id]
29+
FROM employees METADATA _index, _id
2930
| STATS max = MAX(emp_no) BY _index
3031
// end::metaIndexInAggs[]
3132
;
@@ -36,83 +37,84 @@ max:integer |_index:keyword
3637
// end::metaIndexInAggs-result[]
3738
;
3839

39-
metaIndexAliasedInAggs
40-
from employees [metadata _index] | eval _i = _index | stats max = max(emp_no) by _i;
40+
metaIndexAliasedInAggs#[skip:-8.12.99]
41+
from employees metadata _index | eval _i = _index | stats max = max(emp_no) by _i;
42+
4143

4244
max:integer |_i:keyword
4345
10100 |employees
4446
;
4547

46-
metaVersionInAggs
47-
from employees [metadata _version] | stats min = min(emp_no) by _version;
48+
metaVersionInAggs#[skip:-8.12.99]
49+
from employees metadata _version | stats min = min(emp_no) by _version;
4850

4951
min:integer |_version:long
5052
10001 |1
5153
;
5254

53-
metaVersionAliasedInAggs
54-
from employees [metadata _version] | eval _v = _version | stats min = min(emp_no) by _v;
55+
metaVersionAliasedInAggs#[skip:-8.12.99]
56+
from employees metadata _version | eval _v = _version | stats min = min(emp_no) by _v;
5557

5658
min:integer |_v:long
5759
10001 |1
5860
;
5961

60-
inAggsAndAsGroups
61-
from employees [metadata _index, _version] | stats max = max(_version) by _index;
62+
inAggsAndAsGroups#[skip:-8.12.99]
63+
from employees metadata _index, _version | stats max = max(_version) by _index;
6264

6365
max:long |_index:keyword
6466
1 |employees
6567
;
6668

67-
inAggsAndAsGroupsAliased
68-
from employees [metadata _index, _version] | eval _i = _index, _v = _version | stats max = max(_v) by _i;
69+
inAggsAndAsGroupsAliased#[skip:-8.12.99]
70+
from employees metadata _index, _version | eval _i = _index, _v = _version | stats max = max(_v) by _i;
6971

7072
max:long |_i:keyword
7173
1 |employees
7274
;
7375

74-
inFunction
75-
from employees [metadata _index, _version] | sort emp_no | where length(_index) == length("employees") | where abs(_version) == 1 | keep emp_no | limit 2;
76+
inFunction#[skip:-8.12.99]
77+
from employees metadata _index, _version | sort emp_no | where length(_index) == length("employees") | where abs(_version) == 1 | keep emp_no | limit 2;
7678

7779
emp_no:integer
7880
10001
7981
10002
8082
;
8183

82-
inArithmetics
83-
from employees [metadata _index, _version] | eval i = _version + 2 | stats min = min(emp_no) by i;
84+
inArithmetics#[skip:-8.12.99]
85+
from employees metadata _index, _version | eval i = _version + 2 | stats min = min(emp_no) by i;
8486

8587
min:integer |i:long
8688
10001 |3
8789
;
8890

89-
inSort
90-
from employees [metadata _index, _version] | sort _version, _index, emp_no | keep emp_no, _version, _index | limit 2;
91+
inSort#[skip:-8.12.99]
92+
from employees metadata _index, _version | sort _version, _index, emp_no | keep emp_no, _version, _index | limit 2;
9193

9294
emp_no:integer |_version:long |_index:keyword
9395
10001 |1 |employees
9496
10002 |1 |employees
9597
;
9698

97-
withMvFunction
98-
from employees [metadata _version] | eval i = mv_avg(_version) + 2 | stats min = min(emp_no) by i;
99+
withMvFunction#[skip:-8.12.99]
100+
from employees metadata _version | eval i = mv_avg(_version) + 2 | stats min = min(emp_no) by i;
99101

100102
min:integer |i:double
101103
10001 |3.0
102104
;
103105

104-
overwritten
105-
from employees [metadata _index, _version] | sort emp_no | eval _index = 3, _version = "version" | keep emp_no, _index, _version | limit 3;
106+
overwritten#[skip:-8.12.99]
107+
from employees metadata _index, _version | sort emp_no | eval _index = 3, _version = "version" | keep emp_no, _index, _version | limit 3;
106108

107109
emp_no:integer |_index:integer |_version:keyword
108110
10001 |3 |version
109111
10002 |3 |version
110112
10003 |3 |version
111113
;
112114

113-
multipleIndices
115+
multipleIndices#[skip:-8.12.99]
114116
// tag::multipleIndices[]
115-
FROM ul_logs, apps [METADATA _index, _version]
117+
FROM ul_logs, apps METADATA _index, _version
116118
| WHERE id IN (13, 14) AND _version == 1
117119
| EVAL key = CONCAT(_index, "_", TO_STR(id))
118120
| SORT id, _index

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ public void testSimple() {
114114
}
115115

116116
public void testMetadataIndex() {
117-
try (EsqlQueryResponse resp = runQuery("FROM logs*,*:logs* [METADATA _index] | stats sum(v) by _index | sort _index")) {
117+
try (EsqlQueryResponse resp = runQuery("FROM logs*,*:logs* METADATA _index | stats sum(v) by _index | sort _index")) {
118118
List<List<Object>> values = getValuesList(resp);
119119
assertThat(values.get(0), equalTo(List.of(285L, "cluster-a:logs-2")));
120120
assertThat(values.get(1), equalTo(List.of(45L, "logs-1")));

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1222,7 +1222,7 @@ public void testGroupingMultiValueByOrdinals() {
12221222
}
12231223

12241224
public void testLoadId() {
1225-
try (EsqlQueryResponse results = run("from test [metadata _id] | keep _id | sort _id ")) {
1225+
try (EsqlQueryResponse results = run("from test metadata _id | keep _id | sort _id ")) {
12261226
assertThat(results.columns(), equalTo(List.of(new ColumnInfo("_id", "keyword"))));
12271227
ListMatcher values = matchesList();
12281228
for (int i = 10; i < 50; i++) {
@@ -1427,7 +1427,7 @@ public void testQueryOnEmptyMappingIndex() {
14271427

14281428
assertEmptyIndexQueries(from);
14291429

1430-
try (EsqlQueryResponse resp = run(from + "[METADATA _source] | EVAL x = 123")) {
1430+
try (EsqlQueryResponse resp = run(from + "METADATA _source | EVAL x = 123")) {
14311431
assertFalse(resp.values().hasNext());
14321432
assertThat(resp.columns(), equalTo(List.of(new ColumnInfo("_source", "_source"), new ColumnInfo("x", "integer"))));
14331433
}
@@ -1455,7 +1455,7 @@ public void testQueryOnEmptyDataIndex() {
14551455

14561456
assertEmptyIndexQueries(from);
14571457

1458-
try (EsqlQueryResponse resp = run(from + "[METADATA _source] | EVAL x = 123")) {
1458+
try (EsqlQueryResponse resp = run(from + "METADATA _source | EVAL x = 123")) {
14591459
assertFalse(resp.values().hasNext());
14601460
assertThat(
14611461
resp.columns(),
@@ -1470,7 +1470,7 @@ public void testQueryOnEmptyDataIndex() {
14701470
}
14711471

14721472
private void assertEmptyIndexQueries(String from) {
1473-
try (EsqlQueryResponse resp = run(from + "[METADATA _source] | KEEP _source | LIMIT 1")) {
1473+
try (EsqlQueryResponse resp = run(from + "METADATA _source | KEEP _source | LIMIT 1")) {
14741474
assertFalse(resp.values().hasNext());
14751475
assertThat(resp.columns(), equalTo(List.of(new ColumnInfo("_source", "_source"))));
14761476
}

0 commit comments

Comments
 (0)