ESQL: Grammar - FROM METADATA no longer require []

Remove usage of [ ] through-out the grammar, in this case inside
 FROM METADATA.

docs/reference/esql/metadata-fields.asciidoc

@@ -22,7 +22,7 @@ to be provided with a dedicated directive:
 
 [source,esql]
 ----
-FROM index [METADATA _index, _id]
+FROM index METADATA _index, _id
 ----
 
 Metadata fields are only available if the source of the data is an index.

docs/reference/esql/source-commands/from.asciidoc

@@ -66,9 +66,9 @@ or aliases:
 FROM employees-00001,other-employees-*
 ----
 
-Use the `METADATA` directive to enable <<esql-metadata-fields,metadata fields>>:
+Use the optional `METADATA` directive to enable <<esql-metadata-fields,metadata fields>>:
 
 [source,esql]
 ----
-FROM employees [METADATA _id]
+FROM employees METADATA _id
 ----

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java

@@ -441,7 +441,7 @@ public void testMetadataFieldsOnMultipleIndices() throws IOException {
         request.setJsonEntity("{\"a\": 3}");
         assertEquals(201, client().performRequest(request).getStatusLine().getStatusCode());
 
-        var query = fromIndex() + "* [metadata _index, _version, _id] | sort _version";
+        var query = fromIndex() + "* metadata _index, _version, _id | sort _version";
         Map<String, Object> result = runEsql(new RequestObjectBuilder().query(query));
         var columns = List.of(
             Map.of("name", "a", "type", "long"),

x-pack/plugin/esql/qa/testFixtures/src/main/resources/id.csv-spec

@@ -3,7 +3,7 @@
 //
 
 selectAll
-FROM apps [metadata _id];
+FROM apps metadata _id;
 ignoreOrder:true
 
 id:integer |name:keyword   |version:version     | _id:keyword
@@ -24,21 +24,21 @@ id:integer |name:keyword   |version:version     | _id:keyword
 ;
 
 filterById
-FROM apps [metadata _id]| WHERE _id == "4";
+FROM apps metadata _id| WHERE _id == "4";
 
 id:i |name:k |version:v | _id:k
 4    |ddddd  |2.12.0    | 4
 ;
 
 keepId
-FROM apps [metadata _id] | WHERE id == 3 | KEEP _id;
+FROM apps metadata _id | WHERE id == 3 | KEEP _id;
 
 _id:k
 3
 ;
 
 idRangeAndSort
-FROM apps [metadata _id] | WHERE _id >= "2" AND _id <= "7" | SORT _id | keep id, name, _id;
+FROM apps metadata _id | WHERE _id >= "2" AND _id <= "7" | SORT _id | keep id, name, _id;
 
 id:i    |name:k       | _id:k
 2       |bbbbb        | 2
@@ -50,7 +50,7 @@ id:i    |name:k       | _id:k
 ;
 
 orderById
-FROM apps [metadata _id] | KEEP _id, name | SORT _id;
+FROM apps metadata _id | KEEP _id, name | SORT _id;
 
 _id:k  | name:s
 1      | aaaaa
@@ -70,7 +70,7 @@ _id:k  | name:s
 ;
 
 orderByIdDesc
-FROM apps [metadata _id] | KEEP _id, name | SORT _id DESC;
+FROM apps metadata _id | KEEP _id, name | SORT _id DESC;
 
 _id:k  | name:s
 
@@ -91,7 +91,7 @@ _id:k  | name:s
 ;
 
 concatId
-FROM apps [metadata _id] | eval c = concat(_id, name) | SORT _id | KEEP c;
+FROM apps metadata _id | eval c = concat(_id, name) | SORT _id | KEEP c;
 
 c:k
 1aaaaa
@@ -111,15 +111,15 @@ c:k
 ;
 
 statsOnId
-FROM apps [metadata _id] | stats c = count(_id), d = count_distinct(_id);
+FROM apps metadata _id | stats c = count(_id), d = count_distinct(_id);
 
 c:l | d:l
 14  | 14
 ;
 
 
 statsOnIdByGroup
-FROM apps [metadata _id] | stats c = count(_id) by name | sort c desc, name | limit 5;
+FROM apps metadata _id | stats c = count(_id) by name | sort c desc, name | limit 5;
 
 c:l | name:k
 2   | aaaaa

x-pack/plugin/esql/qa/testFixtures/src/main/resources/metadata-IT_tests_only.csv-spec

@@ -1,22 +1,22 @@
 
 simpleKeep
-from employees [metadata _index, _version] | sort emp_no | limit 2 | keep emp_no, _index, _version;
+from employees metadata _index, _version | sort emp_no | limit 2 | keep emp_no, _index, _version;
 
 emp_no:integer |_index:keyword |_version:long
 10001          |employees      |1
 10002          |employees      |1
 ;
 
 aliasWithSameName
-from employees [metadata _index, _version] | sort emp_no | limit 2 | eval _index = _index, _version = _version | keep emp_no, _index, _version;
+from employees metadata _index, _version | sort emp_no | limit 2 | eval _index = _index, _version = _version | keep emp_no, _index, _version;
 
 emp_no:integer |_index:keyword |_version:long
 10001          |employees      |1
 10002          |employees      |1
 ;
 
 inComparison
-from employees [metadata _index, _version] | sort emp_no | where _index == "employees" | where _version == 1 | keep emp_no | limit 2;
+from employees metadata _index, _version | sort emp_no | where _index == "employees" | where _version == 1 | keep emp_no | limit 2;
 
 emp_no:integer
 10001
@@ -25,7 +25,7 @@ emp_no:integer
 
 metaIndexInAggs
 // tag::metaIndexInAggs[]
-FROM employees [METADATA _index, _id]
+FROM employees METADATA _index, _id
 | STATS max = MAX(emp_no) BY _index
 // end::metaIndexInAggs[]
 ;
@@ -37,72 +37,72 @@ max:integer |_index:keyword
 ;
 
 metaIndexAliasedInAggs
-from employees [metadata _index] | eval _i = _index | stats max = max(emp_no) by _i;
+from employees metadata _index | eval _i = _index | stats max = max(emp_no) by _i;
 
 max:integer |_i:keyword
 10100       |employees
 ;
 
 metaVersionInAggs
-from employees [metadata _version] | stats min = min(emp_no) by _version;
+from employees metadata _version | stats min = min(emp_no) by _version;
 
 min:integer |_version:long
 10001       |1
 ;
 
 metaVersionAliasedInAggs
-from employees [metadata _version] | eval _v = _version | stats min = min(emp_no) by _v;
+from employees metadata _version | eval _v = _version | stats min = min(emp_no) by _v;
 
 min:integer |_v:long
 10001       |1
 ;
 
 inAggsAndAsGroups
-from employees [metadata _index, _version] | stats max = max(_version) by _index;
+from employees metadata _index, _version | stats max = max(_version) by _index;
 
 max:long |_index:keyword
 1        |employees
 ;
 
 inAggsAndAsGroupsAliased
-from employees [metadata _index, _version] | eval _i = _index, _v = _version | stats max = max(_v) by _i;
+from employees metadata _index, _version | eval _i = _index, _v = _version | stats max = max(_v) by _i;
 
 max:long |_i:keyword
 1        |employees
 ;
 
 inFunction
-from employees [metadata _index, _version] | sort emp_no | where length(_index) == length("employees") | where abs(_version) == 1 | keep emp_no | limit 2;
+from employees metadata _index, _version | sort emp_no | where length(_index) == length("employees") | where abs(_version) == 1 | keep emp_no | limit 2;
 
 emp_no:integer
 10001
 10002
 ;
 
 inArithmetics
-from employees [metadata _index, _version] | eval i = _version + 2 | stats min = min(emp_no) by i;
+from employees metadata _index, _version | eval i = _version + 2 | stats min = min(emp_no) by i;
 
 min:integer |i:long
 10001       |3
 ;
 
 inSort
-from employees [metadata _index, _version] | sort _version, _index, emp_no | keep emp_no, _version, _index | limit 2;
+from employees metadata _index, _version | sort _version, _index, emp_no | keep emp_no, _version, _index | limit 2;
 
 emp_no:integer |_version:long |_index:keyword
 10001          |1             |employees
 10002          |1             |employees
 ;
 
 withMvFunction
-from employees [metadata _version] | eval i = mv_avg(_version) + 2 | stats min = min(emp_no) by i;
+from employees metadata _version | eval i = mv_avg(_version) + 2 | stats min = min(emp_no) by i;
 
 min:integer |i:double
 10001       |3.0
 ;
 
 overwritten
-from employees [metadata _index, _version] | sort emp_no | eval _index = 3, _version = "version" | keep emp_no, _index, _version | limit 3;
+from employees metadata _index, _version | sort emp_no | eval _index = 3, _version = "version" | keep emp_no, _index, _version | limit 3;
 
 emp_no:integer |_index:integer |_version:keyword
 10001          |3              |version
@@ -112,7 +112,7 @@ emp_no:integer |_index:integer |_version:keyword
 
 multipleIndices
 // tag::multipleIndices[]
-FROM ul_logs, apps [METADATA _index, _version]
+FROM ul_logs, apps metadata _index, _version
 | WHERE id IN (13, 14) AND _version == 1
 | EVAL key = CONCAT(_index, "_", TO_STR(id))
 | SORT id, _index

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java

@@ -114,7 +114,7 @@ public void testSimple() {
     }
 
     public void testMetadataIndex() {
-        try (EsqlQueryResponse resp = runQuery("FROM logs*,*:logs* [METADATA _index] | stats sum(v) by _index | sort _index")) {
+        try (EsqlQueryResponse resp = runQuery("FROM logs*,*:logs* METADATA _index | stats sum(v) by _index | sort _index")) {
             List<List<Object>> values = getValuesList(resp);
             assertThat(values.get(0), equalTo(List.of(285L, "cluster-a:logs-2")));
             assertThat(values.get(1), equalTo(List.of(45L, "logs-1")));

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java

@@ -1222,7 +1222,7 @@ public void testGroupingMultiValueByOrdinals() {
     }
 
     public void testLoadId() {
-        try (EsqlQueryResponse results = run("from test [metadata _id] | keep _id | sort _id ")) {
+        try (EsqlQueryResponse results = run("from test metadata _id | keep _id | sort _id ")) {
             assertThat(results.columns(), equalTo(List.of(new ColumnInfo("_id", "keyword"))));
             ListMatcher values = matchesList();
             for (int i = 10; i < 50; i++) {
@@ -1427,7 +1427,7 @@ public void testQueryOnEmptyMappingIndex() {
 
         assertEmptyIndexQueries(from);
 
-        try (EsqlQueryResponse resp = run(from + "[METADATA _source] | EVAL x = 123")) {
+        try (EsqlQueryResponse resp = run(from + "METADATA _source | EVAL x = 123")) {
             assertFalse(resp.values().hasNext());
             assertThat(resp.columns(), equalTo(List.of(new ColumnInfo("_source", "_source"), new ColumnInfo("x", "integer"))));
         }
@@ -1455,7 +1455,7 @@ public void testQueryOnEmptyDataIndex() {
 
         assertEmptyIndexQueries(from);
 
-        try (EsqlQueryResponse resp = run(from + "[METADATA _source] | EVAL x = 123")) {
+        try (EsqlQueryResponse resp = run(from + "METADATA _source | EVAL x = 123")) {
             assertFalse(resp.values().hasNext());
             assertThat(
                 resp.columns(),
@@ -1470,7 +1470,7 @@ public void testQueryOnEmptyDataIndex() {
     }
 
     private void assertEmptyIndexQueries(String from) {
-        try (EsqlQueryResponse resp = run(from + "[METADATA _source] | KEEP _source | LIMIT 1")) {
+        try (EsqlQueryResponse resp = run(from + "METADATA _source | KEEP _source | LIMIT 1")) {
             assertFalse(resp.values().hasNext());
             assertThat(resp.columns(), equalTo(List.of(new ColumnInfo("_source", "_source"))));
         }

x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4

@@ -102,9 +102,17 @@ fromCommand
     ;
 
 metadata
-    : OPENING_BRACKET METADATA fromIdentifier (COMMA fromIdentifier)* CLOSING_BRACKET
+    : metadataOption
+    | deprecated_metadata
     ;
 
+metadataOption
+    : METADATA fromIdentifier (COMMA fromIdentifier)*
+    ;
+
+deprecated_metadata
+    : OPENING_BRACKET metadataOption CLOSING_BRACKET
+    ;
 
 evalCommand
     : EVAL fields