Merge branch 'main' into ml-transform-fix-transform-list-reload

config/serverless.yml

@@ -119,6 +119,11 @@ xpack.alerting.rules.run.ruleTypeOverrides:
 xpack.alerting.rules.minimumScheduleInterval.enforce: true
 xpack.actions.run.maxAttempts: 10
 
+# Disables ESQL in advanced settings (hides it from the UI)
+uiSettings:
+  overrides:
+    discover:enableESQL: false
+
 # Task Manager
 xpack.task_manager.allow_reading_invalid_state: false
 

docs/management/advanced-options.asciidoc

@@ -288,8 +288,8 @@ in the current data view is used.
 The columns that appear by default on the *Discover* page. The default is
 `_source`.
 
-[[discover:enableSql]]`discover:enableSql`::
-experimental[] Allows SQL queries for search.
+[[discover:enableESQL]]`discover:enableESQL`::
+experimental[] Allows ES|QL queries for search.
 
 [[discover-max-doc-fields-displayed]]`discover:maxDocFieldsDisplayed`::
 Specifies the maximum number of fields to show in the document column of the *Discover* table.

packages/kbn-discover-utils/index.ts

@@ -13,7 +13,7 @@ export {
   DEFAULT_COLUMNS_SETTING,
   DOC_HIDE_TIME_COLUMN_SETTING,
   DOC_TABLE_LEGACY,
-  ENABLE_SQL,
+  ENABLE_ESQL,
   FIELDS_LIMIT_SETTING,
   HIDE_ANNOUNCEMENTS,
   KNOWN_FIELD_TYPE_LIST,

packages/kbn-discover-utils/src/constants.ts

@@ -12,7 +12,7 @@ export const CONTEXT_TIE_BREAKER_FIELDS_SETTING = 'context:tieBreakerFields';
 export const DEFAULT_COLUMNS_SETTING = 'defaultColumns';
 export const DOC_HIDE_TIME_COLUMN_SETTING = 'doc_table:hideTimeColumn';
 export const DOC_TABLE_LEGACY = 'doc_table:legacy';
-export const ENABLE_SQL = 'discover:enableSql';
+export const ENABLE_ESQL = 'discover:enableESQL';
 export const FIELDS_LIMIT_SETTING = 'fields:popularLimit';
 export const HIDE_ANNOUNCEMENTS = 'hideAnnouncements';
 export const MAX_DOC_FIELDS_DISPLAYED = 'discover:maxDocFieldsDisplayed';

packages/kbn-es-query/index.ts

@@ -54,6 +54,8 @@ export {
   isOfAggregateQueryType,
   getAggregateQueryMode,
   getIndexPatternFromSQLQuery,
+  getIndexPatternFromESQLQuery,
+  getLanguageDisplayName,
 } from './src/es_query';
 
 export {

packages/kbn-es-query/src/es_query/build_es_query.ts

@@ -12,7 +12,7 @@ import { buildQueryFromKuery } from './from_kuery';
 import { buildQueryFromFilters } from './from_filters';
 import { buildQueryFromLucene } from './from_lucene';
 import { Filter, Query, AggregateQuery } from '../filters';
-import { isOfQueryType } from './es_query_sql';
+import { isOfQueryType } from './es_aggregate_query';
 import { BoolQuery, DataViewBase } from './types';
 import type { KueryQueryOptions } from '../kuery';
 import type { EsQueryFiltersConfig } from './from_filters';

packages/kbn-es-query/src/es_query/es_query_sql.test.ts → packages/kbn-es-query/src/es_query/es_aggregate_query.test.ts

@@ -11,7 +11,8 @@ import {
   isOfAggregateQueryType,
   getAggregateQueryMode,
   getIndexPatternFromSQLQuery,
-} from './es_query_sql';
+  getIndexPatternFromESQLQuery,
+} from './es_aggregate_query';
 
 describe('sql query helpers', () => {
   describe('isOfQueryType', () => {
@@ -81,4 +82,20 @@ describe('sql query helpers', () => {
       expect(idxPattern8).toBe('logstash-1234!');
     });
   });
+
+  describe('getIndexPatternFromESQLQuery', () => {
+    it('should return the index pattern string from esql queries', () => {
+      const idxPattern1 = getIndexPatternFromESQLQuery('FROM foo');
+      expect(idxPattern1).toBe('foo');
+
+      const idxPattern3 = getIndexPatternFromESQLQuery('from foo | project abc, def');
+      expect(idxPattern3).toBe('foo');
+
+      const idxPattern4 = getIndexPatternFromESQLQuery('from foo | project a | limit 2');
+      expect(idxPattern4).toBe('foo');
+
+      const idxPattern5 = getIndexPatternFromESQLQuery('from foo | limit 2');
+      expect(idxPattern5).toBe('foo');
+    });
+  });
 });

packages/kbn-es-query/src/es_query/es_query_sql.ts → packages/kbn-es-query/src/es_query/es_aggregate_query.ts

@@ -28,7 +28,11 @@ export function getAggregateQueryMode(query: AggregateQuery): Language {
   return Object.keys(query)[0] as Language;
 }
 
-// retrieves the index pattern from the aggregate query
+export function getLanguageDisplayName(language: string): string {
+  return language === 'esql' ? 'es|ql' : language;
+}
+
+// retrieves the index pattern from the aggregate query for SQL
 export function getIndexPatternFromSQLQuery(sqlQuery?: string): string {
   let sql = sqlQuery?.replaceAll('"', '').replaceAll("'", '');
   const splitFroms = sql?.split(new RegExp(/FROM\s/, 'ig'));
@@ -44,3 +48,20 @@ export function getIndexPatternFromSQLQuery(sqlQuery?: string): string {
   }
   return '';
 }
+
+// retrieves the index pattern from the aggregate query for ES|QL
+export function getIndexPatternFromESQLQuery(esql?: string): string {
+  const splitFroms = esql?.split(new RegExp(/FROM\s/, 'ig'));
+  const fromsLength = splitFroms?.length ?? 0;
+  if (splitFroms && splitFroms?.length > 2) {
+    esql = `${splitFroms[fromsLength - 2]} FROM ${splitFroms[fromsLength - 1]}`;
+  }
+  const parsedString = esql?.replaceAll('`', '');
+  // case insensitive match for the index pattern
+  const regex = new RegExp(/FROM\s+([\w*-.!@$^()~;]+)/, 'i');
+  const matches = parsedString?.match(regex);
+  if (matches) {
+    return matches[1];
+  }
+  return '';
+}

packages/kbn-es-query/src/es_query/index.ts

@@ -18,7 +18,9 @@ export {
   isOfAggregateQueryType,
   getAggregateQueryMode,
   getIndexPatternFromSQLQuery,
-} from './es_query_sql';
+  getLanguageDisplayName,
+  getIndexPatternFromESQLQuery,
+} from './es_aggregate_query';
 export { fromCombinedFilter } from './from_combined_filter';
 export type {
   IFieldSubType,

packages/kbn-monaco/src/esql/antlr/.gitignore

@@ -0,0 +1 @@
+.antlr/*

packages/kbn-monaco/src/esql/antlr/esql_lexer.g4

@@ -7,15 +7,23 @@
 
 lexer grammar esql_lexer;
 
-EVAL : 'eval' -> pushMode(EXPRESSION);
-EXPLAIN : 'explain' -> pushMode(EXPRESSION);
-FROM : 'from' -> pushMode(SOURCE_IDENTIFIERS);
-ROW : 'row' -> pushMode(EXPRESSION);
-STATS : 'stats' -> pushMode(EXPRESSION);
-WHERE : 'where' -> pushMode(EXPRESSION);
-SORT : 'sort' -> pushMode(EXPRESSION);
-LIMIT : 'limit' -> pushMode(EXPRESSION);
-PROJECT : 'project' -> pushMode(SOURCE_IDENTIFIERS);
+DISSECT : D I S S E C T -> pushMode(EXPRESSION);
+GROK : G R O K -> pushMode(EXPRESSION);
+EVAL : E V A L -> pushMode(EXPRESSION);
+EXPLAIN : E X P L A I N -> pushMode(EXPLAIN_MODE);
+FROM : F R O M -> pushMode(SOURCE_IDENTIFIERS);
+ROW : R O W -> pushMode(EXPRESSION);
+STATS : S T A T S -> pushMode(EXPRESSION);
+WHERE : W H E R E -> pushMode(EXPRESSION);
+SORT : S O R T -> pushMode(EXPRESSION);
+MV_EXPAND : M V UNDERSCORE E X P A N D -> pushMode(EXPRESSION);
+LIMIT : L I M I T -> pushMode(EXPRESSION);
+PROJECT : P R O J E C T -> pushMode(EXPRESSION);
+DROP : D R O P -> pushMode(EXPRESSION);
+RENAME : R E N A M E -> pushMode(EXPRESSION);
+SHOW : S H O W -> pushMode(EXPRESSION);
+ENRICH : E N R I C H -> pushMode(ENRICH_IDENTIFIERS);
+KEEP : K E E P -> pushMode(EXPRESSION);
 
 LINE_COMMENT
     : '//' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN)
@@ -28,7 +36,12 @@ MULTILINE_COMMENT
 WS
     : [ \r\n\t]+ -> channel(HIDDEN)
     ;
-
+mode EXPLAIN_MODE;
+EXPLAIN_OPENING_BRACKET : '[' -> type(OPENING_BRACKET), pushMode(DEFAULT_MODE);
+EXPLAIN_PIPE : '|' -> type(PIPE), popMode;
+EXPLAIN_WS : WS -> channel(HIDDEN);
+EXPLAIN_LINE_COMMENT : LINE_COMMENT -> channel(HIDDEN);
+EXPLAIN_MULTILINE_COMMENT : MULTILINE_COMMENT -> channel(HIDDEN);
 mode EXPRESSION;
 
 PIPE : '|' -> popMode;
@@ -71,17 +84,34 @@ DECIMAL_LITERAL
 
 BY : 'by';
 
+DATE_LITERAL
+    : 'year'
+    | 'month'
+    | 'day'
+    | 'second'
+    | 'minute'
+    | 'hour'
+    ;
+
 AND : 'and';
 ASSIGN : '=';
 COMMA : ',';
 DOT : '.';
 LP : '(';
-OPENING_BRACKET : '[' -> pushMode(DEFAULT_MODE);
-CLOSING_BRACKET : ']' -> popMode, popMode; // pop twice, once to clear mode of current cmd and once to exit DEFAULT_MODE
-NOT : 'not';
-NULL : 'null';
+OPENING_BRACKET : '[' -> pushMode(EXPRESSION), pushMode(EXPRESSION);
+CLOSING_BRACKET : ']' -> popMode, popMode;
+NOT : N O T;
+LIKE: L I K E;
+RLIKE: R L I K E;
+IN: I N;
+IS: I S;
+AS: A S;
+NULL : N U L L;
 OR : 'or';
 RP : ')';
+UNDERSCORE: '_';
+INFO : 'info';
+FUNCTIONS : 'functions';
 
 BOOLEAN_VALUE
    : 'true'
@@ -102,6 +132,7 @@ MINUS : '-';
 ASTERISK : '*';
 SLASH : '/';
 PERCENT : '%';
+TEN: '10';
 
 ORDERING
     : 'asc'
@@ -114,16 +145,74 @@ NULLS_ORDERING_DIRECTION
     | 'last'
     ;
 
+MATH_FUNCTION
+    : R O U N D
+    | A B S
+    | P O W
+    | L O G TEN
+    | P I
+    | T A U
+    | E
+    | S U B S T R I N G
+    | T R I M
+    | C O N C A T
+    | S T A R T S UNDERSCORE W I T H
+    | D A T E UNDERSCORE F O R M A T
+    | D A T E UNDERSCORE T R U N C
+    | D A T E UNDERSCORE P A R S E
+    | A U T O UNDERSCORE B U C K E T
+    | I S UNDERSCORE F I N I T E
+    | I S UNDERSCORE I N F I N I T E
+    | C A S E
+    | L E N G T H
+    | M V UNDERSCORE M A X
+    | M V UNDERSCORE M I N
+    | M V UNDERSCORE A V G
+    | M V UNDERSCORE S U M
+    | M V UNDERSCORE C O U N T
+    | M V UNDERSCORE C O N C A T
+    | M V UNDERSCORE J O I N
+    | M V UNDERSCORE M E D I A N
+    | M V UNDERSCORE D E D U P E
+    | M E T A D A T A
+    | S P L I T
+    | T O UNDERSCORE S T R I N G
+    | T O UNDERSCORE S T R
+    | T O UNDERSCORE B O O L
+    | T O UNDERSCORE B O O L E A N
+    | T O UNDERSCORE D A T E T I M E
+    | T O UNDERSCORE D T
+    | T O UNDERSCORE D B L
+    | T O UNDERSCORE D O U B L E
+    | T O UNDERSCORE I N T
+    | T O UNDERSCORE I N T E G E R
+    | T O UNDERSCORE L O N G
+    | T O UNDERSCORE I P
+    | T O UNDERSCORE V E R S I O N
+    | T O UNDERSCORE U N S I G N E D UNDERSCORE L O N G
+    ;
+
 UNARY_FUNCTION
-    : 'round'
-    | 'avg'
-    | 'min'
-    | 'max'
-    | 'sum'
+    : A V G
+    | M I N
+    | M A X
+    | S U M
+    | C O U N T
+    | C O U N T UNDERSCORE D I S T I N C T
+    | P E R C E N T I L E
+    | M E D I A N
+    | M E D I A N UNDERSCORE A B S O L U T E UNDERSCORE D E V I A T I O N
+    ;
+
+WHERE_FUNCTIONS
+    : C I D R UNDERSCORE M A T C H
     ;
 
 UNQUOTED_IDENTIFIER
-    : (LETTER | '_') (LETTER | DIGIT | '_')*
+    : LETTER (LETTER | DIGIT | '_' | ASTERISK)*
+    // only allow @ at beginning of identifier to keep the option to allow @ as infix operator in the future
+    // also, single `_` and `@` characters are not valid identifiers
+    | ('_' | '@') (LETTER | DIGIT | '_' | ASTERISK)+
     ;
 
 QUOTED_IDENTIFIER
@@ -146,9 +235,11 @@ EXPR_WS
 mode SOURCE_IDENTIFIERS;
 
 SRC_PIPE : '|' -> type(PIPE), popMode;
+SRC_OPENING_BRACKET : '[' -> type(OPENING_BRACKET), pushMode(SOURCE_IDENTIFIERS), pushMode(SOURCE_IDENTIFIERS);
 SRC_CLOSING_BRACKET : ']' -> popMode, popMode, type(CLOSING_BRACKET);
 SRC_COMMA : ',' -> type(COMMA);
 SRC_ASSIGN : '=' -> type(ASSIGN);
+METADATA: M E T A D A T A;
 
 SRC_UNQUOTED_IDENTIFIER
     : SRC_UNQUOTED_IDENTIFIER_PART+
@@ -174,3 +265,65 @@ SRC_MULTILINE_COMMENT
 SRC_WS
     : WS -> channel(HIDDEN)
     ;
+
+mode ENRICH_IDENTIFIERS;
+
+ON : O N;
+WITH : W I T H;
+
+ENR_PIPE : '|' -> type(PIPE), popMode;
+ENR_CLOSING_BRACKET : ']' -> popMode, popMode, type(CLOSING_BRACKET);
+ENR_COMMA : ',' -> type(COMMA);
+ENR_ASSIGN : '=' -> type(ASSIGN);
+
+ENR_UNQUOTED_IDENTIFIER
+    : ENR_UNQUOTED_IDENTIFIER_PART+
+    ;
+
+fragment ENR_UNQUOTED_IDENTIFIER_PART
+    : ~[=`|,[\]/ \t\r\n]+
+    | '/' ~[*/] // allow single / but not followed by another / or * which would start a comment
+    ;
+
+ENR_QUOTED_IDENTIFIER
+    : QUOTED_IDENTIFIER
+    ;
+
+ENR_LINE_COMMENT
+    : LINE_COMMENT -> channel(HIDDEN)
+    ;
+
+ENR_MULTILINE_COMMENT
+    : MULTILINE_COMMENT -> channel(HIDDEN)
+    ;
+
+ENR_WS
+    : WS -> channel(HIDDEN)
+    ;
+
+fragment A : [aA]; // match either an 'a' or 'A'
+fragment B : [bB];
+fragment C : [cC];
+fragment D : [dD];
+fragment E : [eE];
+fragment F : [fF];
+fragment G : [gG];
+fragment H : [hH];
+fragment I : [iI];
+fragment J : [jJ];
+fragment K : [kK];
+fragment L : [lL];
+fragment M : [mM];
+fragment N : [nN];
+fragment O : [oO];
+fragment P : [pP];
+fragment Q : [qQ];
+fragment R : [rR];
+fragment S : [sS];
+fragment T : [tT];
+fragment U : [uU];
+fragment V : [vV];
+fragment W : [wW];
+fragment X : [xX];
+fragment Y : [yY];
+fragment Z : [zZ];