SQL: Deprecate index_include_frozen request parameter and FROZEN ke…

…yword (#83943)

Resolves #81939 (part of
#70192)

Deprecates the `index_include_frozen` REST parameter in the `/_sql`
endpoint and all the syntax including the `FROZEN` keyword.

docs/changelog/83943.yaml

@@ -0,0 +1,19 @@
+pr: 83943
+summary: Deprecate `index_include_frozen` request parameter
+area: SQL
+type: deprecation
+issues:
+  - 81939
+deprecation:
+  title: Deprecate `index_include_frozen` request parameter in `_sql` API
+  area: REST API
+  details: |-
+    Following the deprecation of frozen indices, the `index_include_frozen`
+    parameter and `FROZEN` syntax is now also deprecated.
+  impact: |-
+    You should unfreeze frozen indices using the
+    {ref}/unfreeze-index-api.html[unfreeze index API] and stop using the
+    `index_include_frozen` parameter or the `FROZEN` keyword in SQL
+    queries. For some use cases, the frozen tier may be a suitable
+    replacement for frozen indices. See {ref}/data-tiers.html[data tiers]
+    for more information.

x-pack/plugin/sql/qa/jdbc/security/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/security/JdbcWarningsIT.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.sql.qa.jdbc.security;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.sql.qa.jdbc.JdbcWarningsTestCase;
+
+import java.util.Properties;
+
+public class JdbcWarningsIT extends JdbcWarningsTestCase {
+
+    @Override
+    protected Settings restClientSettings() {
+        return JdbcConnectionIT.securitySettings();
+    }
+
+    @Override
+    protected String getProtocol() {
+        return JdbcConnectionIT.SSL_ENABLED ? "https" : "http";
+    }
+
+    @Override
+    protected Properties connectionProperties() {
+        Properties properties = super.connectionProperties();
+        properties.putAll(JdbcSecurityUtils.adminProperties());
+        return properties;
+    }
+
+}

x-pack/plugin/sql/qa/jdbc/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/jdbc/single_node/JdbcWarningsIT.java

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.sql.qa.jdbc.single_node;
+
+import org.elasticsearch.xpack.sql.qa.jdbc.JdbcWarningsTestCase;
+
+public class JdbcWarningsIT extends JdbcWarningsTestCase {}

x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcWarningsTestCase.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.sql.qa.jdbc;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
+public abstract class JdbcWarningsTestCase extends JdbcIntegrationTestCase {
+
+    public void testDeprecationWarningsDoNotReachJdbcDriver() throws Exception {
+        index("test_data", b -> b.field("foo", 1));
+
+        try (Connection connection = esJdbc(); Statement statement = connection.createStatement()) {
+            ResultSet rs = statement.executeQuery("SELECT * FROM FROZEN \"test_*\"");
+            assertNull(rs.getWarnings());
+        }
+    }
+
+}

x-pack/plugin/sql/qa/server/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/RestSqlDeprecationIT.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.sql.qa.single_node;
+
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.xpack.sql.qa.rest.BaseRestSqlTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.xpack.sql.qa.rest.RestSqlTestCase.SQL_QUERY_REST_ENDPOINT;
+
+public class RestSqlDeprecationIT extends BaseRestSqlTestCase {
+
+    public void testIndexIncludeParameterIsDeprecated() throws IOException {
+        testDeprecationWarning(
+            query("SELECT * FROM test").mode(randomMode()).indexIncludeFrozen(randomBoolean()),
+            "[index_include_frozen] parameter is deprecated because frozen indices have been deprecated. Consider cold or frozen tiers "
+                + "in place of frozen indices."
+        );
+    }
+
+    public void testIncludeFrozenSyntaxIsDeprecatedInShowTables() throws IOException {
+        testFrozenSyntaxIsDeprecated("SHOW TABLES INCLUDE FROZEN", "INCLUDE FROZEN");
+    }
+
+    public void testIncludeFrozenSyntaxIsDeprecatedInShowColumns() throws IOException {
+        testFrozenSyntaxIsDeprecated("SHOW COLUMNS INCLUDE FROZEN IN test", "INCLUDE FROZEN");
+    }
+
+    public void testIncludeFrozenSyntaxIsDeprecatedInDescribeTable() throws IOException {
+        testFrozenSyntaxIsDeprecated("DESCRIBE INCLUDE FROZEN test", "INCLUDE FROZEN");
+    }
+
+    public void testFrozenSyntaxIsDeprecatedInFromClause() throws IOException {
+        testFrozenSyntaxIsDeprecated("SELECT * FROM FROZEN test", "FROZEN");
+    }
+
+    private void testFrozenSyntaxIsDeprecated(String query, String syntax) throws IOException {
+        testDeprecationWarning(
+            query(query).mode(randomMode()),
+            "["
+                + syntax
+                + "] syntax is deprecated because frozen indices have been deprecated. "
+                + "Consider cold or frozen tiers in place of frozen indices."
+        );
+    }
+
+    private void testDeprecationWarning(RequestObjectBuilder query, String warning) throws IOException {
+        index("{\"foo\": 1}");
+
+        Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT);
+        request.setEntity(new StringEntity(query.toString(), ContentType.APPLICATION_JSON));
+        request.setOptions(expectWarnings(warning));
+    }
+
+}

x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/BaseRestSqlTestCase.java

@@ -29,6 +29,7 @@
 import static org.elasticsearch.xpack.sql.proto.CoreProtocol.FETCH_SIZE_NAME;
 import static org.elasticsearch.xpack.sql.proto.CoreProtocol.FIELD_MULTI_VALUE_LENIENCY_NAME;
 import static org.elasticsearch.xpack.sql.proto.CoreProtocol.FILTER_NAME;
+import static org.elasticsearch.xpack.sql.proto.CoreProtocol.INDEX_INCLUDE_FROZEN_NAME;
 import static org.elasticsearch.xpack.sql.proto.CoreProtocol.KEEP_ALIVE_NAME;
 import static org.elasticsearch.xpack.sql.proto.CoreProtocol.KEEP_ON_COMPLETION_NAME;
 import static org.elasticsearch.xpack.sql.proto.CoreProtocol.MODE_NAME;
@@ -142,6 +143,11 @@ public RequestObjectBuilder runtimeMappings(String runtimeMappings) {
             return this;
         }
 
+        public RequestObjectBuilder indexIncludeFrozen(boolean includeFrozen) {
+            request.append(field(INDEX_INCLUDE_FROZEN_NAME, includeFrozen));
+            return this;
+        }
+
         private static String field(String name, Object value) {
             if (value == null) {
                 return StringUtils.EMPTY;

x-pack/plugin/sql/sql-action/src/main/java/org/elasticsearch/xpack/sql/action/SqlQueryRequest.java

@@ -11,13 +11,16 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationCategory;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xpack.sql.proto.CoreProtocol;
 import org.elasticsearch.xpack.sql.proto.RequestInfo;
 import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue;
 
@@ -53,11 +56,21 @@ public class SqlQueryRequest extends AbstractSqlQueryRequest {
     static final ParseField KEEP_ON_COMPLETION = new ParseField(KEEP_ON_COMPLETION_NAME);
     static final ParseField KEEP_ALIVE = new ParseField(KEEP_ALIVE_NAME);
 
+    private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(SqlQueryRequest.class);
+
+    private static final String INDEX_INCLUDE_FROZEN_DEPRECATION_MESSAGE = "["
+        + CoreProtocol.INDEX_INCLUDE_FROZEN_NAME
+        + "] parameter is deprecated because frozen indices have been deprecated. "
+        + "Consider cold or frozen tiers in place of frozen indices.";
+
     static {
         PARSER.declareString(SqlQueryRequest::cursor, CURSOR);
         PARSER.declareBoolean(SqlQueryRequest::columnar, COLUMNAR);
         PARSER.declareBoolean(SqlQueryRequest::fieldMultiValueLeniency, FIELD_MULTI_VALUE_LENIENCY);
-        PARSER.declareBoolean(SqlQueryRequest::indexIncludeFrozen, INDEX_INCLUDE_FROZEN);
+        PARSER.declareBoolean((r, v) -> {
+            DEPRECATION_LOGGER.warn(DeprecationCategory.API, "sql_index_include_frozen", INDEX_INCLUDE_FROZEN_DEPRECATION_MESSAGE);
+            r.indexIncludeFrozen(v);
+        }, INDEX_INCLUDE_FROZEN);
         PARSER.declareBoolean(SqlQueryRequest::binaryCommunication, BINARY_COMMUNICATION);
         PARSER.declareField(
             SqlQueryRequest::waitForCompletionTimeout,

x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryRequestTests.java

@@ -76,7 +76,7 @@ protected TestSqlQueryRequest createTestInstance() {
             randomAlphaOfLength(10),
             requestInfo,
             randomBoolean(),
-            randomBoolean(),
+            false, // deprecated
             randomTV(),
             randomBoolean(),
             randomTVGreaterThan(MIN_KEEP_ALIVE)

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/CommandBuilder.java

@@ -120,13 +120,17 @@ public Object visitShowFunctions(ShowFunctionsContext ctx) {
     public Object visitShowTables(ShowTablesContext ctx) {
         TableIdentifier ti = visitTableIdentifier(ctx.tableIdent);
         String index = ti != null ? ti.qualifiedIndex() : null;
+
+        boolean includeFrozen = ctx.FROZEN() != null;
+        maybeWarnDeprecatedFrozenSyntax(includeFrozen, "INCLUDE FROZEN");
+
         return new ShowTables(
             source(ctx),
             visitLikePattern(ctx.clusterLike),
             visitString(ctx.cluster),
             index,
             visitLikePattern(ctx.tableLike),
-            ctx.FROZEN() != null
+            includeFrozen
         );
     }
 
@@ -139,7 +143,11 @@ public Object visitShowSchemas(ShowSchemasContext ctx) {
     public Object visitShowColumns(ShowColumnsContext ctx) {
         TableIdentifier ti = visitTableIdentifier(ctx.tableIdent);
         String index = ti != null ? ti.qualifiedIndex() : null;
-        return new ShowColumns(source(ctx), string(ctx.cluster), index, visitLikePattern(ctx.tableLike), ctx.FROZEN() != null);
+
+        boolean includeFrozen = ctx.FROZEN() != null;
+        maybeWarnDeprecatedFrozenSyntax(includeFrozen, "INCLUDE FROZEN");
+
+        return new ShowColumns(source(ctx), string(ctx.cluster), index, visitLikePattern(ctx.tableLike), includeFrozen);
     }
 
     @Override

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/LogicalPlanBuilder.java

@@ -9,6 +9,8 @@
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.tree.TerminalNode;
+import org.elasticsearch.common.logging.DeprecationCategory;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.xpack.ql.expression.Alias;
 import org.elasticsearch.xpack.ql.expression.Expression;
@@ -63,11 +65,23 @@
 import java.util.Map;
 
 import static java.util.Collections.emptyList;
+import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 import static org.elasticsearch.xpack.ql.parser.ParserUtils.source;
 import static org.elasticsearch.xpack.ql.parser.ParserUtils.visitList;
 
 abstract class LogicalPlanBuilder extends ExpressionBuilder {
 
+    private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(CommandBuilder.class);
+
+    private static final String FROZEN_DEPRECATION_WARNING = "[{}] syntax is deprecated because frozen indices have been deprecated. "
+        + "Consider cold or frozen tiers in place of frozen indices.";
+
+    protected void maybeWarnDeprecatedFrozenSyntax(boolean includeFrozen, String syntax) {
+        if (includeFrozen) {
+            DEPRECATION_LOGGER.warn(DeprecationCategory.PARSING, "include_frozen_syntax", format(null, FROZEN_DEPRECATION_WARNING, syntax));
+        }
+    }
+
     protected LogicalPlanBuilder(Map<Token, SqlTypedParamValue> params, ZoneId zoneId) {
         super(params, zoneId);
     }
@@ -267,7 +281,9 @@ public Object visitSubquery(SubqueryContext ctx) {
     public LogicalPlan visitTableName(TableNameContext ctx) {
         String alias = visitQualifiedName(ctx.qualifiedName());
         TableIdentifier tableIdentifier = visitTableIdentifier(ctx.tableIdentifier());
-        return new UnresolvedRelation(source(ctx), tableIdentifier, alias, ctx.FROZEN() != null);
+        boolean includeFrozen = ctx.FROZEN() != null;
+        maybeWarnDeprecatedFrozenSyntax(includeFrozen, "FROZEN");
+        return new UnresolvedRelation(source(ctx), tableIdentifier, alias, includeFrozen);
     }
 
     private Limit limit(LogicalPlan plan, Source source, Token limit) {