[Rest Api Compatibility] CommonTermsQuery and cutoff_frequency param

modules/analysis-common/build.gradle

@@ -30,13 +30,11 @@ def v7compatibilityNotSupportedTests = {
      //marked as not needing compatible api
      'indices.analyze/10_analyze/htmlStrip_deprecated', // Cleanup versioned deprecations in analysis #41560
      'analysis-common/40_token_filters/delimited_payload_filter_error', //Remove preconfigured delimited_payload_filter #43686
-     'analysis-common/20_analyzers/standard_html_strip' // Cleanup versioned deprecations in analysis #41560
+     'analysis-common/20_analyzers/standard_html_strip', // Cleanup versioned deprecations in analysis #41560
+     'search.query/50_queries_with_synonyms/Test common terms query with stacked tokens', // #42654 -  `common` query throws an exception
   ]
 }
 
 tasks.named("yamlRestCompatTest").configure {
-  systemProperty 'tests.rest.blacklist', ([
-    'search.query/50_queries_with_synonyms/Test common terms query with stacked tokens', // #42654 - not sure, remove `common` query
-  ]+ v7compatibilityNotSupportedTests())
-  .join(',')
+  systemProperty 'tests.rest.blacklist', v7compatibilityNotSupportedTests().join(',')
 }

rest-api-spec/build.gradle

@@ -84,6 +84,8 @@ def v7compatibilityNotSupportedTests = {
 
           'search/340_type_query/type query', //#47207 type query throws exception in compatible mode
           'search.aggregation/200_top_hits_metric/top_hits aggregation with sequence numbers', // #42809 the use nested path and filter sort throws an exception
+          'search/310_match_bool_prefix/multi_match multiple fields with cutoff_frequency throws exception', //#42654 cutoff_frequency, common terms are not supported. Throwing an exception
+
 
   ]
 }
@@ -92,11 +94,7 @@ tasks.named("yamlRestCompatTest").configure {
     // Skip these tests on Windows since the blacklist exceeds Windows CLI limits
     OS.current() != OS.WINDOWS
   }
-  systemProperty 'tests.rest.blacklist', ([
-    'search/310_match_bool_prefix/multi_match multiple fields with cutoff_frequency throws exception', //cutoff_frequency
-  ] + v7compatibilityNotSupportedTests())
-    .join(',')
-
+  systemProperty 'tests.rest.blacklist',  v7compatibilityNotSupportedTests().join(',')
 }
 
 tasks.named("transformV7RestTests").configure({ task ->

rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/10_cutoff_frequency.yml

@@ -0,0 +1,105 @@
+---
+setup:
+  - skip:
+      version: "9.0.0 - "
+      reason: "compatible from 8.x to 7.x"
+      features:
+        - "headers"
+        - "allowed_warnings_regex"
+  - do:
+      indices.create:
+        index: "test"
+        body:
+          mappings:
+            properties:
+              my_field1:
+                type: "text"
+              my_field2:
+                type: "text"
+      headers:
+        Content-Type: "application/vnd.elasticsearch+json;compatible-with=7"
+        Accept: "application/vnd.elasticsearch+json;compatible-with=7"
+      allowed_warnings_regex:
+      - "\\[types removal\\].*"
+  - do:
+      index:
+        index: "test"
+        id: 1
+        body:
+          my_field1: "brown fox jump"
+          my_field2: "xylophone"
+      headers:
+        Content-Type: "application/vnd.elasticsearch+json;compatible-with=7"
+        Accept: "application/vnd.elasticsearch+json;compatible-with=7"
+      allowed_warnings_regex:
+      - "\\[types removal\\].*"
+  - do:
+      indices.refresh: {}
+      headers:
+        Content-Type: "application/vnd.elasticsearch+json;compatible-with=7"
+        Accept: "application/vnd.elasticsearch+json;compatible-with=7"
+      allowed_warnings_regex:
+      - "\\[types removal\\].*"
+
+---
+multi_match multiple fields with cutoff_frequency throws exception:
+- do:
+    catch: "/cutoff_freqency is not supported. The \\[multi_match\\] query can skip block of documents efficiently if the total number of hits is not tracked/"
+    search:
+      rest_total_hits_as_int: true
+      index: "test"
+      body:
+        query:
+          multi_match:
+            query: "brown"
+            type: "bool_prefix"
+            fields:
+            - "my_field1"
+            - "my_field2"
+            cutoff_frequency: 0.001
+    headers:
+      Content-Type: "application/vnd.elasticsearch+json;compatible-with=7"
+      Accept: "application/vnd.elasticsearch+json;compatible-with=7"
+    allowed_warnings_regex:
+    - "\\[types removal\\].*"
+
+---
+match with cutoff_frequency throws exception:
+  - do:
+      catch: "/cutoff_freqency is not supported. The \\[match\\] query can skip block of documents efficiently if the total number of hits is not tracked/"
+      search:
+        rest_total_hits_as_int: true
+        index: "test"
+        body:
+          query:
+            match:
+              my_field1:
+                query: "brown"
+                type: "bool_prefix"
+                cutoff_frequency: 0.001
+      headers:
+        Content-Type: "application/vnd.elasticsearch+json;compatible-with=7"
+        Accept: "application/vnd.elasticsearch+json;compatible-with=7"
+      allowed_warnings_regex:
+        - "\\[types removal\\].*"
+
+---
+common querythrows exception:
+  - do:
+      catch: "/Common Terms Query usage is not supported. Use \\[match\\] query which can efficiently skip blocks of documents if the total number of hits is not tracked./"
+      search:
+        rest_total_hits_as_int: true
+        index: "test"
+        body:
+          query:
+            common:
+              my_field1:
+                query: "brown"
+                type: "bool_prefix"
+                cutoff_frequency: 0.001
+                low_freq_operator: or
+      headers:
+        Content-Type: "application/vnd.elasticsearch+json;compatible-with=7"
+        Accept: "application/vnd.elasticsearch+json;compatible-with=7"
+      allowed_warnings_regex:
+        - "\\[types removal\\].*"

server/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.query;
+
+import org.apache.lucene.search.Query;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.xcontent.ParseField;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.core.RestApiVersion;
+
+import java.io.IOException;
+
+public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQueryBuilder> {
+    private static final DeprecationLogger deprecationLogger =  DeprecationLogger.getLogger(CommonTermsQueryBuilder.class);
+    public static final String COMMON_TERMS_QUERY_DEPRECATION_MSG = "Common Terms Query usage is not supported. " +
+        "Use [match] query which can efficiently skip blocks of documents if the total number of hits is not tracked.";
+
+    public static ParseField NAME_V7 = new ParseField("common")
+        .withAllDeprecated(COMMON_TERMS_QUERY_DEPRECATION_MSG)
+        .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7));
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        throw new UnsupportedOperationException("common_term_query is not meant to be serialized.");
+    }
+
+    @Override
+    protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+    }
+
+    @Override
+    protected Query doToQuery(SearchExecutionContext context) throws IOException {
+        return null;
+    }
+
+    @Override
+    protected boolean doEquals(CommonTermsQueryBuilder other) {
+        return false;
+    }
+
+    @Override
+    protected int doHashCode() {
+        return 0;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return null;
+    }
+
+    public static CommonTermsQueryBuilder fromXContent(XContentParser parser) throws IOException {
+        deprecationLogger.compatibleApiWarning("common_term_query", COMMON_TERMS_QUERY_DEPRECATION_MSG);
+        throw new ParsingException(parser.getTokenLocation(), COMMON_TERMS_QUERY_DEPRECATION_MSG);
+    }
+
+}

server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.query.support.QueryParsers;
 import org.elasticsearch.index.search.MatchQueryParser;
 
@@ -31,7 +32,12 @@
  * result of the analysis.
  */
 public class MatchQueryBuilder extends AbstractQueryBuilder<MatchQueryBuilder> {
-
+    private static final String CUTOFF_FREQUENCY_DEPRECATION_MSG = "cutoff_freqency is not supported. " +
+        "The [match] query can skip block of documents efficiently if the total number of hits is not tracked";
+    public static final ParseField CUTOFF_FREQUENCY_FIELD =
+        new ParseField("cutoff_frequency")
+            .withAllDeprecated(CUTOFF_FREQUENCY_DEPRECATION_MSG)
+        .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7));
     public static final ParseField ZERO_TERMS_QUERY_FIELD = new ParseField("zero_terms_query");
     public static final ParseField LENIENT_FIELD = new ParseField("lenient");
     public static final ParseField FUZZY_TRANSPOSITIONS_FIELD = new ParseField("fuzzy_transpositions");
@@ -464,6 +470,9 @@ public static MatchQueryBuilder fromXContent(XContentParser parser) throws IOExc
                             queryName = parser.text();
                         } else if (GENERATE_SYNONYMS_PHRASE_QUERY.match(currentFieldName, parser.getDeprecationHandler())) {
                             autoGenerateSynonymsPhraseQuery = parser.booleanValue();
+                        } else if (parser.getRestApiVersion() == RestApiVersion.V_7 &&
+                            CUTOFF_FREQUENCY_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                            throw new ParsingException(parser.getTokenLocation(), CUTOFF_FREQUENCY_DEPRECATION_MSG);
                         } else {
                             throw new ParsingException(parser.getTokenLocation(),
                                     "[" + NAME + "] query does not support [" + currentFieldName + "]");

server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.query.support.QueryParsers;
 import org.elasticsearch.index.search.MatchQueryParser;
 import org.elasticsearch.index.search.MultiMatchQueryParser;
@@ -42,7 +43,12 @@
 public class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQueryBuilder> {
 
     public static final String NAME = "multi_match";
-
+    private static final String CUTOFF_FREQUENCY_DEPRECATION_MSG = "cutoff_freqency is not supported." +
+        " The [multi_match] query can skip block of documents efficiently if the total number of hits is not tracked";
+    private static final ParseField CUTOFF_FREQUENCY_FIELD =
+        new ParseField("cutoff_frequency")
+            .withAllDeprecated(CUTOFF_FREQUENCY_DEPRECATION_MSG)
+        .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7));
     public static final MultiMatchQueryBuilder.Type DEFAULT_TYPE = MultiMatchQueryBuilder.Type.BEST_FIELDS;
     public static final Operator DEFAULT_OPERATOR = Operator.OR;
     public static final int DEFAULT_PHRASE_SLOP = MatchQueryParser.DEFAULT_PHRASE_SLOP;
@@ -570,7 +576,6 @@ public static MultiMatchQueryBuilder fromXContent(XContentParser parser) throws
         ZeroTermsQueryOption zeroTermsQuery = DEFAULT_ZERO_TERMS_QUERY;
         boolean autoGenerateSynonymsPhraseQuery = true;
         boolean fuzzyTranspositions = DEFAULT_FUZZY_TRANSPOSITIONS;
-
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
         String queryName = null;
 
@@ -633,6 +638,9 @@ public static MultiMatchQueryBuilder fromXContent(XContentParser parser) throws
                     autoGenerateSynonymsPhraseQuery = parser.booleanValue();
                 } else if (FUZZY_TRANSPOSITIONS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     fuzzyTranspositions = parser.booleanValue();
+                }  else if (parser.getRestApiVersion() == RestApiVersion.V_7 &&
+                            CUTOFF_FREQUENCY_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                    throw new ParsingException(parser.getTokenLocation(), CUTOFF_FREQUENCY_DEPRECATION_MSG);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "[" + NAME + "] query does not support [" + currentFieldName + "]");

server/src/main/java/org/elasticsearch/search/SearchModule.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.BoostingQueryBuilder;
 import org.elasticsearch.index.query.CombinedFieldsQueryBuilder;
+import org.elasticsearch.index.query.CommonTermsQueryBuilder;
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
 import org.elasticsearch.index.query.DisMaxQueryBuilder;
 import org.elasticsearch.index.query.DistanceFeatureQueryBuilder;
@@ -482,6 +483,12 @@ private ValuesSourceRegistry registerAggregations(List<SearchPlugin> plugins) {
                 .setAggregatorRegistrar(CompositeAggregationBuilder::registerAggregators),
             builder
         );
+
+        if(RestApiVersion.minimumSupported() == RestApiVersion.V_7) {
+            registerQuery(new QuerySpec<>(CommonTermsQueryBuilder.NAME_V7,
+                (streamInput) -> new CommonTermsQueryBuilder(), CommonTermsQueryBuilder::fromXContent));
+        }
+
         registerFromPlugin(plugins, SearchPlugin::getAggregations, (agg) -> this.registerAggregation(agg, builder));
 
         // after aggs have been registered, see if there are any new VSTypes that need to be linked to core fields

server/src/test/java/org/elasticsearch/search/SearchModuleTests.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.CommonTermsQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.SearchExecutionContext;
@@ -393,7 +394,10 @@ public CheckedBiConsumer<ShardSearchRequest, StreamOutput, IOException> getReque
 
     //add here deprecated queries to make sure we log a deprecation warnings when they are used
     private static final String[] DEPRECATED_QUERIES = new String[] {"field_masking_span", "geo_polygon"};
-    private static final String[] REST_COMPATIBLE_QUERIES = new String[] {TypeQueryV7Builder.NAME_V7.getPreferredName()};
+    private static final String[] REST_COMPATIBLE_QUERIES = new String[] {
+        TypeQueryV7Builder.NAME_V7.getPreferredName(),
+        CommonTermsQueryBuilder.NAME_V7.getPreferredName()
+    };
 
     /**
      * Dummy test {@link AggregationBuilder} used to test registering aggregation builders.
@@ -674,7 +678,8 @@ public List<SearchPlugin.QuerySpec<?>> getQueries() {
                 .filter(e -> RestApiVersion.minimumSupported().matches(e.restApiCompatibility))
                 .filter(e -> RestApiVersion.current().matches(e.restApiCompatibility))
                 .collect(toSet()),
-            hasSize(searchModule.getNamedXContents().size()- REST_COMPATIBLE_QUERIES.length - 1 ));
+            // -1 because of the registered in the test
+            hasSize(searchModule.getNamedXContents().size() - REST_COMPATIBLE_QUERIES.length -1 ));
 
 
         final List<NamedXContentRegistry.Entry> compatEntry = searchModule.getNamedXContents().stream()