[Upgrade] Lucene 9.1.0-snapshot-ea989fe8f30 (#2487)

* [Upgrade] Lucene 9.1.0-snapshot-ea989fe8f30

Upgrades from Lucene 9.0.0 to 9.1.0-snapshot-ea989fe8f30 in preparation for
9.1.0 GA.

Signed-off-by: Nicholas Walter Knize <nknize@apache.org>

* Add spanishplural token filter

Signed-off-by: Nicholas Walter Knize <nknize@apache.org>

* fix KNOWN_TOKENIZERS

Signed-off-by: Nicholas Walter Knize <nknize@apache.org>

buildSrc/src/main/groovy/org/opensearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -123,7 +123,7 @@ class PluginBuildPlugin implements Plugin<Project> {
             naming.clear()
             naming {
                 Tests {
-                    baseClass 'org.apache.lucene.util.LuceneTestCase'
+                    baseClass 'org.apache.lucene.tests.util.LuceneTestCase'
                 }
                 IT {
                     baseClass 'org.opensearch.test.OpenSearchIntegTestCase'

buildSrc/src/main/java/org/opensearch/gradle/precommit/TestingConventionsPrecommitPlugin.java

@@ -43,7 +43,7 @@ public TaskProvider<? extends Task> createTask(Project project) {
             .register("testingConventions", TestingConventionsTasks.class);
         testingConventions.configure(t -> {
             TestingConventionRule testsRule = t.getNaming().maybeCreate("Tests");
-            testsRule.baseClass("org.apache.lucene.util.LuceneTestCase");
+            testsRule.baseClass("org.apache.lucene.tests.util.LuceneTestCase");
             TestingConventionRule itRule = t.getNaming().maybeCreate("IT");
             itRule.baseClass("org.opensearch.test.OpenSearchIntegTestCase");
             itRule.baseClass("org.opensearch.test.rest.OpenSearchRestTestCase");

buildSrc/src/main/resources/forbidden/opensearch-test-signatures.txt

@@ -19,9 +19,9 @@ com.carrotsearch.randomizedtesting.annotations.Seed @ Don't commit hardcoded see
 com.carrotsearch.randomizedtesting.annotations.Repeat @ Don't commit hardcoded repeats
 
 org.apache.lucene.codecs.Codec#setDefault(org.apache.lucene.codecs.Codec) @ Use the SuppressCodecs("*") annotation instead
-org.apache.lucene.util.LuceneTestCase$Slow @ Don't write slow tests
+org.apache.lucene.tests.util.LuceneTestCase$Slow @ Don't write slow tests
 org.junit.Ignore @ Use AwaitsFix instead
-org.apache.lucene.util.LuceneTestCase$Nightly @ We don't run nightly tests at this point!
+org.apache.lucene.tests.util.LuceneTestCase$Nightly @ We don't run nightly tests at this point!
 com.carrotsearch.randomizedtesting.annotations.Nightly @ We don't run nightly tests at this point!
 
 org.junit.Test @defaultMessage Just name your test method testFooBar

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 2.0.0
-lucene            = 9.0.0
+lucene            = 9.1.0-snapshot-ea989fe8f30
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 17.0.2+8

client/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java

@@ -37,7 +37,7 @@
 import org.apache.http.client.methods.HttpHead;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.admin.indices.alias.Alias;
 import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest;

distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreCommandTestCase.java

@@ -43,7 +43,7 @@
 import com.google.common.jimfs.Configuration;
 import com.google.common.jimfs.Jimfs;
 import org.opensearch.core.internal.io.IOUtils;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.opensearch.cli.CommandTestCase;
 import org.opensearch.common.io.PathUtilsForTesting;
 import org.opensearch.env.Environment;

distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java

@@ -35,7 +35,7 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.google.common.jimfs.Configuration;
 import com.google.common.jimfs.Jimfs;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.bouncycastle.bcpg.ArmoredOutputStream;
 import org.bouncycastle.bcpg.BCPGOutputStream;
 import org.bouncycastle.bcpg.HashAlgorithmTags;

distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/ListPluginsCommandTests.java

@@ -40,7 +40,7 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.opensearch.LegacyESVersion;
 import org.opensearch.Version;
 import org.opensearch.cli.ExitCodes;

distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/RemovePluginCommandTests.java

@@ -32,7 +32,7 @@
 
 package org.opensearch.plugins;
 
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.opensearch.Version;
 import org.opensearch.cli.ExitCodes;
 import org.opensearch.cli.MockTerminal;

libs/core/src/test/java/org/opensearch/core/internal/io/IOUtilsTests.java

@@ -30,8 +30,8 @@
 
 package org.opensearch.core.internal.io;
 
-import org.apache.lucene.mockfile.FilterFileSystemProvider;
-import org.apache.lucene.mockfile.FilterPath;
+import org.apache.lucene.tests.mockfile.FilterFileSystemProvider;
+import org.apache.lucene.tests.mockfile.FilterPath;
 import org.apache.lucene.util.Constants;
 import org.opensearch.common.CheckedConsumer;
 import org.opensearch.common.io.PathUtils;

modules/aggs-matrix-stats/src/test/java/org/opensearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java

@@ -36,7 +36,7 @@
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.store.Directory;

modules/analysis-common/src/test/java/org/opensearch/analysis/common/ConcatenateGraphTokenFilterFactoryTests.java

@@ -8,8 +8,8 @@
 
 package org.opensearch.analysis.common;
 
-import org.apache.lucene.analysis.CannedTokenStream;
-import org.apache.lucene.analysis.Token;
+import org.apache.lucene.tests.analysis.CannedTokenStream;
+import org.apache.lucene.tests.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;

modules/analysis-common/src/test/java/org/opensearch/analysis/common/FlattenGraphTokenFilterFactoryTests.java

@@ -32,8 +32,8 @@
 
 package org.opensearch.analysis.common;
 
-import org.apache.lucene.analysis.CannedTokenStream;
-import org.apache.lucene.analysis.Token;
+import org.apache.lucene.tests.analysis.CannedTokenStream;
+import org.apache.lucene.tests.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.index.Index;

modules/analysis-common/src/test/java/org/opensearch/analysis/common/NGramTokenizerFactoryTests.java

@@ -32,7 +32,7 @@
 
 package org.opensearch.analysis.common;
 
-import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.tests.analysis.MockTokenizer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;

modules/analysis-common/src/test/java/org/opensearch/analysis/common/RemoveDuplicatesFilterFactoryTests.java

@@ -32,8 +32,8 @@
 
 package org.opensearch.analysis.common;
 
-import org.apache.lucene.analysis.CannedTokenStream;
-import org.apache.lucene.analysis.Token;
+import org.apache.lucene.tests.analysis.CannedTokenStream;
+import org.apache.lucene.tests.analysis.Token;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.env.Environment;
 import org.opensearch.index.analysis.AnalysisTestsHelper;

modules/analysis-common/src/test/java/org/opensearch/analysis/common/SynonymsAnalysisTests.java

@@ -33,7 +33,7 @@
 package org.opensearch.analysis.common;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.KeywordTokenizer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;

modules/analysis-common/src/test/java/org/opensearch/analysis/common/UniqueTokenFilterTests.java

@@ -33,7 +33,7 @@
 package org.opensearch.analysis.common;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.tests.analysis.MockTokenizer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;

modules/analysis-common/src/test/java/org/opensearch/analysis/common/WhitespaceTokenizerFactoryTests.java

@@ -47,7 +47,7 @@
 import java.io.Reader;
 import java.io.StringReader;
 
-import static org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents;
+import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents;
 
 public class WhitespaceTokenizerFactoryTests extends OpenSearchTestCase {
 

modules/lang-expression/licenses/lucene-expressions-9.1.0-snapshot-ea989fe8f30.jar.sha1

@@ -0,0 +1 @@
+c7317bb4e72b820a516e0c8a90beac5acc82c2e2

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/TokenCountFieldMapperTests.java

@@ -33,9 +33,9 @@
 package org.opensearch.index.mapper;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.CannedTokenStream;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.Token;
+import org.apache.lucene.tests.analysis.CannedTokenStream;
+import org.apache.lucene.tests.analysis.MockTokenizer;
+import org.apache.lucene.tests.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;

modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java

@@ -443,7 +443,7 @@ public Query rewrite(IndexReader reader) throws IOException {
                     // blow up since for this query to work we have to have a DirectoryReader otherwise
                     // we can't load global ordinals - for this to work we simply check if the reader has no leaves
                     // and rewrite to match nothing
-                    return new MatchNoDocsQuery();
+                    return Queries.newMatchNoDocsQueryWithoutRewrite("unable to load global ordinals with an empty directory reader");
                 }
                 throw new IllegalStateException(
                     "can't load global ordinals for reader of type: " + reader.getClass() + " must be a DirectoryReader"

modules/parent-join/src/test/java/org/opensearch/join/aggregations/ChildrenToParentAggregatorTests.java

@@ -38,7 +38,7 @@
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;

modules/parent-join/src/test/java/org/opensearch/join/aggregations/ParentToChildrenAggregatorTests.java

@@ -39,7 +39,7 @@
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;

modules/percolator/src/test/java/org/opensearch/percolator/PercolatorMatchedSlotSubFetchPhaseTests.java

@@ -36,7 +36,7 @@
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.memory.MemoryIndex;
 import org.apache.lucene.search.MatchAllDocsQuery;

modules/reindex/src/test/java/org/opensearch/index/reindex/AsyncBulkByScrollActionTests.java

@@ -121,7 +121,7 @@
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonList;
 import static java.util.Collections.synchronizedSet;
-import static org.apache.lucene.util.TestUtil.randomSimpleString;
+import static org.apache.lucene.tests.util.TestUtil.randomSimpleString;
 import static org.opensearch.action.bulk.BackoffPolicy.constantBackoff;
 import static org.opensearch.common.unit.TimeValue.timeValueMillis;
 import static org.opensearch.common.unit.TimeValue.timeValueSeconds;

modules/reindex/src/test/java/org/opensearch/index/reindex/ClientScrollableHitSourceTests.java

@@ -69,7 +69,7 @@
 import java.util.stream.IntStream;
 
 import static java.util.Collections.emptyMap;
-import static org.apache.lucene.util.TestUtil.randomSimpleString;
+import static org.apache.lucene.tests.util.TestUtil.randomSimpleString;
 import static org.opensearch.common.unit.TimeValue.timeValueSeconds;
 import static org.hamcrest.Matchers.instanceOf;
 

modules/reindex/src/test/java/org/opensearch/index/reindex/RoundTripTests.java

@@ -50,7 +50,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.apache.lucene.util.TestUtil.randomSimpleString;
+import static org.apache.lucene.tests.util.TestUtil.randomSimpleString;
 import static org.opensearch.common.unit.TimeValue.parseTimeValue;
 
 /**

modules/reindex/src/test/java/org/opensearch/index/reindex/UpdateByQueryWhileModifyingTests.java

@@ -40,7 +40,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static org.apache.lucene.util.TestUtil.randomSimpleString;
+import static org.apache.lucene.tests.util.TestUtil.randomSimpleString;
 import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;

modules/transport-netty4/src/yamlRestTest/java/org/opensearch/http/netty4/Netty4ClientYamlTestSuiteIT.java

@@ -36,7 +36,7 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 
-import org.apache.lucene.util.TimeUnits;
+import org.apache.lucene.tests.util.TimeUnits;
 import org.opensearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase;
 import org.junit.BeforeClass;

plugins/analysis-icu/licenses/lucene-analysis-icu-9.1.0-snapshot-ea989fe8f30.jar.sha1

@@ -0,0 +1 @@
+77930f802430648980eded22ca6ed47fedaeaba4

plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/IcuAnalyzerTests.java

@@ -33,7 +33,7 @@
 package org.opensearch.index.analysis;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;
 import org.opensearch.Version;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;

plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/IcuTokenizerFactoryTests.java

@@ -48,7 +48,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 
-import static org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents;
+import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents;
 
 public class IcuTokenizerFactoryTests extends OpenSearchTestCase {
 

plugins/analysis-icu/src/test/java/org/opensearch/index/analysis/IndexableBinaryStringToolsTests.java

@@ -35,8 +35,8 @@
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TimeUnits;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TimeUnits;
 import org.opensearch.test.junit.listeners.ReproduceInfoPrinter;
 import org.junit.BeforeClass;
 

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.1.0-snapshot-ea989fe8f30.jar.sha1

@@ -0,0 +1 @@
+c66f568fa9138c6ab6f3abf1efbfab3c7b5991d4

plugins/analysis-kuromoji/src/test/java/org/opensearch/index/analysis/KuromojiAnalysisTests.java

@@ -53,7 +53,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 
-import static org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents;
+import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;

plugins/analysis-nori/licenses/lucene-analysis-nori-9.1.0-snapshot-ea989fe8f30.jar.sha1

@@ -0,0 +1 @@
+e8c47600ea859b999a5f5647341b0350b03dafcd

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.1.0-snapshot-ea989fe8f30.jar.sha1

@@ -0,0 +1 @@
+6f0f5c71052beee26e4ce99e1147ce406234f417

plugins/analysis-phonetic/src/test/java/org/opensearch/index/analysis/SimplePhoneticAnalysisTests.java

@@ -32,7 +32,7 @@
 
 package org.opensearch.index.analysis;
 
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.apache.lucene.analysis.phonetic.DaitchMokotoffSoundexFilter;

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.1.0-snapshot-ea989fe8f30.jar.sha1

@@ -0,0 +1 @@
+32aad8b8491df3c9862e7fe75e98bccdb6a25bda

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.1.0-snapshot-ea989fe8f30.jar.sha1

@@ -0,0 +1 @@
+ef546cfaaf727d93c4e86ddc7f77b525af135623

plugins/analysis-stempel/src/test/java/org/opensearch/index/analysis/AnalysisPolishFactoryTests.java

@@ -33,8 +33,8 @@
 package org.opensearch.index.analysis;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.tests.analysis.MockTokenizer;
 import org.apache.lucene.analysis.Tokenizer;
 import org.opensearch.Version;
 import org.opensearch.cluster.metadata.IndexMetadata;

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.1.0-snapshot-ea989fe8f30.jar.sha1

@@ -0,0 +1 @@
+21c3511469f67019804e41a8d83ffc5c36de6479

plugins/analysis-ukrainian/src/main/java/org/opensearch/index/analysis/UkrainianAnalyzerProvider.java

@@ -45,7 +45,7 @@ public class UkrainianAnalyzerProvider extends AbstractIndexAnalyzerProvider<Ukr
     public UkrainianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
         analyzer = new UkrainianMorfologikAnalyzer(
-            Analysis.parseStopWords(env, settings, UkrainianMorfologikAnalyzer.getDefaultStopSet()),
+            Analysis.parseStopWords(env, settings, UkrainianMorfologikAnalyzer.getDefaultStopwords()),
             Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)
         );
     }

plugins/ingest-attachment/src/test/java/org/opensearch/ingest/attachment/TikaDocTests.java

@@ -32,8 +32,8 @@
 
 package org.opensearch.ingest.attachment;
 
-import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
-import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.tests.util.LuceneTestCase.SuppressFileSystems;
+import org.apache.lucene.tests.util.TestUtil;
 import org.apache.tika.metadata.Metadata;
 import org.opensearch.common.io.PathUtils;
 import org.opensearch.test.OpenSearchTestCase;

plugins/mapper-annotated-text/src/test/java/org/opensearch/search/fetch/subphase/highlight/AnnotatedTextHighlighterTests.java

@@ -41,7 +41,7 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;