Upgrade to Lucene 8.9 (#1080) (#1115)

This commit upgrades to the official lucene 8.9 release Signed-off-by: Nicholas Walter Knize <nknize@apache.org>
opensearch-project · Aug 23, 2021 · e153629 · e153629
1 parent c15e687
commit e153629
Show file tree

Hide file tree

Showing 48 changed files with 29 additions and 25 deletions.
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 opensearch        = 1.1.0
-lucene            = 8.8.2
+lucene            = 8.9.0
 
 bundled_jdk_vendor = adoptopenjdk
 bundled_jdk = 15.0.1+9

diff --git a/modules/lang-expression/licenses/lucene-expressions-8.8.2.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.8.2.jar.sha1
diff --git a/modules/lang-expression/licenses/lucene-expressions-8.9.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.9.0.jar.sha1
@@ -0,0 +1 @@
+c52e0f197d8f37fec8e679660a4814b8928e4db2
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.8.2.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.8.2.jar.sha1
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.9.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.9.0.jar.sha1
@@ -0,0 +1 @@
+390ebbb1cdb3ab1e8e5286ae4d77d336fd3895ba
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.8.2.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.8.2.jar.sha1
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.9.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.9.0.jar.sha1
@@ -0,0 +1 @@
+d7553997ad55671bb8c4c023f607f71463da4d0f
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.8.2.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.8.2.jar.sha1
diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.9.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.9.0.jar.sha1
@@ -0,0 +1 @@
+99ae12d053d27e2b8d78d084e1bf1c893b896554
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.8.2.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.8.2.jar.sha1
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.9.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.9.0.jar.sha1
@@ -0,0 +1 @@
+144cbfb8c5a66ae9ac7abddb8855974599d1ea67
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.8.2.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.8.2.jar.sha1
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.9.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.9.0.jar.sha1
@@ -0,0 +1 @@
+35f24955ae3d079a262a3aa7ac2ca2449c6d10ee
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.8.2.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.8.2.jar.sha1
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.9.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.9.0.jar.sha1
@@ -0,0 +1 @@
+d2a6fb6327e54c4eccfaa0f8e74953cd69a31e07
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.8.2.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.8.2.jar.sha1
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.9.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.9.0.jar.sha1
@@ -0,0 +1 @@
+37fd6e23bbb1c146c729bb32b6c043ed0642a73e
diff --git a/server/licenses/lucene-analyzers-common-8.8.2.jar.sha1 b/server/licenses/lucene-analyzers-common-8.8.2.jar.sha1
diff --git a/server/licenses/lucene-analyzers-common-8.9.0.jar.sha1 b/server/licenses/lucene-analyzers-common-8.9.0.jar.sha1
@@ -0,0 +1 @@
+18f3bbff2b7672ea0b9cc18c8110ef69c763ae6b
diff --git a/server/licenses/lucene-backward-codecs-8.8.2.jar.sha1 b/server/licenses/lucene-backward-codecs-8.8.2.jar.sha1
diff --git a/server/licenses/lucene-backward-codecs-8.9.0.jar.sha1 b/server/licenses/lucene-backward-codecs-8.9.0.jar.sha1
@@ -0,0 +1 @@
+fec88b5e71c699ceddc3ae0369481697ac9a5c96
diff --git a/server/licenses/lucene-core-8.8.2.jar.sha1 b/server/licenses/lucene-core-8.8.2.jar.sha1
diff --git a/server/licenses/lucene-core-8.9.0.jar.sha1 b/server/licenses/lucene-core-8.9.0.jar.sha1
@@ -0,0 +1 @@
+5c3f72357089f7f0c1ef44bbe7b4c67b6149a5af
diff --git a/server/licenses/lucene-grouping-8.8.2.jar.sha1 b/server/licenses/lucene-grouping-8.8.2.jar.sha1
diff --git a/server/licenses/lucene-grouping-8.9.0.jar.sha1 b/server/licenses/lucene-grouping-8.9.0.jar.sha1
@@ -0,0 +1 @@
+9440fdd430b1c2dadbf3bc72656848d61e6f747f
diff --git a/server/licenses/lucene-highlighter-8.8.2.jar.sha1 b/server/licenses/lucene-highlighter-8.8.2.jar.sha1
diff --git a/server/licenses/lucene-highlighter-8.9.0.jar.sha1 b/server/licenses/lucene-highlighter-8.9.0.jar.sha1
@@ -0,0 +1 @@
+94e80bdeab170b0ce1b36a32b6a790d23d7f6d7b
diff --git a/server/licenses/lucene-join-8.8.2.jar.sha1 b/server/licenses/lucene-join-8.8.2.jar.sha1
diff --git a/server/licenses/lucene-join-8.9.0.jar.sha1 b/server/licenses/lucene-join-8.9.0.jar.sha1
@@ -0,0 +1 @@
+5ae97803efd3344597f6b6bdf823b18d130e8851
diff --git a/server/licenses/lucene-memory-8.8.2.jar.sha1 b/server/licenses/lucene-memory-8.8.2.jar.sha1
diff --git a/server/licenses/lucene-memory-8.9.0.jar.sha1 b/server/licenses/lucene-memory-8.9.0.jar.sha1
@@ -0,0 +1 @@
+09423a6dca2a9ba665e644d86a713d9a6b2b0d3f
diff --git a/server/licenses/lucene-misc-8.8.2.jar.sha1 b/server/licenses/lucene-misc-8.8.2.jar.sha1
diff --git a/server/licenses/lucene-misc-8.9.0.jar.sha1 b/server/licenses/lucene-misc-8.9.0.jar.sha1
@@ -0,0 +1 @@
+067494d621ba2ef1f2e4da3ef167106f00b52051
diff --git a/server/licenses/lucene-queries-8.8.2.jar.sha1 b/server/licenses/lucene-queries-8.8.2.jar.sha1
diff --git a/server/licenses/lucene-queries-8.9.0.jar.sha1 b/server/licenses/lucene-queries-8.9.0.jar.sha1
@@ -0,0 +1 @@
+c6bda4622abf240da6567a128242f46708fa6c00
diff --git a/server/licenses/lucene-queryparser-8.8.2.jar.sha1 b/server/licenses/lucene-queryparser-8.8.2.jar.sha1
diff --git a/server/licenses/lucene-queryparser-8.9.0.jar.sha1 b/server/licenses/lucene-queryparser-8.9.0.jar.sha1
@@ -0,0 +1 @@
+95a9d8cf8ca8eaf9f241fd323697d26d211721b2
diff --git a/server/licenses/lucene-sandbox-8.8.2.jar.sha1 b/server/licenses/lucene-sandbox-8.8.2.jar.sha1
diff --git a/server/licenses/lucene-sandbox-8.9.0.jar.sha1 b/server/licenses/lucene-sandbox-8.9.0.jar.sha1
@@ -0,0 +1 @@
+af9f6c0287465e17a520b93b684474712433b293
diff --git a/server/licenses/lucene-spatial-extras-8.8.2.jar.sha1 b/server/licenses/lucene-spatial-extras-8.8.2.jar.sha1
diff --git a/server/licenses/lucene-spatial-extras-8.9.0.jar.sha1 b/server/licenses/lucene-spatial-extras-8.9.0.jar.sha1
@@ -0,0 +1 @@
+c89f4e78712806e8d5bb4adfb21cf0722ad3f175
diff --git a/server/licenses/lucene-spatial3d-8.8.2.jar.sha1 b/server/licenses/lucene-spatial3d-8.8.2.jar.sha1
diff --git a/server/licenses/lucene-spatial3d-8.9.0.jar.sha1 b/server/licenses/lucene-spatial3d-8.9.0.jar.sha1
@@ -0,0 +1 @@
+0d1238c4e8bf4409b3bb3fbddf2e977b0f19b24b
diff --git a/server/licenses/lucene-suggest-8.8.2.jar.sha1 b/server/licenses/lucene-suggest-8.8.2.jar.sha1
diff --git a/server/licenses/lucene-suggest-8.9.0.jar.sha1 b/server/licenses/lucene-suggest-8.9.0.jar.sha1
@@ -0,0 +1 @@
+98cb69950b48b829b6605a003c99aa7eb86fa9eb
diff --git a/server/src/main/java/org/opensearch/Version.java b/server/src/main/java/org/opensearch/Version.java
@@ -71,7 +71,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_EMPTY = new Version(V_EMPTY_ID, org.apache.lucene.util.Version.LATEST);
 
     public static final Version V_1_0_0 = new Version(1000099, org.apache.lucene.util.Version.LUCENE_8_8_2);
-    public static final Version V_1_1_0 = new Version(1010099, org.apache.lucene.util.Version.LUCENE_8_8_2);
+    public static final Version V_1_1_0 = new Version(1010099, org.apache.lucene.util.Version.LUCENE_8_9_0);
     public static final Version CURRENT = V_1_1_0;
 
     public static Version readVersion(StreamInput in) throws IOException {

diff --git a/server/src/main/java/org/opensearch/index/engine/NoOpEngine.java b/server/src/main/java/org/opensearch/index/engine/NoOpEngine.java
@@ -86,7 +86,7 @@ protected DirectoryReader open(final IndexCommit commit) throws IOException {
         final Directory directory = commit.getDirectory();
         final List<IndexCommit> indexCommits = DirectoryReader.listCommits(directory);
         final IndexCommit indexCommit = indexCommits.get(indexCommits.size() - 1);
-        return new DirectoryReader(directory, new LeafReader[0]) {
+        return new DirectoryReader(directory, new LeafReader[0], null) {
             @Override
             protected DirectoryReader doOpenIfChanged() {
                 return null;

diff --git a/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java
@@ -208,6 +208,10 @@ public abstract class AnalysisFactoryTestCase extends OpenSearchTestCase {
         // LUCENE-8936
         .put("spanishminimalstem", Void.class)
         .put("delimitedboost", Void.class)
+        // LUCENE-9574: test flags on tokens vs a bitmask and drops tokens that have all specified flags
+        .put("dropifflagged", Void.class)
+        // LUCENE-9575: recognize arbitrary patterns that include punctuation
+        .put("patterntyping", Void.class)
         .immutableMap();
 
     static final Map<String,Class<?>> KNOWN_CHARFILTERS = new MapBuilder<String,Class<?>>()