opensearch-project · reta · May 21, 2022 · May 20, 2022
@@ -19,7 +19,6 @@ com.carrotsearch.randomizedtesting.annotations.Seed @ Don't commit hardcoded see
 com.carrotsearch.randomizedtesting.annotations.Repeat @ Don't commit hardcoded repeats
 
 org.apache.lucene.codecs.Codec#setDefault(org.apache.lucene.codecs.Codec) @ Use the SuppressCodecs("*") annotation instead
-org.apache.lucene.tests.util.LuceneTestCase$Slow @ Don't write slow tests
 org.junit.Ignore @ Use AwaitsFix instead
 org.apache.lucene.tests.util.LuceneTestCase$Nightly @ We don't run nightly tests at this point!
 com.carrotsearch.randomizedtesting.annotations.Nightly @ We don't run nightly tests at this point!

@@ -1,5 +1,5 @@
 opensearch        = 2.1.0
-lucene            = 9.2.0-snapshot-f4f1f70
+lucene            = 9.2.0-snapshot-ba8c3a8
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 17.0.3+7

@@ -40,7 +40,6 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.net.URI;
 import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.AccessDeniedException;
@@ -172,10 +171,8 @@ public void runTestRm(final boolean exception) throws IOException {
         for (int i = 0; i < numberOfLocations; i++) {
             if (exception && randomBoolean()) {
                 final Path location = createTempDir();
-                final FileSystem fs = new AccessDeniedWhileDeletingFileSystem(location.getFileSystem()).getFileSystem(
-                    URI.create("file:///")
-                );
-                final Path wrapped = new FilterPath(location, fs);
+                final FilterFileSystemProvider ffsp = new AccessDeniedWhileDeletingFileSystem(location.getFileSystem());
+                final Path wrapped = ffsp.wrapPath(location);
                 locations[i] = wrapped.resolve(randomAlphaOfLength(8));
                 Files.createDirectory(locations[i]);
                 locationsThrowingException.add(locations[i]);
@@ -256,8 +253,8 @@ public FileChannel newFileChannel(final Path path, final Set<? extends OpenOptio
 
     public void testFsyncAccessDeniedOpeningDirectory() throws Exception {
         final Path path = createTempDir().toRealPath();
-        final FileSystem fs = new AccessDeniedWhileOpeningDirectoryFileSystem(path.getFileSystem()).getFileSystem(URI.create("file:///"));
-        final Path wrapped = new FilterPath(path, fs);
+        final FilterFileSystemProvider ffsp = new AccessDeniedWhileOpeningDirectoryFileSystem(path.getFileSystem());
+        final Path wrapped = ffsp.wrapPath(path);
         if (Constants.WINDOWS) {
             // no exception, we early return and do not even try to open the directory
             IOUtils.fsync(wrapped, true);

@@ -0,0 +1 @@
+fe5e4cf94d26bbe1d982808f34fa132bba5565e3
@@ -0,0 +1 @@
+9fc73c790c037e817635fcc30ea0891e6acd2fac
@@ -0,0 +1 @@
+058ffd84388f9ffcf0bfdd7f43a6e832836a2927
@@ -0,0 +1 @@
+fb166f35ba04a7687b3073afb9972f6669ac722e
@@ -0,0 +1 @@
+abf9eb24601ec11ce5b61e4753b41444a869b29d
@@ -0,0 +1 @@
+b310130fe9e0f31ce4218cda921309b1143f3541
@@ -0,0 +1 @@
+ad8783255cdcb6e7ab23a505123716ad979f3484
@@ -0,0 +1 @@
+75f8fbb94a303d04c5dc2b25436300a463003dd6
@@ -0,0 +1 @@
+b2aa0739c95f1f715f407087dbcf96c5c21f4cc7
@@ -0,0 +1 @@
+e4a2f89c03e98e0fc211bba2c090047a007eb442
@@ -0,0 +1 @@
+06df30c240dfb970002c104d44370ae58b7cb60a
@@ -0,0 +1 @@
+1991e0f0f71c3c99ba726fcfa372f7ba7c75bcf0
@@ -0,0 +1 @@
+cac793b5cfbccf5c310d51bc78cf97ce3befceac
@@ -0,0 +1 @@
+79344146c032fda532def9771de589c4798117e5
@@ -0,0 +1 @@
+1d5f1c88786bcdfc50466f963ef07cbd9c6c7827
@@ -0,0 +1 @@
+adba90f90cf6815eeb9009c1a42d7c86f916d9da
@@ -0,0 +1 @@
+d0f38091eee45a118173c9201677ebafa9ed9e89
@@ -0,0 +1 @@
+05e2ca9fc81e8b73f746c5ec40321d6d90e3bcdd
@@ -0,0 +1 @@
+c4ebbf7fd05e2889624b4dd9afb3f7b22aad94f3
@@ -0,0 +1 @@
+08836d9dee5a2e9e92b538023285de3d620abd4b
@@ -0,0 +1 @@
+d492d0c7b4bb76c3de7cfc1b4fe224ef9e9e7056
@@ -0,0 +1 @@
+71b5b0cfb5b5809c4a86e947b1f4d9202d6f1b75
@@ -125,7 +125,7 @@
  * @opensearch.internal
  */
 public class Lucene {
-    public static final String LATEST_CODEC = "Lucene91";
+    public static final String LATEST_CODEC = "Lucene92";
 
     public static final String SOFT_DELETES_FIELD = "__soft_deletes";
 

@@ -34,8 +34,8 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene91.Lucene91Codec;
-import org.apache.lucene.codecs.lucene91.Lucene91Codec.Mode;
+import org.apache.lucene.codecs.lucene92.Lucene92Codec;
+import org.apache.lucene.codecs.lucene92.Lucene92Codec.Mode;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.collect.MapBuilder;
 import org.opensearch.index.mapper.MapperService;
@@ -62,8 +62,8 @@ public class CodecService {
     public CodecService(@Nullable MapperService mapperService, Logger logger) {
         final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
         if (mapperService == null) {
-            codecs.put(DEFAULT_CODEC, new Lucene91Codec());
-            codecs.put(BEST_COMPRESSION_CODEC, new Lucene91Codec(Mode.BEST_COMPRESSION));
+            codecs.put(DEFAULT_CODEC, new Lucene92Codec());
+            codecs.put(BEST_COMPRESSION_CODEC, new Lucene92Codec(Mode.BEST_COMPRESSION));
         } else {
             codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
             codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));

@@ -36,7 +36,7 @@
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene91.Lucene91Codec;
+import org.apache.lucene.codecs.lucene92.Lucene92Codec;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
 import org.opensearch.common.lucene.Lucene;
 import org.opensearch.index.mapper.CompletionFieldMapper;
@@ -53,7 +53,7 @@
  *
  * @opensearch.internal
  */
-public class PerFieldMappingPostingFormatCodec extends Lucene91Codec {
+public class PerFieldMappingPostingFormatCodec extends Lucene92Codec {
     private final Logger logger;
     private final MapperService mapperService;
     private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat();

@@ -112,4 +112,8 @@ public long cost() {
         return values.cost();
     }
 
+    @Override
+    public long docValueCount() {
+        return values.docValueCount();
+    }
 }
@@ -229,5 +229,10 @@ public long nextOrd() throws IOException {
         public BytesRef lookupOrd(long ord) {
             return values.lookupOrd(ord);
         }
+
+        @Override
+        public long docValueCount() {
+            return currentEndOffset - currentOffset;
+        }
     }
 }
@@ -310,6 +310,11 @@ public boolean advanceExact(int doc) throws IOException {
                 return true;
             }
 
+            @Override
+            public long docValueCount() {
+                return values.docValueCount();
+            }
+
             @Override
             public String toString() {
                 return "anon AbstractSortedDocValues of [" + super.toString() + "]";
@@ -340,6 +345,11 @@ public long getValueCount() {
                 return 1 + values.getValueCount();
             }
 
+            @Override
+            public long docValueCount() {
+                return values.docValueCount();
+            }
+
             @Override
             public long nextOrd() throws IOException {
                 if (hasOrds) {

@@ -34,7 +34,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene91.Lucene91Codec;
+import org.apache.lucene.codecs.lucene92.Lucene92Codec;
 import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
@@ -65,21 +65,21 @@ public class CodecTests extends OpenSearchTestCase {
     public void testResolveDefaultCodecs() throws Exception {
         CodecService codecService = createCodecService();
         assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class));
-        assertThat(codecService.codec("default"), instanceOf(Lucene91Codec.class));
+        assertThat(codecService.codec("default"), instanceOf(Lucene92Codec.class));
     }
 
     public void testDefault() throws Exception {
         Codec codec = createCodecService().codec("default");
-        assertStoredFieldsCompressionEquals(Lucene91Codec.Mode.BEST_SPEED, codec);
+        assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_SPEED, codec);
     }
 
     public void testBestCompression() throws Exception {
         Codec codec = createCodecService().codec("best_compression");
-        assertStoredFieldsCompressionEquals(Lucene91Codec.Mode.BEST_COMPRESSION, codec);
+        assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_COMPRESSION, codec);
     }
 
     // write some docs with it, inspect .si to see this was the used compression
-    private void assertStoredFieldsCompressionEquals(Lucene91Codec.Mode expected, Codec actual) throws Exception {
+    private void assertStoredFieldsCompressionEquals(Lucene92Codec.Mode expected, Codec actual) throws Exception {
         Directory dir = newDirectory();
         IndexWriterConfig iwc = newIndexWriterConfig(null);
         iwc.setCodec(actual);
@@ -91,7 +91,7 @@ private void assertStoredFieldsCompressionEquals(Lucene91Codec.Mode expected, Co
         SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
         String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY);
         assertNotNull(v);
-        assertEquals(expected, Lucene91Codec.Mode.valueOf(v));
+        assertEquals(expected, Lucene92Codec.Mode.valueOf(v));
         ir.close();
         dir.close();
     }

@@ -32,7 +32,7 @@
 package org.opensearch.index.engine;
 
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene91.Lucene91Codec;
+import org.apache.lucene.codecs.lucene92.Lucene92Codec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
@@ -70,7 +70,7 @@ public void testExceptionsAreNotCached() {
     public void testCompletionStatsCache() throws IOException, InterruptedException {
         final IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
         final PostingsFormat postingsFormat = new Completion90PostingsFormat();
-        indexWriterConfig.setCodec(new Lucene91Codec() {
+        indexWriterConfig.setCodec(new Lucene92Codec() {
             @Override
             public PostingsFormat getPostingsFormatForField(String field) {
                 return postingsFormat; // all fields are suggest fields

@@ -763,6 +763,11 @@ public BytesRef lookupOrd(long ord) {
             public long getValueCount() {
                 return 1 << 20;
             }
+
+            @Override
+            public long docValueCount() {
+                return array[doc].length;
+            }
         };
         verifySortedSet(multiValues, numDocs);
         final FixedBitSet rootDocs = randomRootDocs(numDocs);

@@ -83,6 +83,10 @@ public long getValueCount() {
             return terms.length;
         }
 
+        @Override
+        public long docValueCount() {
+            return ords.length;
+        }
     }
 
     private void doTestSortedSetRangeLeafCollector(int maxNumValuesPerDoc) throws Exception {

@@ -98,6 +98,10 @@ public long getValueCount() {
                 return 1;
             }
 
+            @Override
+            public long docValueCount() {
+                return 1;
+            }
         };
         IncludeExclude inexcl = new IncludeExclude(new TreeSet<>(Collections.singleton(new BytesRef("foo"))), null);
         OrdinalsFilter filter = inexcl.convertToOrdinalsFilter(DocValueFormat.RAW);

@@ -155,6 +155,11 @@ public long nextOrd() {
                     return NO_MORE_ORDS;
                 }
             }
+
+            @Override
+            public long docValueCount() {
+                return ords[doc].length;
+            }
         };
 
         final BytesRef existingMissing = RandomPicks.randomFrom(random(), values);
@@ -257,6 +262,11 @@ public BytesRef lookupOrd(long ord) throws IOException {
                 return values[Math.toIntExact(ord)];
             }
 
+            @Override
+            public long docValueCount() {
+                throw new UnsupportedOperationException();
+            }
+
             @Override
             public long getValueCount() {
                 return values.length;

@@ -221,6 +221,7 @@ public abstract class AnalysisFactoryTestCase extends OpenSearchTestCase {
         .put("spanishpluralstem", Void.class)
         // LUCENE-10352
         .put("daitchmokotoffsoundex", Void.class)
+        .put("persianstem", Void.class)
         .immutableMap();
 
     static final Map<String, Class<?>> KNOWN_CHARFILTERS = new MapBuilder<String, Class<?>>()