From 5f056186884961997a5c2a78616560018f351ad5 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 8 Nov 2022 11:05:10 -0500 Subject: [PATCH 01/23] Only run backport workflow it a backport label was added. (#5144) Signed-off-by: dblock --- .github/workflows/backport.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index c1c2505a62245..b0ea0f782cb62 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -7,6 +7,7 @@ on: jobs: backport: + if: ${{ contains(github.event.label.name, 'backport') }} runs-on: ubuntu-latest permissions: contents: write From 38d510c86327750a8647169b4e11596490ac76ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 11:24:09 -0500 Subject: [PATCH 02/23] Bump gson from 2.9.0 to 2.10 in /plugins/repository-hdfs (#4948) * Bump gson from 2.9.0 to 2.10 in /plugins/repository-hdfs Bumps [gson](https://github.com/google/gson) from 2.9.0 to 2.10. - [Release notes](https://github.com/google/gson/releases) - [Changelog](https://github.com/google/gson/blob/master/CHANGELOG.md) - [Commits](https://github.com/google/gson/compare/gson-parent-2.9.0...gson-parent-2.10) --- updated-dependencies: - dependency-name: com.google.code.gson:gson dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 4 ++-- plugins/repository-hdfs/build.gradle | 2 +- plugins/repository-hdfs/licenses/gson-2.10.jar.sha1 | 1 + plugins/repository-hdfs/licenses/gson-2.9.0.jar.sha1 | 1 - 4 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 plugins/repository-hdfs/licenses/gson-2.10.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/gson-2.9.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index f06aec1204ee3..eaf65a12c8c8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,7 +68,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Bumps `jettison` from 1.5.0 to 1.5.1 - Bumps `azure-storage-common` from 12.18.0 to 12.18.1 - Bumps `forbiddenapis` from 3.3 to 3.4 -- Bumps `gson` from 2.9.0 to 2.9.1 +- Bumps `gson` from 2.9.0 to 2.10 - Bumps `protobuf-java` from 3.21.2 to 3.21.7k - Bumps `azure-core` from 1.31.0 to 1.33.0 - Bumps `avro` from 1.11.0 to 1.11.1 @@ -239,4 +239,4 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Security [Unreleased]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...HEAD -[2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...2.x +[2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...2.x \ No newline at end of file diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index a66e3686c1818..0fd28154fc2fb 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -67,7 +67,7 @@ dependencies { api "org.apache.logging.log4j:log4j-core:${versions.log4j}" api 'org.apache.avro:avro:1.11.1' api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" - api 'com.google.code.gson:gson:2.9.0' + api 'com.google.code.gson:gson:2.10' runtimeOnly 'com.google.guava:guava:31.1-jre' api 'com.google.protobuf:protobuf-java:3.21.7' api "commons-logging:commons-logging:${versions.commonslogging}" diff --git a/plugins/repository-hdfs/licenses/gson-2.10.jar.sha1 b/plugins/repository-hdfs/licenses/gson-2.10.jar.sha1 new file mode 100644 index 0000000000000..64f28f71ab421 --- /dev/null +++ b/plugins/repository-hdfs/licenses/gson-2.10.jar.sha1 @@ -0,0 +1 @@ +dd9b193aef96e973d5a11ab13cd17430c2e4306b \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/gson-2.9.0.jar.sha1 b/plugins/repository-hdfs/licenses/gson-2.9.0.jar.sha1 deleted file mode 100644 index 8e9626b0c949b..0000000000000 --- a/plugins/repository-hdfs/licenses/gson-2.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8a1167e089096758b49f9b34066ef98b2f4b37aa \ No newline at end of file From ab85c67bb002e2fb440be659cf156b3f5c06e0f1 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 8 Nov 2022 12:54:00 -0500 Subject: [PATCH 03/23] Update to Gradle 7.6 and JDK-19 (#4973) * Update to Gradle 7.6 and JDK-19 Signed-off-by: Andriy Redko * Address code review comments Signed-off-by: Andriy Redko * Address code review comments Signed-off-by: Andriy Redko * Update Gradle to 7.6-rc2 Signed-off-by: Andriy Redko * Removed unnecessary override Signed-off-by: Andriy Redko Signed-off-by: Andriy Redko --- CHANGELOG.md | 2 + build.gradle | 4 ++ buildSrc/build.gradle | 2 +- .../gradle/info/GlobalBuildInfoPlugin.java | 29 +++++++++++++- .../ThirdPartyAuditPrecommitPlugin.java | 2 +- buildSrc/version.properties | 3 +- gradle/wrapper/gradle-wrapper.properties | 4 +- test/framework/build.gradle | 4 ++ .../MockSinglePrioritizingExecutor.java | 39 +++++++++++-------- 9 files changed, 64 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eaf65a12c8c8c..d58c98214ee88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - on-boarding of tasks([#4542](https://github.com/opensearch-project/OpenSearch/pull/4542)) - Integs ([4588](https://github.com/opensearch-project/OpenSearch/pull/4588)) - Integration tests for searchable snapshots ([#5048](https://github.com/opensearch-project/OpenSearch/pull/5048)) +- Update to Gradle 7.6 and JDK-19 ([#4973](https://github.com/opensearch-project/OpenSearch/pull/4973)) + ### Dependencies - Bumps `log4j-core` from 2.18.0 to 2.19.0 - Bumps `reactor-netty-http` from 1.0.18 to 1.0.23 diff --git a/build.gradle b/build.gradle index 11ba3bf9fe105..1fb6cc3849710 100644 --- a/build.gradle +++ b/build.gradle @@ -316,6 +316,10 @@ allprojects { javadoc.options.encoding = 'UTF8' javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet') javadoc.options.tags = ["opensearch.internal", "opensearch.api", "opensearch.experimental"] + if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_19) { + javadoc.options.addBooleanOption("-enable-preview", true) + javadoc.options.addStringOption("-release", BuildParams.runtimeJavaVersion.majorVersion) + } } // support for reproducible builds diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 2f886968ca510..6f85088104021 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -114,7 +114,7 @@ dependencies { api 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2' api 'org.jdom:jdom2:2.0.6.1' api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}" - api 'de.thetaphi:forbiddenapis:3.3' + api 'de.thetaphi:forbiddenapis:3.4' api 'com.avast.gradle:gradle-docker-compose-plugin:0.15.2' api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}" api 'org.apache.maven:maven-model:3.6.2' diff --git a/buildSrc/src/main/java/org/opensearch/gradle/info/GlobalBuildInfoPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/info/GlobalBuildInfoPlugin.java index 62e743d513193..7708e4583ed7a 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/info/GlobalBuildInfoPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/info/GlobalBuildInfoPlugin.java @@ -45,12 +45,15 @@ import org.gradle.internal.jvm.Jvm; import org.gradle.internal.jvm.inspection.JvmInstallationMetadata; import org.gradle.internal.jvm.inspection.JvmMetadataDetector; +import org.gradle.jvm.toolchain.internal.InstallationLocation; import org.gradle.util.GradleVersion; import javax.inject.Inject; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -195,7 +198,29 @@ private JavaVersion determineJavaVersion(String description, File javaHome, Java } private JvmInstallationMetadata getJavaInstallation(File javaHome) { - return jvmMetadataDetector.getMetadata(javaHome); + final InstallationLocation location = new InstallationLocation(javaHome, "Java home"); + + try { + try { + // The getMetadata(File) is used by Gradle pre-7.6 + return (JvmInstallationMetadata) MethodHandles.publicLookup() + .findVirtual(JvmMetadataDetector.class, "getMetadata", MethodType.methodType(JvmInstallationMetadata.class, File.class)) + .bindTo(jvmMetadataDetector) + .invokeExact(location.getLocation()); + } catch (NoSuchMethodException | IllegalAccessException ex) { + // The getMetadata(InstallationLocation) is used by Gradle post-7.6 + return (JvmInstallationMetadata) MethodHandles.publicLookup() + .findVirtual( + JvmMetadataDetector.class, + "getMetadata", + MethodType.methodType(JvmInstallationMetadata.class, InstallationLocation.class) + ) + .bindTo(jvmMetadataDetector) + .invokeExact(location); + } + } catch (Throwable ex) { + throw new IllegalStateException("Unable to find suitable JvmMetadataDetector::getMetadata", ex); + } } private List getAvailableJavaVersions(JavaVersion minimumCompilerVersion) { @@ -205,7 +230,7 @@ private List getAvailableJavaVersions(JavaVersion minimumCompilerVersi String javaHomeEnvVarName = getJavaHomeEnvVarName(Integer.toString(version)); if (System.getenv(javaHomeEnvVarName) != null) { File javaHomeDirectory = new File(findJavaHome(Integer.toString(version))); - JvmInstallationMetadata javaInstallation = jvmMetadataDetector.getMetadata(javaHomeDirectory); + JvmInstallationMetadata javaInstallation = getJavaInstallation(javaHomeDirectory); JavaHome javaHome = JavaHome.of(version, providers.provider(() -> { int actualVersion = Integer.parseInt(javaInstallation.getLanguageVersion().getMajorVersion()); if (actualVersion != version) { diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java index 5d707ce2b9f28..b2d12c4fc2f79 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java @@ -51,7 +51,7 @@ public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin { public TaskProvider createTask(Project project) { project.getPlugins().apply(CompileOnlyResolvePlugin.class); project.getConfigurations().create("forbiddenApisCliJar"); - project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.2"); + project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.4"); Configuration jdkJarHellConfig = project.getConfigurations().create(JDK_JAR_HELL_CONFIG_NAME); if (BuildParams.isInternal() && project.getPath().equals(":libs:opensearch-core") == false) { diff --git a/buildSrc/version.properties b/buildSrc/version.properties index fc2020dbc4000..2054ec9047942 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -2,8 +2,7 @@ opensearch = 3.0.0 lucene = 9.5.0-snapshot-a4ef70f bundled_jdk_vendor = adoptium -bundled_jdk = 17.0.5+8 - +bundled_jdk = 19.0.1+10 # optional dependencies diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 58e9a16f424db..57dec44f2a013 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -11,7 +11,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-rc-2-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=db9c8211ed63f61f60292c69e80d89196f9eb36665e369e7f00ac4cc841c2219 +distributionSha256Sum=be5387477162265eac882b9c83d756d8d2db170380e36fba2fdbee83d87de0d7 diff --git a/test/framework/build.gradle b/test/framework/build.gradle index 096e8c1e58243..816ca66c9a255 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -88,6 +88,10 @@ test { systemProperty 'tests.gradle_index_compat_versions', BuildParams.bwcVersions.indexCompatible.join(',') systemProperty 'tests.gradle_wire_compat_versions', BuildParams.bwcVersions.wireCompatible.join(',') systemProperty 'tests.gradle_unreleased_versions', BuildParams.bwcVersions.unreleased.join(',') + + if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_18) { + jvmArgs += ["-Djava.security.manager=allow"] + } } tasks.register("integTest", Test) { diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/MockSinglePrioritizingExecutor.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/MockSinglePrioritizingExecutor.java index 10a6ec791ec66..f9b78b26c170e 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/MockSinglePrioritizingExecutor.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/MockSinglePrioritizingExecutor.java @@ -44,25 +44,30 @@ public class MockSinglePrioritizingExecutor extends PrioritizedOpenSearchThreadPoolExecutor { public MockSinglePrioritizingExecutor(String name, DeterministicTaskQueue deterministicTaskQueue, ThreadPool threadPool) { - super(name, 0, 1, 0L, TimeUnit.MILLISECONDS, r -> new Thread() { - @Override - public void start() { - deterministicTaskQueue.scheduleNow(new Runnable() { - @Override - public void run() { - try { - r.run(); - } catch (KillWorkerError kwe) { - // hacks everywhere - } + super(name, 0, 1, 0L, TimeUnit.MILLISECONDS, r -> { + // This executor used to override Thread::start method so the actual runnable is + // being scheduled in the scope of current thread of execution. In JDK-19, the Thread::start + // is not called anymore (https://bugs.openjdk.org/browse/JDK-8292027) and there is no + // suitable option to alter the executor's behavior in the similar way. The closest we + // could get to is to schedule the runnable once the ThreadFactory is being asked to + // allocate the new thread. + deterministicTaskQueue.scheduleNow(new Runnable() { + @Override + public void run() { + try { + r.run(); + } catch (KillWorkerError kwe) { + // hacks everywhere } + } - @Override - public String toString() { - return r.toString(); - } - }); - } + @Override + public String toString() { + return r.toString(); + } + }); + + return new Thread(() -> {}); }, threadPool.getThreadContext(), threadPool.scheduler()); } From 7cbf86a483668b356a6316f2b1918f039253b208 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 13:09:28 -0500 Subject: [PATCH 04/23] Bump protobuf-java from 3.21.8 to 3.21.9 in /test/fixtures/hdfs-fixture (#4996) * Bump protobuf-java from 3.21.8 to 3.21.9 in /test/fixtures/hdfs-fixture Bumps [protobuf-java](https://github.com/protocolbuffers/protobuf) from 3.21.8 to 3.21.9. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/main/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.21.8...v3.21.9) --- updated-dependencies: - dependency-name: com.google.protobuf:protobuf-java dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 2 +- test/fixtures/hdfs-fixture/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d58c98214ee88..3cfb0c4a93365 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,7 +71,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Bumps `azure-storage-common` from 12.18.0 to 12.18.1 - Bumps `forbiddenapis` from 3.3 to 3.4 - Bumps `gson` from 2.9.0 to 2.10 -- Bumps `protobuf-java` from 3.21.2 to 3.21.7k +- Bumps `protobuf-java` from 3.21.2 to 3.21.9 - Bumps `azure-core` from 1.31.0 to 1.33.0 - Bumps `avro` from 1.11.0 to 1.11.1 - Bumps `woodstox-core` from 6.3.0 to 6.3.1 diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index 5bb25f2e43c42..8307443defa3a 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -49,7 +49,7 @@ dependencies { api "com.fasterxml.woodstox:woodstox-core:${versions.woodstox}" api 'net.minidev:json-smart:2.4.8' api "org.mockito:mockito-core:${versions.mockito}" - api "com.google.protobuf:protobuf-java:3.21.8" + api "com.google.protobuf:protobuf-java:3.21.9" api "org.jetbrains.kotlin:kotlin-stdlib:${versions.kotlin}" api 'org.eclipse.jetty:jetty-server:9.4.49.v20220914' api 'org.apache.zookeeper:zookeeper:3.8.0' From 5c1d0848c2d71b76a6d6f9b3135a5e2619f9820f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 13:32:38 -0500 Subject: [PATCH 05/23] Bump bcpg-fips from 1.0.5.1 to 1.0.7.1 in /distribution/tools/plugin-cli (#5102) * Bump bcpg-fips from 1.0.5.1 to 1.0.7.1 in /distribution/tools/plugin-cli Bumps bcpg-fips from 1.0.5.1 to 1.0.7.1. --- updated-dependencies: - dependency-name: org.bouncycastle:bcpg-fips dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + distribution/tools/plugin-cli/build.gradle | 2 +- .../tools/plugin-cli/licenses/bcpg-fips-1.0.5.1.jar.sha1 | 1 - .../tools/plugin-cli/licenses/bcpg-fips-1.0.7.1.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 distribution/tools/plugin-cli/licenses/bcpg-fips-1.0.5.1.jar.sha1 create mode 100644 distribution/tools/plugin-cli/licenses/bcpg-fips-1.0.7.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cfb0c4a93365..cab2798456d33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Upgrade zookeeper dependency in hdfs-fixture ([#5007](https://github.com/opensearch-project/OpenSearch/pull/5007)) - Update Jackson to 2.14.0 ([#5105](https://github.com/opensearch-project/OpenSearch/pull/5105)) - Bump nebula-publishing-plugin from 4.4.4 to 4.6.0 ([#5127](https://github.com/opensearch-project/OpenSearch/pull/5127)) +- Bumps `bcpg-fips` from 1.0.5.1 to 1.0.7.1 ### Changed - Dependency updates (httpcore, mockito, slf4j, httpasyncclient, commons-codec) ([#4308](https://github.com/opensearch-project/OpenSearch/pull/4308)) diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index b2e81491da6bd..29592d965c2c3 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -35,7 +35,7 @@ archivesBaseName = 'opensearch-plugin-cli' dependencies { compileOnly project(":server") compileOnly project(":libs:opensearch-cli") - api "org.bouncycastle:bcpg-fips:1.0.5.1" + api "org.bouncycastle:bcpg-fips:1.0.7.1" api "org.bouncycastle:bc-fips:1.0.2.3" testImplementation project(":test:framework") testImplementation 'com.google.jimfs:jimfs:1.2' diff --git a/distribution/tools/plugin-cli/licenses/bcpg-fips-1.0.5.1.jar.sha1 b/distribution/tools/plugin-cli/licenses/bcpg-fips-1.0.5.1.jar.sha1 deleted file mode 100644 index 30c30bb4af8e0..0000000000000 --- a/distribution/tools/plugin-cli/licenses/bcpg-fips-1.0.5.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -63a454936d930fadb1c7a3206b8e758378dd0a26 \ No newline at end of file diff --git a/distribution/tools/plugin-cli/licenses/bcpg-fips-1.0.7.1.jar.sha1 b/distribution/tools/plugin-cli/licenses/bcpg-fips-1.0.7.1.jar.sha1 new file mode 100644 index 0000000000000..44cebc7c92d87 --- /dev/null +++ b/distribution/tools/plugin-cli/licenses/bcpg-fips-1.0.7.1.jar.sha1 @@ -0,0 +1 @@ +5e1952428655ea822066f86df2e3ecda8fa0ba2b \ No newline at end of file From 90fefa262c7089eefe846e49c3acb96469d98eb6 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Tue, 8 Nov 2022 11:19:17 -0800 Subject: [PATCH 06/23] Update changelog on main (#5092) This branch copies the release notes from 2.4 and removes all the redundant entries from the changelog on main. All the remaining entries in the changelog should reflect changes intended for 3.0. This also adds an FAQ detailing how the changelog should work with our branching strategy. Signed-off-by: Andrew Ross Signed-off-by: Andrew Ross --- CHANGELOG.md | 190 ++---------------- CONTRIBUTING.md | 31 ++- .../opensearch.release-notes-2.4.0.md | 100 +++++++++ 3 files changed, 137 insertions(+), 184 deletions(-) create mode 100644 release-notes/opensearch.release-notes-2.4.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index cab2798456d33..7920007bad2cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,68 +1,16 @@ # CHANGELOG -Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) +All notable changes to this project are documented in this file. -## [Unreleased] +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). See the [CONTRIBUTING guide](./CONTRIBUTING.md#Changelog) for instructions on how to add changelog entries. + +## [Unreleased 3.0] ### Added -- Add fix for auto expand replica validation ([#4994](https://github.com/opensearch-project/OpenSearch/pull/4994)) -- Add support for s390x architecture ([#4001](https://github.com/opensearch-project/OpenSearch/pull/4001)) -- Github workflow for changelog verification ([#4085](https://github.com/opensearch-project/OpenSearch/pull/4085)) -- Point in time rest layer changes for create and delete PIT API ([#4064](https://github.com/opensearch-project/OpenSearch/pull/4064)) -- Point in time rest layer changes for list PIT and PIT segments API ([#4388](https://github.com/opensearch-project/OpenSearch/pull/4388)) -- Added @dreamer-89 as an Opensearch maintainer ([#4342](https://github.com/opensearch-project/OpenSearch/pull/4342)) -- Added release notes for 1.3.5 ([#4343](https://github.com/opensearch-project/OpenSearch/pull/4343)) -- Added release notes for 2.2.1 ([#4344](https://github.com/opensearch-project/OpenSearch/pull/4344)) -- Label configuration for dependabot PRs ([#4348](https://github.com/opensearch-project/OpenSearch/pull/4348)) - Support for HTTP/2 (server-side) ([#3847](https://github.com/opensearch-project/OpenSearch/pull/3847)) -- BWC version 2.2.2 ([#4383](https://github.com/opensearch-project/OpenSearch/pull/4383)) -- Support for labels on version bump PRs, skip label support for changelog verifier ([#4391](https://github.com/opensearch-project/OpenSearch/pull/4391)) -- Update previous release bwc version to 2.4.0 ([#4455](https://github.com/opensearch-project/OpenSearch/pull/4455)) -- 2.3.0 release notes ([#4457](https://github.com/opensearch-project/OpenSearch/pull/4457)) -- Added missing javadocs for `:distribution:tools` modules ([#4483](https://github.com/opensearch-project/OpenSearch/pull/4483)) -- Add BWC version 2.3.1 ([#4513](https://github.com/opensearch-project/OpenSearch/pull/4513)) -- [Segment Replication] Add snapshot and restore tests for segment replication feature ([#3993](https://github.com/opensearch-project/OpenSearch/pull/3993)) -- Added missing javadocs for `:example-plugins` modules ([#4540](https://github.com/opensearch-project/OpenSearch/pull/4540)) -- Add missing Javadoc tag descriptions in unit tests ([#4629](https://github.com/opensearch-project/OpenSearch/pull/4629)) - Add getter for path field in NestedQueryBuilder ([#4636](https://github.com/opensearch-project/OpenSearch/pull/4636)) -- Added precommit support for windows ([#4676](https://github.com/opensearch-project/OpenSearch/pull/4676)) -- Added release notes for 1.3.6 ([#4681](https://github.com/opensearch-project/OpenSearch/pull/4681)) -- Added precommit support for MacOS ([#4682](https://github.com/opensearch-project/OpenSearch/pull/4682)) -- Recommission API changes for service layer ([#4320](https://github.com/opensearch-project/OpenSearch/pull/4320)) -- Update GeoGrid base class access modifier to support extensibility ([#4572](https://github.com/opensearch-project/OpenSearch/pull/4572)) -- Add a new node role 'search' which is dedicated to provide search capability ([#4689](https://github.com/opensearch-project/OpenSearch/pull/4689)) -- Introduce experimental searchable snapshot API ([#4680](https://github.com/opensearch-project/OpenSearch/pull/4680)) -- Implement Searchable Snapshot using OnDemandBlockIndexInput phase 1 ([#4892](https://github.com/opensearch-project/OpenSearch/pull/4892)) -- Recommissioning of zone. REST layer support. ([#4624](https://github.com/opensearch-project/OpenSearch/pull/4604)) -- Make searchable snapshot indexes read-only but allow deletion ([#4764](https://github.com/opensearch-project/OpenSearch/pull/4764)) -- Added in-flight cancellation of SearchShardTask based on resource consumption ([#4565](https://github.com/opensearch-project/OpenSearch/pull/4565)) - Apply reproducible builds configuration for OpenSearch plugins through gradle plugin ([#4746](https://github.com/opensearch-project/OpenSearch/pull/4746)) -- Add icon for IntelliJidea toolbox ([#4882](https://github.com/opensearch-project/OpenSearch/pull/4882)) -- Add groupId value propagation tests for ZIP publication task ([#4772](https://github.com/opensearch-project/OpenSearch/pull/4772)) -- Add support for GeoJson Point type in GeoPoint field ([#4597](https://github.com/opensearch-project/OpenSearch/pull/4597)) -- Add dev guide for dealing with flaky tests ([4868](https://github.com/opensearch-project/OpenSearch/pull/4868)) -- Update pull request template ([#4851](https://github.com/opensearch-project/OpenSearch/pull/4851)) -- Added missing no-jdk distributions ([#4722](https://github.com/opensearch-project/OpenSearch/pull/4722)) -- Add dev help in gradle check CI failures ([4872](https://github.com/opensearch-project/OpenSearch/pull/4872)) -- Copy `build.sh` over from opensearch-build ([#4887](https://github.com/opensearch-project/OpenSearch/pull/4887)) - Add project health badges to the README.md ([#4843](https://github.com/opensearch-project/OpenSearch/pull/4843)) -- Added changes for graceful node decommission ([#4586](https://github.com/opensearch-project/OpenSearch/pull/4586)) -- Build no-jdk distributions as part of release build ([#4902](https://github.com/opensearch-project/OpenSearch/pull/4902)) -- Added resource usage trackers for in-flight cancellation of SearchShardTask ([#4805](https://github.com/opensearch-project/OpenSearch/pull/4805)) -- Renamed flaky tests ([#4912](https://github.com/opensearch-project/OpenSearch/pull/4912)) -- Update previous release bwc version to 2.5.0 ([#5003](https://github.com/opensearch-project/OpenSearch/pull/5003)) -- Use getParameterCount instead of getParameterTypes ([#4821](https://github.com/opensearch-project/OpenSearch/pull/4821)) -- Remote shard balancer support for searchable snapshots ([#4870](https://github.com/opensearch-project/OpenSearch/pull/4870)) -- Added search backpressure stats API ([#4932](https://github.com/opensearch-project/OpenSearch/pull/4932)) - [Test] Add IAE test for deprecated edgeNGram analyzer name ([#5040](https://github.com/opensearch-project/OpenSearch/pull/5040)) -- Add Cluster manager task throttling framework. Cluster manager node will throttle task submission based on throttling thresholds. - This throttling will be at task type level. Data nodes will perform retries on these throttling exception with exponential delay. (PR: [#4986](https://github.com/opensearch-project/OpenSearch/pull/4986)) ( Issue : [#479](https://github.com/opensearch-project/OpenSearch/issues/479)) - - Throttling Exception / New Backoff policy([#3527](https://github.com/opensearch-project/OpenSearch/pull/3527)) - - Cluster Manager node side change([#3882](https://github.com/opensearch-project/OpenSearch/pull/3882)) - - Data node side change([#4204](https://github.com/opensearch-project/OpenSearch/pull/4204)) - - on-boarding of tasks([#4542](https://github.com/opensearch-project/OpenSearch/pull/4542)) - - Integs ([4588](https://github.com/opensearch-project/OpenSearch/pull/4588)) -- Integration tests for searchable snapshots ([#5048](https://github.com/opensearch-project/OpenSearch/pull/5048)) -- Update to Gradle 7.6 and JDK-19 ([#4973](https://github.com/opensearch-project/OpenSearch/pull/4973)) ### Dependencies - Bumps `log4j-core` from 2.18.0 to 2.19.0 @@ -85,58 +33,18 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Bumps `com.diffplug.spotless` from 6.10.0 to 6.11.0 ([#4547](https://github.com/opensearch-project/OpenSearch/pull/4547)) - Bumps `reactor-core` from 3.4.18 to 3.4.23 ([#4548](https://github.com/opensearch-project/OpenSearch/pull/4548)) - Bumps `jempbox` from 1.8.16 to 1.8.17 ([#4550](https://github.com/opensearch-project/OpenSearch/pull/4550)) -- Bumps `hadoop-hdfs` from 3.3.3 to 3.3.4 ([#4644](https://github.com/opensearch-project/OpenSearch/pull/4644)) -- Update Jackson Databind to 2.13.4.2 (addressing CVE-2022-42003) ([#4779](https://github.com/opensearch-project/OpenSearch/pull/4779)) -- Bumps `tika` from 2.4.0 to 2.5.0 ([#4791](https://github.com/opensearch-project/OpenSearch/pull/4791)) -- Exclude jettison version brought in with hadoop-minicluster. ([#4787](https://github.com/opensearch-project/OpenSearch/pull/4787)) -- Bump protobuf-java to 3.21.7 in repository-gcs and repository-hdfs ([#4790](https://github.com/opensearch-project/OpenSearch/pull/4790)) -- Bump reactor-netty-http to 1.0.24 in repository-azure ([#4880](https://github.com/opensearch-project/OpenSearch/pull/4880)) -- Bumps `protobuf-java` from 3.21.7 to 3.21.8 ([#4886](https://github.com/opensearch-project/OpenSearch/pull/4886)) -- Upgrade netty to 4.1.84.Final ([#4893](https://github.com/opensearch-project/OpenSearch/pull/4893)) -- Dependency updates: asm 9.3 -> 9.4, bytebuddy 1.12.12 -> 1.12.18 ([#4889](https://github.com/opensearch-project/OpenSearch/pull/4889)) -- Update Apache Lucene to 9.4.1 ([#4922](https://github.com/opensearch-project/OpenSearch/pull/4922)) -- Bump `woodstox-core` to 6.4.0 ([#4947](https://github.com/opensearch-project/OpenSearch/pull/4947)) - Update Apache Lucene to 9.5.0-snapshot-a4ef70f ([#4979](https://github.com/opensearch-project/OpenSearch/pull/4979)) -- Upgrade jetty-http, kotlin-stdlib and snakeyaml ([#4963](https://github.com/opensearch-project/OpenSearch/pull/4963)) -- OpenJDK Update (October 2022 Patch releases) ([#4997](https://github.com/opensearch-project/OpenSearch/pull/4997)) -- Upgrade zookeeper dependency in hdfs-fixture ([#5007](https://github.com/opensearch-project/OpenSearch/pull/5007)) -- Update Jackson to 2.14.0 ([#5105](https://github.com/opensearch-project/OpenSearch/pull/5105)) -- Bump nebula-publishing-plugin from 4.4.4 to 4.6.0 ([#5127](https://github.com/opensearch-project/OpenSearch/pull/5127)) - Bumps `bcpg-fips` from 1.0.5.1 to 1.0.7.1 +- Update to Gradle 7.6 and JDK-19 ([#4973](https://github.com/opensearch-project/OpenSearch/pull/4973)) ### Changed -- Dependency updates (httpcore, mockito, slf4j, httpasyncclient, commons-codec) ([#4308](https://github.com/opensearch-project/OpenSearch/pull/4308)) -- Use RemoteSegmentStoreDirectory instead of RemoteDirectory ([#4240](https://github.com/opensearch-project/OpenSearch/pull/4240)) -- Plugin ZIP publication groupId value is configurable ([#4156](https://github.com/opensearch-project/OpenSearch/pull/4156)) -- Weighted round-robin scheduling policy for shard coordination traffic ([#4241](https://github.com/opensearch-project/OpenSearch/pull/4241)) -- Add index specific setting for remote repository ([#4253](https://github.com/opensearch-project/OpenSearch/pull/4253)) -- [Segment Replication] Update replicas to commit SegmentInfos instead of relying on SIS files from primary shards. ([#4402](https://github.com/opensearch-project/OpenSearch/pull/4402)) - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948)) -- [Remote Store] Change behaviour in replica recovery for remote translog enabled indices ([#4318](https://github.com/opensearch-project/OpenSearch/pull/4318)) -- PUT api for weighted shard routing ([#4272](https://github.com/opensearch-project/OpenSearch/pull/4272)) -- GET api for weighted shard routing([#4275](https://github.com/opensearch-project/OpenSearch/pull/4275/)) -- Unmute test RelocationIT.testRelocationWhileIndexingRandom ([#4580](https://github.com/opensearch-project/OpenSearch/pull/4580)) -- Add DecommissionService and helper to execute awareness attribute decommissioning ([#4084](https://github.com/opensearch-project/OpenSearch/pull/4084)) -- Delete api for weighted shard routing([#4400](https://github.com/opensearch-project/OpenSearch/pull/4400/)) -- Further simplification of the ZIP publication implementation ([#4360](https://github.com/opensearch-project/OpenSearch/pull/4360)) - Relax visibility of the HTTP_CHANNEL_KEY and HTTP_SERVER_CHANNEL_KEY to make it possible for the plugins to access associated Netty4HttpChannel / Netty4HttpServerChannel instance ([#4638](https://github.com/opensearch-project/OpenSearch/pull/4638)) -- Load the deprecated master role in a dedicated method instead of in setAdditionalRoles() ([#4582](https://github.com/opensearch-project/OpenSearch/pull/4582)) -- Add APIs (GET/PUT) to decommission awareness attribute ([#4261](https://github.com/opensearch-project/OpenSearch/pull/4261)) -- Improve Gradle pre-commit checks to pre-empt Jenkins build ([#4660](https://github.com/opensearch-project/OpenSearch/pull/4660)) -- Update to Apache Lucene 9.4.0 ([#4661](https://github.com/opensearch-project/OpenSearch/pull/4661)) -- Controlling discovery for decommissioned nodes ([#4590](https://github.com/opensearch-project/OpenSearch/pull/4590)) -- Backport Apache Lucene version change for 2.4.0 ([#4677](https://github.com/opensearch-project/OpenSearch/pull/4677)) -- Integ Tests for Awareness Attribute Decommissioning ([#4715](https://github.com/opensearch-project/OpenSearch/pull/4715)) - Use ReplicationFailedException instead of OpensearchException in ReplicationTarget ([#4725](https://github.com/opensearch-project/OpenSearch/pull/4725)) -- Fix weighted routing metadata deserialization error on process restart ([#4691](https://github.com/opensearch-project/OpenSearch/pull/4691)) -- Refactor Base Action class javadocs to OpenSearch.API ([#4732](https://github.com/opensearch-project/OpenSearch/pull/4732)) - Migrate client transports to Apache HttpClient / Core 5.x ([#4459](https://github.com/opensearch-project/OpenSearch/pull/4459)) -- Refactored BalancedAllocator.Balancer to LocalShardsBalancer ([#4761](https://github.com/opensearch-project/OpenSearch/pull/4761)) -- Fail weight update when decommission ongoing and fail decommission when attribute not weighed away ([#4839](https://github.com/opensearch-project/OpenSearch/pull/4839)) -- Skip SymbolicLinkPreservingTarIT when running on Windows ([#5023](https://github.com/opensearch-project/OpenSearch/pull/5023)) -- Change the output error message back to use OpenSearchException in the cause chain. ([#5081](https://github.com/opensearch-project/OpenSearch/pull/5081)) ### Deprecated + ### Removed - Remove deprecated code to add node name into log pattern of log4j property file ([#4568](https://github.com/opensearch-project/OpenSearch/pull/4568)) - Unused object and import within TransportClusterAllocationExplainAction ([#4639](https://github.com/opensearch-project/OpenSearch/pull/4639)) @@ -147,99 +55,25 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Remove Legacy Version support from Snapshot/Restore Service ([#4728](https://github.com/opensearch-project/OpenSearch/pull/4728)) - Remove deprecated serialization logic from pipeline aggs ([#4847](https://github.com/opensearch-project/OpenSearch/pull/4847)) - Remove unused private methods ([#4926](https://github.com/opensearch-project/OpenSearch/pull/4926)) -- Revert PR 4656 to unblock Windows CI ([#4949](https://github.com/opensearch-project/OpenSearch/pull/4949)) - Remove LegacyESVersion.V_7_8_ and V_7_9_ Constants ([#4855](https://github.com/opensearch-project/OpenSearch/pull/4855)) - Remove LegacyESVersion.V_7_6_ and V_7_7_ Constants ([#4837](https://github.com/opensearch-project/OpenSearch/pull/4837)) - Remove LegacyESVersion.V_7_10_ Constants ([#5018](https://github.com/opensearch-project/OpenSearch/pull/5018)) - Remove Version.V_1_ Constants ([#5021](https://github.com/opensearch-project/OpenSearch/pull/5021)) ### Fixed -- `opensearch-service.bat start` and `opensearch-service.bat manager` failing to run ([#4289](https://github.com/opensearch-project/OpenSearch/pull/4289)) -- PR reference to checkout code for changelog verifier ([#4296](https://github.com/opensearch-project/OpenSearch/pull/4296)) -- `opensearch.bat` and `opensearch-service.bat install` failing to run, missing logs directory ([#4305](https://github.com/opensearch-project/OpenSearch/pull/4305)) -- Restore using the class ClusterInfoRequest and ClusterInfoRequestBuilder from package 'org.opensearch.action.support.master.info' for subclasses ([#4307](https://github.com/opensearch-project/OpenSearch/pull/4307)) -- Do not fail replica shard due to primary closure ([#4133](https://github.com/opensearch-project/OpenSearch/pull/4133)) -- Add timeout on Mockito.verify to reduce flakyness in testReplicationOnDone test([#4314](https://github.com/opensearch-project/OpenSearch/pull/4314)) -- Commit workflow for dependabot changelog helper ([#4331](https://github.com/opensearch-project/OpenSearch/pull/4331)) -- Fixed cancellation of segment replication events ([#4225](https://github.com/opensearch-project/OpenSearch/pull/4225)) -- [Segment Replication] Bump segment infos counter before commit during replica promotion ([#4365](https://github.com/opensearch-project/OpenSearch/pull/4365)) -- Bugs for dependabot changelog verifier workflow ([#4364](https://github.com/opensearch-project/OpenSearch/pull/4364)) -- Fix flaky random test `NRTReplicationEngineTests.testUpdateSegments` ([#4352](https://github.com/opensearch-project/OpenSearch/pull/4352)) -- [Segment Replication] Extend FileChunkWriter to allow cancel on transport client ([#4386](https://github.com/opensearch-project/OpenSearch/pull/4386)) -- [Segment Replication] Add check to cancel ongoing replication with old primary on onNewCheckpoint on replica ([#4363](https://github.com/opensearch-project/OpenSearch/pull/4363)) -- Fix NoSuchFileExceptions with segment replication when computing primary metadata snapshots ([#4366](https://github.com/opensearch-project/OpenSearch/pull/4366)) -- [Segment Replication] Update flaky testOnNewCheckpointFromNewPrimaryCancelOngoingReplication unit test ([#4414](https://github.com/opensearch-project/OpenSearch/pull/4414)) -- Fixed the `_cat/shards/10_basic.yml` test cases fix. -- [Segment Replication] Fix timeout issue by calculating time needed to process getSegmentFiles ([#4426](https://github.com/opensearch-project/OpenSearch/pull/4426)) -- [Bug]: gradle check failing with java heap OutOfMemoryError ([#4328](https://github.com/opensearch-project/OpenSearch/)) -- `opensearch.bat` fails to execute when install path includes spaces ([#4362](https://github.com/opensearch-project/OpenSearch/pull/4362)) -- Getting security exception due to access denied 'java.lang.RuntimePermission' 'accessDeclaredMembers' when trying to get snapshot with S3 IRSA ([#4469](https://github.com/opensearch-project/OpenSearch/pull/4469)) -- Fixed flaky test `ResourceAwareTasksTests.testTaskIdPersistsInThreadContext` ([#4484](https://github.com/opensearch-project/OpenSearch/pull/4484)) -- Fixed the ignore_malformed setting to also ignore objects ([#4494](https://github.com/opensearch-project/OpenSearch/pull/4494)) -- [Segment Replication] Ignore lock file when testing cleanupAndPreserveLatestCommitPoint ([#4544](https://github.com/opensearch-project/OpenSearch/pull/4544)) -- Updated jackson to 2.13.4 and snakeyml to 1.32 ([#4556](https://github.com/opensearch-project/OpenSearch/pull/4556)) -- Fixing PIT flaky tests ([4632](https://github.com/opensearch-project/OpenSearch/pull/4632)) -- Fixed day of year defaulting for round up parser ([#4627](https://github.com/opensearch-project/OpenSearch/pull/4627)) -- Fixed the SnapshotsInProgress error during index deletion ([#4570](https://github.com/opensearch-project/OpenSearch/pull/4570)) -- [Segment Replication] Adding check to make sure checkpoint is not processed when a shard's shard routing is primary ([#4630](https://github.com/opensearch-project/OpenSearch/pull/4630)) -- [Bug]: Fixed invalid location of JDK dependency for arm64 architecture([#4613](https://github.com/opensearch-project/OpenSearch/pull/4613)) -- [Bug]: Alias filter lost after rollover ([#4499](https://github.com/opensearch-project/OpenSearch/pull/4499)) -- Fixed the SnapshotsInProgress error during index deletion ([#4570](https://github.com/opensearch-project/OpenSearch/pull/4570)) -- [Segment Replication] Adding check to make sure checkpoint is not processed when a shard's shard routing is primary ([#4630](https://github.com/opensearch-project/OpenSearch/pull/4630)) -- [Bug]: Fixed invalid location of JDK dependency for arm64 architecture([#4613](https://github.com/opensearch-project/OpenSearch/pull/4613)) -- [Bug]: Alias filter lost after rollover ([#4499](https://github.com/opensearch-project/OpenSearch/pull/4499)) -- Fixed misunderstanding message "No OpenSearchException found" when detailed_error disabled ([#4669](https://github.com/opensearch-project/OpenSearch/pull/4669)) -- Attempt to fix Github workflow for Gradle Check job ([#4679](https://github.com/opensearch-project/OpenSearch/pull/4679)) -- Fix flaky DecommissionControllerTests.testTimesOut ([4683](https://github.com/opensearch-project/OpenSearch/pull/4683)) -- Fix new race condition in DecommissionControllerTests ([4688](https://github.com/opensearch-project/OpenSearch/pull/4688)) -- Fix SearchStats (de)serialization (caused by https://github.com/opensearch-project/OpenSearch/pull/4616) ([#4697](https://github.com/opensearch-project/OpenSearch/pull/4697)) -- Fixing Gradle warnings associated with publishPluginZipPublicationToXxx tasks ([#4696](https://github.com/opensearch-project/OpenSearch/pull/4696)) -- [BUG]: Remove redundant field from GetDecommissionStateResponse ([#4751](https://github.com/opensearch-project/OpenSearch/pull/4751)) -- Fixed randomly failing test ([4774](https://github.com/opensearch-project/OpenSearch/pull/4774)) -- Update version check after backport ([4786](https://github.com/opensearch-project/OpenSearch/pull/4786)) -- Fix decommission status update to non leader nodes ([4800](https://github.com/opensearch-project/OpenSearch/pull/4800)) -- Fix recovery path for searchable snapshots ([4813](https://github.com/opensearch-project/OpenSearch/pull/4813)) -- Fix bug in AwarenessAttributeDecommissionIT([4822](https://github.com/opensearch-project/OpenSearch/pull/4822)) - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827)) -- [BUG]: flaky test index/80_geo_point/Single point test([#4860](https://github.com/opensearch-project/OpenSearch/pull/4860)) -- Fix bug in SlicedInputStream with zero length ([#4863](https://github.com/opensearch-project/OpenSearch/pull/4863)) -- Fix a bug on handling an invalid array value for point type field ([#4900](https://github.com/opensearch-project/OpenSearch/pull/4900)) -- [Segment Replication] Fix bug of replica shard's translog not purging on index flush when segment replication is enabled ([4928](https://github.com/opensearch-project/OpenSearch/pull/4928)) -- Fix a bug on handling an invalid array value for point type field #4900([#4900](https://github.com/opensearch-project/OpenSearch/pull/4900)) -- [BUG]: Allow decommission to support delay timeout ([#4930](https://github.com/opensearch-project/OpenSearch/pull/4930)) -- Fix failing test: VerifyVersionConstantsIT ([#4946](https://github.com/opensearch-project/OpenSearch/pull/4946)) - Fixed compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944)) -- Fix for failing checkExtraction, checkLicense and checkNotice tasks for windows gradle check ([#4941](https://github.com/opensearch-project/OpenSearch/pull/4941)) -- Backport failures for merge conflicts on CHANGELOG.md file ([#4977](https://github.com/opensearch-project/OpenSearch/pull/4977)) -- Remove gradle-check dependency on precommit [#5027](https://github.com/opensearch-project/OpenSearch/pull/5027) -- Fix version check for 2.x release for awareness attribute decommission([#5034](https://github.com/opensearch-project/OpenSearch/pull/5034)) -- Fix flaky test ResourceAwareTasksTests on Windows ([#5077](https://github.com/opensearch-project/OpenSearch/pull/5077)) -- Length calculation for block based fetching ([#5055](https://github.com/opensearch-project/OpenSearch/pull/5055)) -- [Segment Replication] Fix for AlreadyClosedException for engine ([#4743](https://github.com/opensearch-project/OpenSearch/pull/4743)) -- Fix error handling while reading analyzer mapping rules + ### Security -- CVE-2022-25857 org.yaml:snakeyaml DOS vulnerability ([#4341](https://github.com/opensearch-project/OpenSearch/pull/4341)) -## [2.x] +## [Unreleased 2.x] ### Added -- Github workflow for changelog verification ([#4085](https://github.com/opensearch-project/OpenSearch/pull/4085)) -- Label configuration for dependabot PRs ([#4348](https://github.com/opensearch-project/OpenSearch/pull/4348)) -- Added RestLayer Changes for PIT stats ([#4217](https://github.com/opensearch-project/OpenSearch/pull/4217)) -- Added GeoBounds aggregation on GeoShape field type.([#4266](https://github.com/opensearch-project/OpenSearch/pull/4266)) - - Addition of Doc values on the GeoShape Field - - Addition of GeoShape ValueSource level code interfaces for accessing the DocValues. - - Addition of Missing Value feature in the GeoShape Aggregations. -- Install and configure Log4j JUL Adapter for Lucene 9.4 ([#4754](https://github.com/opensearch-project/OpenSearch/pull/4754)) -- Added feature to ignore indexes starting with dot during shard limit validation.([#4695](https://github.com/opensearch-project/OpenSearch/pull/4695)) +### Dependencies ### Changed ### Deprecated ### Removed ### Fixed -- PR reference to checkout code for changelog verifier ([#4296](https://github.com/opensearch-project/OpenSearch/pull/4296)) -- Commit workflow for dependabot changelog helper ([#4331](https://github.com/opensearch-project/OpenSearch/pull/4331)) -- Better plural stemmer than minimal_english ([#4738](https://github.com/opensearch-project/OpenSearch/pull/4738)) -- Disable merge on refresh in DiskThresholdDeciderIT ([#4828](https://github.com/opensearch-project/OpenSearch/pull/4828)) - ### Security -[Unreleased]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...HEAD -[2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...2.x \ No newline at end of file + +[Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.4...HEAD +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.4...2.x diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc02d52f0bc3b..d379d78829318 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -119,20 +119,39 @@ You may type this line on your own when writing your commit messages. However, i ## Changelog -OpenSearch maintains version specific changelog by enforcing a change to the ongoing [CHANGELOG](CHANGELOG.md) file adhering to the [Keep A Changelog](https://keepachangelog.com/en/1.0.0/) format. +OpenSearch maintains version specific changelog by enforcing a change to the ongoing [CHANGELOG](CHANGELOG.md) file adhering to the [Keep A Changelog](https://keepachangelog.com/en/1.0.0/) format. The purpose of the changelog is for the contributors and maintainers to incrementally build the release notes throughout the development process to avoid a painful and error-prone process of attempting to compile the release notes at release time. On each release the "unreleased" entries of the changelog are moved to the appropriate release notes document in the `./release-notes` folder. Also, incrementally building the changelog provides a concise, human-readable list of significant features that have been added to the unreleased version under development. -Briefly, the changes are curated by version, with the changes to the main branch added chronologically to `Unreleased` version. Further, each version has corresponding sections which list out the category of the change - `Added`, `Changed`, `Deprecated`, `Removed`, `Fixed`, `Security`. +### Which changes require a CHANGELOG entry? +Changelogs are intended for operators/administrators, developers integrating with libraries and APIs, and end-users interacting with OpenSearch Dashboards and/or the REST API (collectively referred to as "user"). In short, any change that a user of OpenSearch might want to be aware of should be included in the changelog. The changelog is _not_ intended to replace the git commit log that developers of OpenSearch itself rely upon. The following are some examples of changes that should be in the changelog: +- A newly added feature +- A fix for a user-facing bug +- Dependency updates +- Fixes for security issues -### How to add my changes to [CHANGELOG](CHANGELOG.md)? +The following are some examples where a changelog entry is not necessary: -As a contributor, you must ensure that every pull request has the changes listed out within the corresponding version and appropriate section of [CHANGELOG](CHANGELOG.md) file. +- Adding, modifying, or fixing tests +- An incremental PR for a larger feature (such features should include _one_ changelog entry for the feature) +- Documentation changes or code refactoring +- Build-related changes -Adding in the change is two step process - -1. Add your changes to the corresponding section within the CHANGELOG file with dummy pull request information, publish the PR +Any PR that does not include a changelog entry will result in a failure of the validation workflow in GitHub. If the contributor and maintainers agree that no changelog entry is required, then the `skip-changelog` label can be applied to the PR which will result in the workflow passing. + +### How to add my changes to [CHANGELOG](CHANGELOG.md)? +Adding in the change is two step process: +1. Add your changes to the corresponding section within the CHANGELOG file with dummy pull request information, publish the PR 2. Update the entry for your change in [`CHANGELOG.md`](CHANGELOG.md) and make sure that you reference the pull request there. +### Where should I put my CHANGELOG entry? +Please review the [branching strategy](https://github.com/opensearch-project/.github/blob/main/RELEASING.md#opensearch-branching) document. The changelog on the `main` branch will contain sections for the _next major_ and _next minor_ releases. Your entry should go into the section it is intended to be released in. In practice, most changes to `main` will be backported to the next minor release so most entries will likely be in that section. + +The following examples assume the _next major_ release on main is 3.0, then _next minor_ release is 2.5, and the _current_ release is 2.4. + +- **Add a new feature to release in next minor:** Add a changelog entry to `[Unreleased 2.x]` on main, then backport to 2.x (including the changelog entry). +- **Introduce a breaking API change to release in next major:** Add a changelog entry to `[Unreleased 3.0]` on main, do not backport. +- **Upgrade a dependency to fix a CVE:** Add a changelog entry to `[Unreleased 2.x]` on main, then backport to 2.x (including the changelog entry), then backport to 2.4 and ensure the changelog entry is added to `[Unreleased 2.4.1]`. ## Review Process diff --git a/release-notes/opensearch.release-notes-2.4.0.md b/release-notes/opensearch.release-notes-2.4.0.md new file mode 100644 index 0000000000000..a2c9311f4159b --- /dev/null +++ b/release-notes/opensearch.release-notes-2.4.0.md @@ -0,0 +1,100 @@ +## 2022-11-04 Version 2.4.0 Release Notes + +## [2.4] +### Added +- Introduce point in time search feature ([#3959](https://github.com/opensearch-project/OpenSearch/issues/3959)) +- Introduce experimental searchable snapshot feature ([#2919](https://github.com/opensearch-project/OpenSearch/issues/2919)) +- Add API for decommissioning/recommissioning zone and weighted zonal search request routing policy ([#3639](https://github.com/opensearch-project/OpenSearch/issues/3639)) +- Introduce cluster manager task throttling framework [#479](https://github.com/opensearch-project/OpenSearch/issues/479) +- Add support for s390x architecture ([#4001](https://github.com/opensearch-project/OpenSearch/pull/4001)) +- Github workflow for changelog verification ([#4085](https://github.com/opensearch-project/OpenSearch/pull/4085)) +- Add failover support with Segment Replication enabled. ([#4325](https://github.com/opensearch-project/OpenSearch/pull/4325) +- Add a new node role 'search' which is dedicated to provide search capability ([#4689](https://github.com/opensearch-project/OpenSearch/pull/4689)) +- Introduce Remote translog feature flag([#4158](https://github.com/opensearch-project/OpenSearch/pull/4158)) +- Add groupId value propagation tests for ZIP publication task ([#4848](https://github.com/opensearch-project/OpenSearch/pull/4848)) +- Add support for GeoJson Point type in GeoPoint field ([#4597](https://github.com/opensearch-project/OpenSearch/pull/4597)) +- Add missing no-jdk distributions ([#4722](https://github.com/opensearch-project/OpenSearch/pull/4722)) +- Copy `build.sh` over from opensearch-build ([#4887](https://github.com/opensearch-project/OpenSearch/pull/4887)) +- Update GeoGrid base class access modifier to support extensibility ([#4921](https://github.com/opensearch-project/OpenSearch/pull/4921)) +- Build no-jdk distributions as part of release build ([#4902](https://github.com/opensearch-project/OpenSearch/pull/4902)) +- Add in-flight cancellation of SearchShardTask based on resource consumption ([#4565](https://github.com/opensearch-project/OpenSearch/pull/4565)) +- Add resource usage trackers for in-flight cancellation of SearchShardTask ([#4805](https://github.com/opensearch-project/OpenSearch/pull/4805)) +- Add search backpressure stats API ([#4932](https://github.com/opensearch-project/OpenSearch/pull/4932)) +- Add feature to ignore indexes starting with dot during shard limit validation.([#4695](https://github.com/opensearch-project/OpenSearch/pull/4695)) + +### Dependencies +- Bump `com.diffplug.spotless` from 6.9.1 to 6.10.0 +- Bump `xmlbeans` from 5.1.0 to 5.1.1 +- Bump `hadoop-hdfs` from 3.3.3 to 3.3.4 +- Exclude jettison version brought in with hadoop-minicluster. ([#4787](https://github.com/opensearch-project/OpenSearch/pull/4787)) +- Bump protobuf-java to 3.21.7 in repository-gcs and repository-hdfs ([#4790](https://github.com/opensearch-project/OpenSearch/pull/4790)) +- Bump reactor-netty-http to 1.0.24 in repository-azure ([#4880](https://github.com/opensearch-project/OpenSearch/pull/4880)) +- Add dev help in gradle check CI failures ([4872](https://github.com/opensearch-project/OpenSearch/pull/4872)) +- Upgrade netty to 4.1.84.Final ([#4893](https://github.com/opensearch-project/OpenSearch/pull/4893)) +- Dependency updates: asm 9.3 -> 9.4, bytebuddy 1.12.12 -> 1.12.18 ([#4889](https://github.com/opensearch-project/OpenSearch/pull/4889)) +- Bump `tika` from 2.4.0 to 2.5.0 ([#4791](https://github.com/opensearch-project/OpenSearch/pull/4791)) +- Update Apache Lucene to 9.4.1 ([#4922](https://github.com/opensearch-project/OpenSearch/pull/4922)) +- Bump `woodstox-core` to 6.4.0 ([#4951](https://github.com/opensearch-project/OpenSearch/pull/4951)) +- Upgrade jetty-http, kotlin-stdlib and snakeyaml ([#4982](https://github.com/opensearch-project/OpenSearch/pull/4982)) +- OpenJDK Update (October 2022 Patch releases) ([#4997](https://github.com/opensearch-project/OpenSearch/pull/4997)) +- Upgrade zookeeper dependency in hdfs-fixture ([#5007](https://github.com/opensearch-project/OpenSearch/pull/5007)) +- Dependency updates (httpcore, mockito, slf4j, httpasyncclient, commons-codec) ([#4308](https://github.com/opensearch-project/OpenSearch/pull/4308)) +- Update to Apache Lucene 9.4.0 ([#4661](https://github.com/opensearch-project/OpenSearch/pull/4661)) +- Updated jackson to 2.13.4 and snakeyml to 1.32 ([#4556](https://github.com/opensearch-project/OpenSearch/pull/4556)) +- Update Jackson Databind to 2.13.4.2 (addressing CVE-2022-42003) ([#4781](https://github.com/opensearch-project/OpenSearch/pull/4781)) +- Bump protobuf-java to 3.21.8 ([#5005](https://github.com/opensearch-project/OpenSearch/pull/5005)) +- Upgrade zookeeper dependency in hdfs-fixture ([#5047](https://github.com/opensearch-project/OpenSearch/pull/5047)) + +### Changed +- Use RemoteSegmentStoreDirectory instead of RemoteDirectory ([#4240](https://github.com/opensearch-project/OpenSearch/pull/4240)) +- Weighted round-robin scheduling policy for shard coordination traffic ([#4241](https://github.com/opensearch-project/OpenSearch/pull/4241)) +- Add index specific setting for remote repository ([#4253](https://github.com/opensearch-project/OpenSearch/pull/4253)) +- Update replicas to commit SegmentInfos instead of relying on SIS files from primary shards. ([#4402](https://github.com/opensearch-project/OpenSearch/pull/4402)) +- Change the version to remove deprecated code of adding node name into log pattern of log4j property file ([#4569](https://github.com/opensearch-project/OpenSearch/pull/4569)) +- Load the deprecated master role in a dedicated method instead of in setAdditionalRoles() ([#4582](https://github.com/opensearch-project/OpenSearch/pull/4582)) +- Plugin ZIP publication groupId value is configurable ([#4156](https://github.com/opensearch-project/OpenSearch/pull/4156)) +- Further simplification of the ZIP publication implementation ([#4360](https://github.com/opensearch-project/OpenSearch/pull/4360)) +- Change behaviour in replica recovery for remote translog enabled indices ([#4318](https://github.com/opensearch-project/OpenSearch/pull/4318)) +- PUT api for weighted shard routing ([#4272](https://github.com/opensearch-project/OpenSearch/pull/4272)) +- GET api for weighted shard routing([#4275](https://github.com/opensearch-project/OpenSearch/pull/4275/)) +- Delete api for weighted shard routing([#4400](https://github.com/opensearch-project/OpenSearch/pull/4400/)) +- Fix weighted routing metadata deserialization error on process restart ([#4691](https://github.com/opensearch-project/OpenSearch/pull/4691)) +- Install and configure Log4j JUL Adapter for Lucene 9.4 ([#4754](https://github.com/opensearch-project/OpenSearch/pull/4754)) +- Use getParameterCount instead of getParameterTypes ([#4821](https://github.com/opensearch-project/OpenSearch/pull/4821)) + +### Removed +- Remove RepositoryData.MIN_VERSION support for next major release ([4729](https://github.com/opensearch-project/OpenSearch/pull/4729)) + +### Fixed +- `opensearch-service.bat start` and `opensearch-service.bat manager` failing to run ([#4289](https://github.com/opensearch-project/OpenSearch/pull/4289)) +- PR reference to checkout code for changelog verifier ([#4296](https://github.com/opensearch-project/OpenSearch/pull/4296)) +- `opensearch.bat` and `opensearch-service.bat install` failing to run, missing logs directory ([#4305](https://github.com/opensearch-project/OpenSearch/pull/4305)) +- Restore using the class ClusterInfoRequest and ClusterInfoRequestBuilder from package 'org.opensearch.action.support.master.info' for subclasses ([#4307](https://github.com/opensearch-project/OpenSearch/pull/4307)) +- Do not fail replica shard due to primary closure ([#4133](https://github.com/opensearch-project/OpenSearch/pull/4133)) +- Fixed cancellation of segment replication events ([#4225](https://github.com/opensearch-project/OpenSearch/pull/4225)) +- `opensearch.bat` fails to execute when install path includes spaces ([#4362](https://github.com/opensearch-project/OpenSearch/pull/4362)) +- Getting security exception due to access denied 'java.lang.RuntimePermission' 'accessDeclaredMembers' when trying to get snapshot with S3 IRSA ([#4469](https://github.com/opensearch-project/OpenSearch/pull/4469)) +- Fix the ignore_malformed setting to also ignore objects ([#4494](https://github.com/opensearch-project/OpenSearch/pull/4494)) +- Fix day of year defaulting for round up parser ([#4627](https://github.com/opensearch-project/OpenSearch/pull/4627)) +- Fix the SnapshotsInProgress error during index deletion ([#4570](https://github.com/opensearch-project/OpenSearch/pull/4570)) +- Fix invalid location of JDK dependency for arm64 architecture([#4613](https://github.com/opensearch-project/OpenSearch/pull/4613)) +- Alias filter lost after rollover ([#4499](https://github.com/opensearch-project/OpenSearch/pull/4499)) +- Fix Gradle warnings associated with publishPluginZipPublicationToXxx tasks ([#4696](https://github.com/opensearch-project/OpenSearch/pull/4696)) +- Fix a bug on handling an invalid array value for point type field #4900([#4900](https://github.com/opensearch-project/OpenSearch/pull/4900)) +- Set analyzer to regex query string search ([4219](https://github.com/opensearch-project/OpenSearch/pull/4219)) +- Better plural stemmer than minimal_english ([#4738](https://github.com/opensearch-project/OpenSearch/pull/4738)) +- Add check to cancel ongoing replication with old primary on onNewCheckpoint on replica ([#4363](https://github.com/opensearch-project/OpenSearch/pull/4363)) +- Bump segment infos counter before commit during replica promotion ([#4365](https://github.com/opensearch-project/OpenSearch/pull/4365)) +- Update flaky testOnNewCheckpointFromNewPrimaryCancelOngoingReplication unit test ([#4414](https://github.com/opensearch-project/OpenSearch/pull/4414)) +- Extend FileChunkWriter to allow cancel on transport client ([#4386](https://github.com/opensearch-project/OpenSearch/pull/4386)) +- Fix NoSuchFileExceptions with segment replication when computing primary metadata snapshots ([#4366](https://github.com/opensearch-project/OpenSearch/pull/4366)) +- Fix timeout issue by calculating time needed to process getSegmentFiles ([#4434](https://github.com/opensearch-project/OpenSearch/pull/4434)) +- Update replicas to commit SegmentInfos instead of relying on segments_N from primary shards ([#4450](https://github.com/opensearch-project/OpenSearch/pull/4450)) +- Adding check to make sure checkpoint is not processed when a shard's shard routing is primary ([#4716](https://github.com/opensearch-project/OpenSearch/pull/4716)) +- Fix bug of replica shard's translog not purging on index flush when segment replication is enabled ([4975](https://github.com/opensearch-project/OpenSearch/pull/4975)) +- Fix message "No OpenSearchException found" when detailed_error disabled by return meaningful messages ([#4708](https://github.com/opensearch-project/OpenSearch/pull/4708)) +- Add fix for auto expand replica validation ([#4994](https://github.com/opensearch-project/OpenSearch/pull/4994)) +- Fix build failures on the Windows platform ([#4924](https://github.com/opensearch-project/OpenSearch/issues/4924)) + +### Security +- CVE-2022-25857 org.yaml:snakeyaml DOS vulnerability ([#4341](https://github.com/opensearch-project/OpenSearch/pull/4341)) From 88757b4c3a2cd772527a8867957ca17e785629a4 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 8 Nov 2022 15:22:16 -0500 Subject: [PATCH 07/23] Fix: org.opensearch.clustermanager.ClusterManagerTaskThrottlingIT is flaky. (#5152) Signed-off-by: dblock Signed-off-by: dblock --- .../clustermanager/ClusterManagerTaskThrottlingIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/clustermanager/ClusterManagerTaskThrottlingIT.java b/server/src/internalClusterTest/java/org/opensearch/clustermanager/ClusterManagerTaskThrottlingIT.java index 2f3f0539602dd..328bc14883208 100644 --- a/server/src/internalClusterTest/java/org/opensearch/clustermanager/ClusterManagerTaskThrottlingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/clustermanager/ClusterManagerTaskThrottlingIT.java @@ -164,9 +164,9 @@ public void onResponse(Object o) { @Override public void onFailure(Exception e) { + timedoutRequest.incrementAndGet(); latch.countDown(); assertTrue(e instanceof ProcessClusterEventTimeoutException); - timedoutRequest.incrementAndGet(); } }; executePutMappingRequests(totalRequest, node, listener); From 6cbca612c4fd0d9d374236e85e31abb5692107fc Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 8 Nov 2022 17:25:43 -0500 Subject: [PATCH 08/23] Fix: raise error on malformed CSV. (#5143) Signed-off-by: dblock --- .../index/analysis/KuromojiTokenizerFactory.java | 3 +++ .../opensearch/index/analysis/KuromojiAnalysisTests.java | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/plugins/analysis-kuromoji/src/main/java/org/opensearch/index/analysis/KuromojiTokenizerFactory.java b/plugins/analysis-kuromoji/src/main/java/org/opensearch/index/analysis/KuromojiTokenizerFactory.java index ac4def8ac9a11..2939711f6f7e1 100644 --- a/plugins/analysis-kuromoji/src/main/java/org/opensearch/index/analysis/KuromojiTokenizerFactory.java +++ b/plugins/analysis-kuromoji/src/main/java/org/opensearch/index/analysis/KuromojiTokenizerFactory.java @@ -79,6 +79,9 @@ public KuromojiTokenizerFactory(IndexSettings indexSettings, Environment env, St private static String parse(String rule, Set dup) { String[] values = CSVUtil.parse(rule); + if (values.length == 0) { + throw new IllegalArgumentException("Malformed csv in user dictionary."); + } if (dup.add(values[0]) == false) { throw new IllegalArgumentException("Found duplicate term [" + values[0] + "] in user dictionary."); } diff --git a/plugins/analysis-kuromoji/src/test/java/org/opensearch/index/analysis/KuromojiAnalysisTests.java b/plugins/analysis-kuromoji/src/test/java/org/opensearch/index/analysis/KuromojiAnalysisTests.java index 23e6ef9fea059..03d9df6ebd6b2 100644 --- a/plugins/analysis-kuromoji/src/test/java/org/opensearch/index/analysis/KuromojiAnalysisTests.java +++ b/plugins/analysis-kuromoji/src/test/java/org/opensearch/index/analysis/KuromojiAnalysisTests.java @@ -379,6 +379,15 @@ public void testKuromojiAnalyzerInvalidUserDictOption() throws Exception { ); } + public void testKuromojiAnalyzerEmptyDictRule() throws Exception { + Settings settings = Settings.builder() + .put("index.analysis.analyzer.my_analyzer.type", "kuromoji") + .putList("index.analysis.analyzer.my_analyzer.user_dictionary_rules", "\"") + .build(); + RuntimeException exc = expectThrows(RuntimeException.class, () -> createTestAnalysis(settings)); + assertThat(exc.getMessage(), equalTo("Line [1]: Malformed csv in user dictionary.")); + } + public void testKuromojiAnalyzerDuplicateUserDictRule() throws Exception { Settings settings = Settings.builder() .put("index.analysis.analyzer.my_analyzer.type", "kuromoji") From fc475721d261d719aeb7b4c2f635f1de63f0f345 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 8 Nov 2022 17:27:38 -0500 Subject: [PATCH 09/23] Allow mmap to use new preview JDK-19 APIs in Apache Lucene 9.4+ (#5151) * Allow mmap to use new preview JDK-19 APIs in Apache Lucene 9.4+ Signed-off-by: Andriy Redko * Align javadoc release with target compatibility settings Signed-off-by: Andriy Redko Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + build.gradle | 8 ++++---- distribution/src/config/jvm.options | 3 +++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7920007bad2cc..26ba49fcbdd76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Apply reproducible builds configuration for OpenSearch plugins through gradle plugin ([#4746](https://github.com/opensearch-project/OpenSearch/pull/4746)) - Add project health badges to the README.md ([#4843](https://github.com/opensearch-project/OpenSearch/pull/4843)) - [Test] Add IAE test for deprecated edgeNGram analyzer name ([#5040](https://github.com/opensearch-project/OpenSearch/pull/5040)) +- Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151)) ### Dependencies - Bumps `log4j-core` from 2.18.0 to 2.19.0 diff --git a/build.gradle b/build.gradle index 1fb6cc3849710..076894863b1bf 100644 --- a/build.gradle +++ b/build.gradle @@ -316,10 +316,7 @@ allprojects { javadoc.options.encoding = 'UTF8' javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet') javadoc.options.tags = ["opensearch.internal", "opensearch.api", "opensearch.experimental"] - if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_19) { - javadoc.options.addBooleanOption("-enable-preview", true) - javadoc.options.addStringOption("-release", BuildParams.runtimeJavaVersion.majorVersion) - } + javadoc.options.addStringOption("-release", targetCompatibility.majorVersion) } // support for reproducible builds @@ -416,6 +413,9 @@ gradle.projectsEvaluated { if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17) { task.jvmArgs += ["-Djava.security.manager=allow"] } + if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_19) { + task.jvmArgs += ["--enable-preview"] + } } } diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index ef1035489c9fc..6cd5feadbef87 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -78,3 +78,6 @@ ${error.file} # Explicitly allow security manager (https://bugs.openjdk.java.net/browse/JDK-8270380) 18-:-Djava.security.manager=allow + +# Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4 (https://github.com/opensearch-project/OpenSearch/issues/4637) +19-:--enable-preview From 9970617eb34599127865fe01574452d54c95dfee Mon Sep 17 00:00:00 2001 From: Rabi Panda Date: Tue, 8 Nov 2022 14:49:03 -0800 Subject: [PATCH 10/23] Fix test to use a file from another temp directory (#5158) The test used an absolute path which is incorrect as it made a platform dependent assumption. This change fixes that. Signed-off-by: Rabi Panda Signed-off-by: Rabi Panda --- .../opensearch/index/analysis/AnalysisTests.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/opensearch/index/analysis/AnalysisTests.java b/server/src/test/java/org/opensearch/index/analysis/AnalysisTests.java index 01281ea323e60..0446ac78d4efc 100644 --- a/server/src/test/java/org/opensearch/index/analysis/AnalysisTests.java +++ b/server/src/test/java/org/opensearch/index/analysis/AnalysisTests.java @@ -136,14 +136,24 @@ public void testParseWordListError() throws IOException { assertEquals("Line [1]: Error while parsing rule = abcd", ex.getMessage()); } - public void testParseWordListOutsideConfigDirError() { + public void testParseWordListOutsideConfigDirError() throws IOException { Path home = createTempDir(); - Path dict = home.resolve("/etc/os-release"); + Path temp = createTempDir(); + Path dict = temp.resolve("foo.dict"); + try (BufferedWriter writer = Files.newBufferedWriter(dict, StandardCharsets.UTF_8)) { + writer.write("abcd"); + writer.write('\n'); + } Settings nodeSettings = Settings.builder().put("foo.bar_path", dict).put(Environment.PATH_HOME_SETTING.getKey(), home).build(); Environment env = TestEnvironment.newEnvironment(nodeSettings); RuntimeException ex = expectThrows( RuntimeException.class, - () -> Analysis.parseWordList(env, nodeSettings, "foo.bar", s -> { throw new RuntimeException("Error while parsing"); }) + () -> Analysis.parseWordList( + env, + nodeSettings, + "foo.bar", + s -> { throw new RuntimeException("Error while parsing rule = " + s); } + ) ); assertEquals("Line [1]: Invalid rule", ex.getMessage()); } From 140ac9753db373a3511cd35372385b83c9ddbf78 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 8 Nov 2022 18:36:26 -0500 Subject: [PATCH 11/23] Fix race condition in testThrottlingForSingleNode. (#5159) Signed-off-by: dblock Signed-off-by: dblock --- .../clustermanager/ClusterManagerTaskThrottlingIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/clustermanager/ClusterManagerTaskThrottlingIT.java b/server/src/internalClusterTest/java/org/opensearch/clustermanager/ClusterManagerTaskThrottlingIT.java index 328bc14883208..9817861c88e9a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/clustermanager/ClusterManagerTaskThrottlingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/clustermanager/ClusterManagerTaskThrottlingIT.java @@ -116,8 +116,8 @@ public void testThrottlingForSingleNode() throws Exception { ActionListener listener = new ActionListener() { @Override public void onResponse(Object o) { - latch.countDown(); successfulRequest.incrementAndGet(); + latch.countDown(); } @Override From 0eb6b2f49286a92c7f78e5c55c994c2bec4656f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 16:41:22 -0800 Subject: [PATCH 12/23] Bump azure-storage-blob from 12.16.1 to 12.20.0 in /plugins/repository-azure (#4995) * Bump azure-storage-blob in /plugins/repository-azure Bumps [azure-storage-blob](https://github.com/Azure/azure-sdk-for-java) from 12.16.1 to 12.20.0. - [Release notes](https://github.com/Azure/azure-sdk-for-java/releases) - [Commits](https://github.com/Azure/azure-sdk-for-java/compare/azure-storage-blob_12.16.1...azure-storage-blob_12.20.0) --- updated-dependencies: - dependency-name: com.azure:azure-storage-blob dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/repository-azure/build.gradle | 2 +- .../licenses/azure-storage-blob-12.16.1.jar.sha1 | 1 - .../licenses/azure-storage-blob-12.20.0.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-azure/licenses/azure-storage-blob-12.16.1.jar.sha1 create mode 100644 plugins/repository-azure/licenses/azure-storage-blob-12.20.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 26ba49fcbdd76..e2e52ce57e799 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Update Apache Lucene to 9.5.0-snapshot-a4ef70f ([#4979](https://github.com/opensearch-project/OpenSearch/pull/4979)) - Bumps `bcpg-fips` from 1.0.5.1 to 1.0.7.1 - Update to Gradle 7.6 and JDK-19 ([#4973](https://github.com/opensearch-project/OpenSearch/pull/4973)) +- Bumps `azure-storage-blob` from 12.16.1 to 12.20.0 ([#4995](https://github.com/opensearch-project/OpenSearch/pull/4995)) ### Changed - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948)) diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 063851b3a7edf..3aa2bbb7dd2f6 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -54,7 +54,7 @@ dependencies { api "io.netty:netty-resolver-dns:${versions.netty}" api "io.netty:netty-transport-native-unix-common:${versions.netty}" implementation project(':modules:transport-netty4') - api 'com.azure:azure-storage-blob:12.16.1' + api 'com.azure:azure-storage-blob:12.20.0' api 'org.reactivestreams:reactive-streams:1.0.4' api 'io.projectreactor:reactor-core:3.4.23' api 'io.projectreactor.netty:reactor-netty:1.0.18' diff --git a/plugins/repository-azure/licenses/azure-storage-blob-12.16.1.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-blob-12.16.1.jar.sha1 deleted file mode 100644 index 71014103e517b..0000000000000 --- a/plugins/repository-azure/licenses/azure-storage-blob-12.16.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -84054ca8a6660eb77910925d71f70330fd3d83aa \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-storage-blob-12.20.0.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-blob-12.20.0.jar.sha1 new file mode 100644 index 0000000000000..de86848c9fd06 --- /dev/null +++ b/plugins/repository-azure/licenses/azure-storage-blob-12.20.0.jar.sha1 @@ -0,0 +1 @@ +e682920b0e3115433f25d65b0718f8763035357e \ No newline at end of file From dcb80bc3aac9530073d5d87b10ca918cbadbe95f Mon Sep 17 00:00:00 2001 From: Xiao Date: Wed, 9 Nov 2022 12:04:44 -0500 Subject: [PATCH 13/23] Cleanup: remove unused semicolon (#5169) [QA: None] Signed-off-by: Xiao Cui Signed-off-by: Xiao Cui --- .../java/org/opensearch/percolator/PercolateQueryBuilder.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java index b2130eca3bb02..4f4665c434c67 100644 --- a/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java @@ -546,7 +546,6 @@ protected Analyzer getWrappedAnalyzer(String fieldName) { String name = this.name != null ? this.name : pft.name(); QueryShardContext percolateShardContext = wrap(context); PercolatorFieldMapper.configureContext(percolateShardContext, pft.mapUnmappedFieldsAsText); - ; PercolateQuery.QueryStore queryStore = createStore(pft.queryBuilderField, percolateShardContext); return pft.percolateQuery(name, queryStore, documents, docSearcher, excludeNestedDocuments, context.indexVersionCreated()); From fadf409e3c861256b5b0b49936646025c9047c1d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Nov 2022 09:34:47 -0800 Subject: [PATCH 14/23] Bump commons-compress from 1.21 to 1.22 in /plugins/ingest-attachment (#5104) * Bump commons-compress from 1.21 to 1.22 in /plugins/ingest-attachment Bumps commons-compress from 1.21 to 1.22. --- updated-dependencies: - dependency-name: org.apache.commons:commons-compress dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Update changelog Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- CHANGELOG.md | 1 + plugins/ingest-attachment/build.gradle | 2 +- .../ingest-attachment/licenses/commons-compress-1.21.jar.sha1 | 1 - .../ingest-attachment/licenses/commons-compress-1.22.jar.sha1 | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 plugins/ingest-attachment/licenses/commons-compress-1.21.jar.sha1 create mode 100644 plugins/ingest-attachment/licenses/commons-compress-1.22.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index e2e52ce57e799..35a2c18f35792 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bumps `bcpg-fips` from 1.0.5.1 to 1.0.7.1 - Update to Gradle 7.6 and JDK-19 ([#4973](https://github.com/opensearch-project/OpenSearch/pull/4973)) - Bumps `azure-storage-blob` from 12.16.1 to 12.20.0 ([#4995](https://github.com/opensearch-project/OpenSearch/pull/4995)) +- Bumps `commons-compress` from 1.21 to 1.22 ([#5104](https://github.com/opensearch-project/OpenSearch/pull/5104)) ### Changed - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948)) diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle index 7bf67769cda10..f42b44b56ccb8 100644 --- a/plugins/ingest-attachment/build.gradle +++ b/plugins/ingest-attachment/build.gradle @@ -84,7 +84,7 @@ dependencies { // MS Office api "org.apache.poi:poi-scratchpad:${versions.poi}" // Apple iWork - api 'org.apache.commons:commons-compress:1.21' + api 'org.apache.commons:commons-compress:1.22' // Outlook documents api "org.apache.james:apache-mime4j-core:${versions.mime4j}" api "org.apache.james:apache-mime4j-dom:${versions.mime4j}" diff --git a/plugins/ingest-attachment/licenses/commons-compress-1.21.jar.sha1 b/plugins/ingest-attachment/licenses/commons-compress-1.21.jar.sha1 deleted file mode 100644 index 81ac609a1aa26..0000000000000 --- a/plugins/ingest-attachment/licenses/commons-compress-1.21.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4ec95b60d4e86b5c95a0e919cb172a0af98011ef \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/commons-compress-1.22.jar.sha1 b/plugins/ingest-attachment/licenses/commons-compress-1.22.jar.sha1 new file mode 100644 index 0000000000000..9ab7216c8050a --- /dev/null +++ b/plugins/ingest-attachment/licenses/commons-compress-1.22.jar.sha1 @@ -0,0 +1 @@ +691a8b4e6cf4248c3bc72c8b719337d5cb7359fa \ No newline at end of file From f4571dc687bdec6fdc53cd720056b9e13e63dc8d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Nov 2022 10:16:22 -0800 Subject: [PATCH 15/23] Bump opencensus-contrib-http-util from 0.18.0 to 0.31.1 in /plugins/repository-gcs (#3633) * Bump opencensus-contrib-http-util in /plugins/repository-gcs Bumps [opencensus-contrib-http-util](https://github.com/census-instrumentation/opencensus-java) from 0.18.0 to 0.31.1. - [Release notes](https://github.com/census-instrumentation/opencensus-java/releases) - [Changelog](https://github.com/census-instrumentation/opencensus-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/census-instrumentation/opencensus-java/compare/v0.18.0...v0.31.1) --- updated-dependencies: - dependency-name: io.opencensus:opencensus-contrib-http-util dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * Adding missing classes Signed-off-by: Vacha Shah * changelog change Signed-off-by: Poojita Raj Signed-off-by: dependabot[bot] Signed-off-by: Vacha Shah Signed-off-by: Poojita Raj Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: Vacha Shah Co-authored-by: Poojita Raj --- CHANGELOG.md | 1 + plugins/repository-gcs/build.gradle | 4 +++- .../licenses/opencensus-contrib-http-util-0.18.0.jar.sha1 | 1 - .../licenses/opencensus-contrib-http-util-0.31.1.jar.sha1 | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.18.0.jar.sha1 create mode 100644 plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.31.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 35a2c18f35792..9b97a0754fbbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Update to Gradle 7.6 and JDK-19 ([#4973](https://github.com/opensearch-project/OpenSearch/pull/4973)) - Bumps `azure-storage-blob` from 12.16.1 to 12.20.0 ([#4995](https://github.com/opensearch-project/OpenSearch/pull/4995)) - Bumps `commons-compress` from 1.21 to 1.22 ([#5104](https://github.com/opensearch-project/OpenSearch/pull/5104)) +- Bump `opencensus-contrib-http-util` from 0.18.0 to 0.31.1 ([#3633](https://github.com/opensearch-project/OpenSearch/pull/3633)) ### Changed - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948)) diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index 05e879547a4b0..9528537a3dd5e 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -81,7 +81,7 @@ dependencies { api 'com.google.api:gax-httpjson:0.103.1' api 'io.grpc:grpc-context:1.46.0' api 'io.opencensus:opencensus-api:0.18.0' - api 'io.opencensus:opencensus-contrib-http-util:0.18.0' + api 'io.opencensus:opencensus-contrib-http-util:0.31.1' api 'com.google.apis:google-api-services-storage:v1-rev20220608-1.32.1' testImplementation project(':test:fixtures:gcs-fixture') @@ -209,6 +209,8 @@ thirdPartyAudit { 'javax.jms.Message', 'javax.servlet.ServletContextEvent', 'javax.servlet.ServletContextListener', + 'io.opencensus.tags.TagMetadata', + 'io.opencensus.tags.TagMetadata$TagTtl', ) } diff --git a/plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.18.0.jar.sha1 b/plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.18.0.jar.sha1 deleted file mode 100644 index 1757e00591110..0000000000000 --- a/plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.18.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -76a37e4a931d5801a9e25b0c0353e5f37c4d1e8e \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.31.1.jar.sha1 b/plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.31.1.jar.sha1 new file mode 100644 index 0000000000000..4e123da3ab45f --- /dev/null +++ b/plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.31.1.jar.sha1 @@ -0,0 +1 @@ +3c13fc5715231fadb16a9b74a44d9d59c460cfa8 \ No newline at end of file From 3423f44440ddcb8525887f9eeb76befebacc01f8 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Wed, 9 Nov 2022 10:39:25 -0800 Subject: [PATCH 16/23] Fix boundary condition in indexing pressure test (#5168) This updates the boundary condition in an assertion in two tests in ShardIndexingPressureConcurrentExecutionTests. I could reliably reproduce errors here by running: ``` ./gradlew ':server:test' -Dtests.iters=10000 --tests "org.opensearch.index.ShardIndexingPressureConcurrentExecutionTests.testReplicaThreadedUpdateToShardLimits" ``` On every error the value that failed was exactly 0.95 and failed the less than check. The change here is to accept 0.95, and also refactor the test to give a better error message on failure. Signed-off-by: Andrew Ross Signed-off-by: Andrew Ross --- ...exingPressureConcurrentExecutionTests.java | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/server/src/test/java/org/opensearch/index/ShardIndexingPressureConcurrentExecutionTests.java b/server/src/test/java/org/opensearch/index/ShardIndexingPressureConcurrentExecutionTests.java index 72ca8bff4087d..faab2f405010a 100644 --- a/server/src/test/java/org/opensearch/index/ShardIndexingPressureConcurrentExecutionTests.java +++ b/server/src/test/java/org/opensearch/index/ShardIndexingPressureConcurrentExecutionTests.java @@ -8,6 +8,8 @@ package org.opensearch.index; +import org.hamcrest.Matcher; +import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; import org.opensearch.action.admin.indices.stats.CommonStatsFlags; import org.opensearch.cluster.service.ClusterService; @@ -23,6 +25,10 @@ import java.util.concurrent.atomic.AtomicInteger; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + public class ShardIndexingPressureConcurrentExecutionTests extends OpenSearchTestCase { private final Settings settings = Settings.builder() @@ -34,8 +40,8 @@ public class ShardIndexingPressureConcurrentExecutionTests extends OpenSearchTes .put(ShardIndexingPressureSettings.REQUEST_SIZE_WINDOW.getKey(), 100) .build(); - final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - final ClusterService clusterService = new ClusterService(settings, clusterSettings, null); + private final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + private final ClusterService clusterService = new ClusterService(settings, clusterSettings, null); public enum OperationType { COORDINATING, @@ -71,15 +77,11 @@ public void testCoordinatingPrimaryThreadedUpdateToShardLimits() throws Exceptio NUM_THREADS * 15, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId1).getCurrentCombinedCoordinatingAndPrimaryBytes() ); - assertTrue( + MatcherAssert.assertThat( (double) (NUM_THREADS * 15) / shardIndexingPressure.shardStats() .getIndexingPressureShardStats(shardId1) - .getCurrentPrimaryAndCoordinatingLimits() < 0.95 - ); - assertTrue( - (double) (NUM_THREADS * 15) / shardIndexingPressure.shardStats() - .getIndexingPressureShardStats(shardId1) - .getCurrentPrimaryAndCoordinatingLimits() > 0.75 + .getCurrentPrimaryAndCoordinatingLimits(), + isInOperatingFactorRange() ); for (int i = 0; i < NUM_THREADS; i++) { @@ -112,15 +114,11 @@ public void testReplicaThreadedUpdateToShardLimits() throws Exception { Releasable[] releasable = fireConcurrentRequests(NUM_THREADS, shardIndexingPressure, shardId1, 15, OperationType.REPLICA); assertEquals(NUM_THREADS * 15, shardIndexingPressure.shardStats().getIndexingPressureShardStats(shardId1).getCurrentReplicaBytes()); - assertTrue( - (double) (NUM_THREADS * 15) / shardIndexingPressure.shardStats() - .getIndexingPressureShardStats(shardId1) - .getCurrentReplicaLimits() < 0.95 - ); - assertTrue( + MatcherAssert.assertThat( (double) (NUM_THREADS * 15) / shardIndexingPressure.shardStats() .getIndexingPressureShardStats(shardId1) - .getCurrentReplicaLimits() > 0.75 + .getCurrentReplicaLimits(), + isInOperatingFactorRange() ); for (int i = 0; i < NUM_THREADS; i++) { @@ -1087,4 +1085,11 @@ private void fireConcurrentAndParallelRequestsForUniformThroughPut( t.join(); } } + + private Matcher isInOperatingFactorRange() { + return allOf( + greaterThan(ShardIndexingPressureMemoryManager.LOWER_OPERATING_FACTOR.get(settings)), + lessThanOrEqualTo(ShardIndexingPressureMemoryManager.UPPER_OPERATING_FACTOR.get(settings)) + ); + } } From 650039c74ade58295c8b673d5f1f4c4641264a72 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 9 Nov 2022 19:29:15 -0500 Subject: [PATCH 17/23] [BUG] Gradle Check Failed on Windows due to JDK19 pulling by gradle (#5188) Signed-off-by: Andriy Redko Signed-off-by: Andriy Redko --- buildSrc/src/main/java/org/opensearch/gradle/Jdk.java | 3 ++- gradle/runtime-jdk-provision.gradle | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java b/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java index 1073ba01dafab..08b7054d7d53a 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java @@ -37,6 +37,7 @@ import org.gradle.api.model.ObjectFactory; import org.gradle.api.provider.Property; import org.gradle.api.tasks.TaskDependency; +import org.gradle.internal.os.OperatingSystem; import java.io.File; import java.util.Arrays; @@ -169,7 +170,7 @@ public Object getBinJavaPath() { return new Object() { @Override public String toString() { - return getHomeRoot() + "/bin/java"; + return OperatingSystem.current().getExecutableName(getHomeRoot() + "/bin/java"); } }; } diff --git a/gradle/runtime-jdk-provision.gradle b/gradle/runtime-jdk-provision.gradle index 2f0c2f74d6803..cb4bb834f89c9 100644 --- a/gradle/runtime-jdk-provision.gradle +++ b/gradle/runtime-jdk-provision.gradle @@ -20,7 +20,11 @@ if (BuildParams.getIsRuntimeJavaHomeSet()) { configure(allprojects - project(':build-tools')) { project.tasks.withType(Test).configureEach { Test test -> if (BuildParams.getIsRuntimeJavaHomeSet()) { - test.executable = "${BuildParams.runtimeJavaHome}/bin/java" + if (OS.current() == OS.WINDOWS) { + test.executable = "${BuildParams.runtimeJavaHome}/bin/java.exe" + } else { + test.executable = "${BuildParams.runtimeJavaHome}/bin/java" + } } } } From e10c9253d621af08698d5b0966de4599ec342318 Mon Sep 17 00:00:00 2001 From: Rabi Panda Date: Wed, 9 Nov 2022 23:49:40 -0800 Subject: [PATCH 18/23] Add jvm option to allow security manager (#5194) This change explicitly sets JVM options to allow security manager. Signed-off-by: Rabi Panda Signed-off-by: Rabi Panda --- CHANGELOG.md | 2 +- .../java/org/opensearch/gradle/OpenSearchTestBasePlugin.java | 2 +- .../java/org/opensearch/tools/launchers/SystemJvmOptions.java | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b97a0754fbbd..13c12cf1f7568 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,7 +67,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827)) - Fixed compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944)) - +- Add jvm option to allow security manager ([#5194](https://github.com/opensearch-project/OpenSearch/pull/5194)) ### Security ## [Unreleased 2.x] diff --git a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java index 9d6e78014916d..4638cee1d96a6 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java @@ -109,7 +109,7 @@ public void execute(Task t) { test.systemProperty("java.locale.providers", "SPI,JRE"); } else { test.systemProperty("java.locale.providers", "SPI,COMPAT"); - test.jvmArgs("--illegal-access=warn"); + test.jvmArgs("--illegal-access=warn", "-Djava.security.manager=allow"); } } }); diff --git a/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java b/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java index fc613ccdaae68..30969a0df2a2c 100644 --- a/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java +++ b/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java @@ -77,7 +77,8 @@ static List systemJvmOptions() { // log4j 2 "-Dlog4j.shutdownHookEnabled=false", "-Dlog4j2.disable.jmx=true", - + // security manager + "-Djava.security.manager=allow", javaLocaleProviders() ) ).stream().filter(e -> e.isEmpty() == false).collect(Collectors.toList()); From de2b84d7566ce65428ce4fad9821cd71f0339c96 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Nov 2022 11:28:03 -0600 Subject: [PATCH 19/23] Bump geoip2 from 3.0.1 to 3.0.2 in /modules/ingest-geoip (#5103) * Bump geoip2 from 3.0.1 to 3.0.2 in /modules/ingest-geoip Bumps [geoip2](https://github.com/maxmind/GeoIP2-java) from 3.0.1 to 3.0.2. - [Release notes](https://github.com/maxmind/GeoIP2-java/releases) - [Changelog](https://github.com/maxmind/GeoIP2-java/blob/main/CHANGELOG.md) - [Commits](https://github.com/maxmind/GeoIP2-java/compare/v3.0.1...v3.0.2) --- updated-dependencies: - dependency-name: com.maxmind.geoip2:geoip2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Updating SHAs Signed-off-by: dependabot[bot] * modify changelog Signed-off-by: Poojita Raj Signed-off-by: dependabot[bot] Signed-off-by: Poojita Raj Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] Co-authored-by: Poojita Raj --- CHANGELOG.md | 9 +++++---- modules/ingest-geoip/build.gradle | 2 +- modules/ingest-geoip/licenses/geoip2-3.0.1.jar.sha1 | 1 - modules/ingest-geoip/licenses/geoip2-3.0.2.jar.sha1 | 1 + 4 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 modules/ingest-geoip/licenses/geoip2-3.0.1.jar.sha1 create mode 100644 modules/ingest-geoip/licenses/geoip2-3.0.2.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 13c12cf1f7568..d2f99925e2929 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,11 +35,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bumps `reactor-core` from 3.4.18 to 3.4.23 ([#4548](https://github.com/opensearch-project/OpenSearch/pull/4548)) - Bumps `jempbox` from 1.8.16 to 1.8.17 ([#4550](https://github.com/opensearch-project/OpenSearch/pull/4550)) - Update Apache Lucene to 9.5.0-snapshot-a4ef70f ([#4979](https://github.com/opensearch-project/OpenSearch/pull/4979)) -- Bumps `bcpg-fips` from 1.0.5.1 to 1.0.7.1 - Update to Gradle 7.6 and JDK-19 ([#4973](https://github.com/opensearch-project/OpenSearch/pull/4973)) -- Bumps `azure-storage-blob` from 12.16.1 to 12.20.0 ([#4995](https://github.com/opensearch-project/OpenSearch/pull/4995)) -- Bumps `commons-compress` from 1.21 to 1.22 ([#5104](https://github.com/opensearch-project/OpenSearch/pull/5104)) -- Bump `opencensus-contrib-http-util` from 0.18.0 to 0.31.1 ([#3633](https://github.com/opensearch-project/OpenSearch/pull/3633)) ### Changed - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948)) @@ -73,6 +69,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x] ### Added ### Dependencies +- Bumps `bcpg-fips` from 1.0.5.1 to 1.0.7.1 +- Bumps `azure-storage-blob` from 12.16.1 to 12.20.0 ([#4995](https://github.com/opensearch-project/OpenSearch/pull/4995)) +- Bumps `commons-compress` from 1.21 to 1.22 ([#5104](https://github.com/opensearch-project/OpenSearch/pull/5104)) +- Bump `opencensus-contrib-http-util` from 0.18.0 to 0.31.1 ([#3633](https://github.com/opensearch-project/OpenSearch/pull/3633)) +- Bump `geoip2` from 3.0.1 to 3.0.2 ([#5103](https://github.com/opensearch-project/OpenSearch/pull/5103)) ### Changed ### Deprecated ### Removed diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index 7dce788f3a4a4..8c6f279c445b3 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -39,7 +39,7 @@ opensearchplugin { } dependencies { - api('com.maxmind.geoip2:geoip2:3.0.1') + api('com.maxmind.geoip2:geoip2:3.0.2') // geoip2 dependencies: api("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}") api("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}") diff --git a/modules/ingest-geoip/licenses/geoip2-3.0.1.jar.sha1 b/modules/ingest-geoip/licenses/geoip2-3.0.1.jar.sha1 deleted file mode 100644 index f1d5ac5aea546..0000000000000 --- a/modules/ingest-geoip/licenses/geoip2-3.0.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8a814ae92a1d8c35f82d0ff76d86927c191b7916 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/geoip2-3.0.2.jar.sha1 b/modules/ingest-geoip/licenses/geoip2-3.0.2.jar.sha1 new file mode 100644 index 0000000000000..2ff70cf499713 --- /dev/null +++ b/modules/ingest-geoip/licenses/geoip2-3.0.2.jar.sha1 @@ -0,0 +1 @@ +f0ab0a451309c93f0fb6bf3cb203ba19d452c800 \ No newline at end of file From f0925f7477ef39404969235468c13e6dd85f1441 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Thu, 10 Nov 2022 09:51:04 -0800 Subject: [PATCH 20/23] Do not skip CHANGELOG.md in backport (#5202) The updated changelog process is to include changes intended to be backported into the `[Unreleased 2.x]` section of the changelog on main. It should be backported to the 2.x branch so the CHANGELOG.md file should not be skipped. Signed-off-by: Andrew Ross Signed-off-by: Andrew Ross --- .github/workflows/backport.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index b0ea0f782cb62..5a75d2c877992 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -27,4 +27,3 @@ jobs: with: github_token: ${{ steps.github_app_token.outputs.token }} head_template: backport/backport-<%= number %>-to-<%= base %> - files_to_skip: 'CHANGELOG.md' From 4fc5c8d3fe17a87d2dc6e14a4d1c153b2e06e3e0 Mon Sep 17 00:00:00 2001 From: Rabi Panda Date: Thu, 10 Nov 2022 13:48:41 -0800 Subject: [PATCH 21/23] Check for JDK version while setting java.security.manager option (#5205) Signed-off-by: Rabi Panda --- .../opensearch/gradle/OpenSearchTestBasePlugin.java | 5 ++++- .../opensearch/tools/launchers/SystemJvmOptions.java | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java index 4638cee1d96a6..67df8bbafcb03 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java @@ -109,7 +109,10 @@ public void execute(Task t) { test.systemProperty("java.locale.providers", "SPI,JRE"); } else { test.systemProperty("java.locale.providers", "SPI,COMPAT"); - test.jvmArgs("--illegal-access=warn", "-Djava.security.manager=allow"); + test.jvmArgs("--illegal-access=warn"); + } + if (test.getJavaVersion().compareTo(JavaVersion.VERSION_17) > 0) { + test.jvmArgs("-Djava.security.manager=allow"); } } }); diff --git a/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java b/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java index 30969a0df2a2c..aa3dfbe39ee96 100644 --- a/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java +++ b/distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/SystemJvmOptions.java @@ -78,12 +78,20 @@ static List systemJvmOptions() { "-Dlog4j.shutdownHookEnabled=false", "-Dlog4j2.disable.jmx=true", // security manager - "-Djava.security.manager=allow", + allowSecurityManagerOption(), javaLocaleProviders() ) ).stream().filter(e -> e.isEmpty() == false).collect(Collectors.toList()); } + private static String allowSecurityManagerOption() { + if (Runtime.version().feature() > 17) { + return "-Djava.security.manager=allow"; + } else { + return ""; + } + } + private static String maybeShowCodeDetailsInExceptionMessages() { if (Runtime.version().feature() >= 14) { return "-XX:+ShowCodeDetailsInExceptionMessages"; From 3b46a99f88b00771eeabff048ef37a16c74b1fd0 Mon Sep 17 00:00:00 2001 From: Rabi Panda Date: Fri, 11 Nov 2022 11:51:17 -0800 Subject: [PATCH 22/23] Clean up common logging code (#5212) Remove Elasticsearch from HTTP header warning message. Remove unused variables, redundant method calls and simplify conditions. Signed-off-by: Rabi Panda Signed-off-by: Rabi Panda --- .../java/org/opensearch/client/Response.java | 2 +- .../common/logging/HeaderWarning.java | 3 +-- .../common/logging/LogConfigurator.java | 11 ++------ .../common/logging/LoggerMessageFormat.java | 26 +++++++------------ 4 files changed, 13 insertions(+), 29 deletions(-) diff --git a/client/rest/src/main/java/org/opensearch/client/Response.java b/client/rest/src/main/java/org/opensearch/client/Response.java index c758826b776ba..b062d937ed630 100644 --- a/client/rest/src/main/java/org/opensearch/client/Response.java +++ b/client/rest/src/main/java/org/opensearch/client/Response.java @@ -147,7 +147,7 @@ public HttpEntity getEntity() { * @return {@code true} if the input string matches the specification */ private static boolean matchWarningHeaderPatternByPrefix(final String s) { - return s.startsWith("299 OpenSearch-") || s.startsWith("299 Elasticsearch-"); + return s.startsWith("299 OpenSearch-"); } /** diff --git a/server/src/main/java/org/opensearch/common/logging/HeaderWarning.java b/server/src/main/java/org/opensearch/common/logging/HeaderWarning.java index df5a94ccd34e4..44eefe1e78580 100644 --- a/server/src/main/java/org/opensearch/common/logging/HeaderWarning.java +++ b/server/src/main/java/org/opensearch/common/logging/HeaderWarning.java @@ -60,8 +60,7 @@ public class HeaderWarning { * Regular expression to test if a string matches the RFC7234 specification for warning headers. This pattern assumes that the warn code * is always 299. Further, this pattern assumes that the warn agent represents a version of OpenSearch including the build hash. */ - public static final Pattern WARNING_HEADER_PATTERN = Pattern.compile("299 " + // warn code - "(?:Elasticsearch-|OpenSearch-)" + // warn agent (note: Elasticsearch needed for bwc mixedCluster testing) + public static final Pattern WARNING_HEADER_PATTERN = Pattern.compile("299 OpenSearch-" + // warn code "\\d+\\.\\d+\\.\\d+(?:-(?:alpha|beta|rc)\\d+)?(?:-SNAPSHOT)?-" + // warn agent "(?:[a-f0-9]{7}(?:[a-f0-9]{33})?|unknown) " + // warn agent "\"((?:\t| |!|[\\x23-\\x5B]|[\\x5D-\\x7E]|[\\x80-\\xFF]|\\\\|\\\\\")*)\"( " + // quoted warning value, captured diff --git a/server/src/main/java/org/opensearch/common/logging/LogConfigurator.java b/server/src/main/java/org/opensearch/common/logging/LogConfigurator.java index c0405f9e52b77..4438bf53fd62c 100644 --- a/server/src/main/java/org/opensearch/common/logging/LogConfigurator.java +++ b/server/src/main/java/org/opensearch/common/logging/LogConfigurator.java @@ -66,9 +66,7 @@ import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; -import java.util.Collections; import java.util.EnumSet; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; @@ -182,7 +180,6 @@ private static void configure(final Settings settings, final Path configsPath, f final LoggerContext context = (LoggerContext) LogManager.getContext(false); - final Set locationsWithDeprecatedPatterns = Collections.synchronizedSet(new HashSet<>()); final List configurations = new ArrayList<>(); final PropertiesConfigurationFactory factory = new PropertiesConfigurationFactory(); final Set options = EnumSet.of(FileVisitOption.FOLLOW_LINKS); @@ -206,12 +203,8 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr // Redirect stdout/stderr to log4j. While we ensure Elasticsearch code does not write to those streams, // third party libraries may do that - System.setOut( - new PrintStream(new LoggingOutputStream(LogManager.getLogger("stdout"), Level.INFO), false, StandardCharsets.UTF_8.name()) - ); - System.setErr( - new PrintStream(new LoggingOutputStream(LogManager.getLogger("stderr"), Level.WARN), false, StandardCharsets.UTF_8.name()) - ); + System.setOut(new PrintStream(new LoggingOutputStream(LogManager.getLogger("stdout"), Level.INFO), false, StandardCharsets.UTF_8)); + System.setErr(new PrintStream(new LoggingOutputStream(LogManager.getLogger("stderr"), Level.WARN), false, StandardCharsets.UTF_8)); } private static void configureStatusLogger() { diff --git a/server/src/main/java/org/opensearch/common/logging/LoggerMessageFormat.java b/server/src/main/java/org/opensearch/common/logging/LoggerMessageFormat.java index ad9981809ae3a..a0dec5b8d0b70 100644 --- a/server/src/main/java/org/opensearch/common/logging/LoggerMessageFormat.java +++ b/server/src/main/java/org/opensearch/common/logging/LoggerMessageFormat.java @@ -79,34 +79,34 @@ public static String format(final String prefix, final String messagePattern, fi return messagePattern; } else { // add the tail string which contains no variables and return // the result. - sbuf.append(messagePattern.substring(i, messagePattern.length())); + sbuf.append(messagePattern.substring(i)); return sbuf.toString(); } } else { if (isEscapedDelimiter(messagePattern, j)) { if (!isDoubleEscaped(messagePattern, j)) { L--; // DELIM_START was escaped, thus should not be incremented - sbuf.append(messagePattern.substring(i, j - 1)); + sbuf.append(messagePattern, i, j - 1); sbuf.append(DELIM_START); i = j + 1; } else { // The escape character preceding the delimiter start is // itself escaped: "abc x:\\{}" // we have to consume one backward slash - sbuf.append(messagePattern.substring(i, j - 1)); - deeplyAppendParameter(sbuf, argArray[L], new HashSet()); + sbuf.append(messagePattern, i, j - 1); + deeplyAppendParameter(sbuf, argArray[L], new HashSet<>()); i = j + 2; } } else { // normal case - sbuf.append(messagePattern.substring(i, j)); - deeplyAppendParameter(sbuf, argArray[L], new HashSet()); + sbuf.append(messagePattern, i, j); + deeplyAppendParameter(sbuf, argArray[L], new HashSet<>()); i = j + 2; } } } // append the characters following the last {} pair. - sbuf.append(messagePattern.substring(i, messagePattern.length())); + sbuf.append(messagePattern.substring(i)); return sbuf.toString(); } @@ -116,19 +116,11 @@ static boolean isEscapedDelimiter(String messagePattern, int delimiterStartIndex return false; } char potentialEscape = messagePattern.charAt(delimiterStartIndex - 1); - if (potentialEscape == ESCAPE_CHAR) { - return true; - } else { - return false; - } + return potentialEscape == ESCAPE_CHAR; } static boolean isDoubleEscaped(String messagePattern, int delimiterStartIndex) { - if (delimiterStartIndex >= 2 && messagePattern.charAt(delimiterStartIndex - 2) == ESCAPE_CHAR) { - return true; - } else { - return false; - } + return delimiterStartIndex >= 2 && messagePattern.charAt(delimiterStartIndex - 2) == ESCAPE_CHAR; } private static void deeplyAppendParameter(StringBuilder sbuf, Object o, Set seen) { From 831b3a05015cfbe8fd2d07aa8b64d23f18b8ecfa Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 11 Nov 2022 15:50:44 -0500 Subject: [PATCH 23/23] Update Gradle to 7.6-rc3 (#5216) Signed-off-by: Andriy Redko Signed-off-by: Andriy Redko --- gradle/wrapper/gradle-wrapper.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 57dec44f2a013..ea7765b9bc58d 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -11,7 +11,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-rc-2-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-rc-3-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=be5387477162265eac882b9c83d756d8d2db170380e36fba2fdbee83d87de0d7 +distributionSha256Sum=abc6de2653ec9befb00cc0d064ce1ca4e4dab2c91955e830661505189c0b2f08