Merge branch 'main' into main-opentelemetry

Signed-off-by: suraj kumar <surajkumar.tu@gmail.com>

.github/workflows/stalled.yml

@@ -0,0 +1,28 @@
+name: Close Stalled PRs
+on:
+  schedule:
+    - cron: '15 15 * * *' # Run every day at 15:15 UTC / 7:15 PST / 8:15 PDT
+permissions:
+  pull-requests: write
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/github-app-token@v1.5.0
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+      - name: Stale PRs
+        uses: actions/stale@v8
+        with:
+          repo-token: ${{ steps.github_app_token.outputs.token }}
+          stale-pr-label: 'stalled'
+          stale-pr-message: 'This PR is stalled because it has been open for 30 days with no activity. Remove stalled label or comment or this will be closed in 7 days.'
+          close-pr-message: 'This PR was closed because it has been stalled for 7 days with no activity.'
+          days-before-pr-stale: 30
+          days-before-pr-close: 7
+          days-before-issue-stale: -1
+          days-before-issue-close: -1

CHANGELOG.md

@@ -44,6 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773))
 - Change http code for DecommissioningFailedException from 500 to 400 ([#5283](https://github.com/opensearch-project/OpenSearch/pull/5283))
 - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792))
+- Pass localNode info to all plugins on node start ([#7919](https://github.com/opensearch-project/OpenSearch/pull/7919))
 
 ### Deprecated
 
@@ -69,7 +70,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Support OpenSSL Provider with default Netty allocator ([#5460](https://github.com/opensearch-project/OpenSearch/pull/5460))
 - Replaces ZipInputStream with ZipFile to fix Zip Slip vulnerability ([#7230](https://github.com/opensearch-project/OpenSearch/pull/7230))
 - Add missing validation/parsing of SearchBackpressureMode of SearchBackpressureSettings ([#7541](https://github.com/opensearch-project/OpenSearch/pull/7541))
-- Fix mapping char_filter when mapping a hashtag ([#7591](https://github.com/opensearch-project/OpenSearch/pull/7591))
 
 ### Security
 
@@ -83,6 +83,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add descending order search optimization through reverse segment read. ([#7967](https://github.com/opensearch-project/OpenSearch/pull/7967))
 - Update components of segrep backpressure to support remote store. ([#8020](https://github.com/opensearch-project/OpenSearch/pull/8020))
 - Make remote cluster connection setup in async ([#8038](https://github.com/opensearch-project/OpenSearch/pull/8038))
+- Add API to initialize extensions ([#8029]()https://github.com/opensearch-project/OpenSearch/pull/8029)
 - Add distributed tracing framework ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543))
 
 ### Dependencies
@@ -107,7 +108,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `io.projectreactor:reactor-core` from 3.4.18 to 3.5.6 in /plugins/repository-azure ([#8016](https://github.com/opensearch-project/OpenSearch/pull/8016))
 - Bump `spock-core` from 2.1-groovy-3.0 to 2.3-groovy-3.0 ([#8122](https://github.com/opensearch-project/OpenSearch/pull/8122))
 - Bump `com.networknt:json-schema-validator` from 1.0.83 to 1.0.84 (#8141)
+- Bump `com.netflix.nebula:gradle-info-plugin` from 12.1.3 to 12.1.4 (#8139)
 - Bump `commons-io:commons-io` from 2.12.0 to 2.13.0 in /plugins/discovery-azure-classic ([#8140](https://github.com/opensearch-project/OpenSearch/pull/8140))
+- Bump `mockito` from 5.2.0 to 5.4.0 ([#8181](https://github.com/opensearch-project/OpenSearch/pull/8181))
+- Bump `netty` from 4.1.93.Final to 4.1.94.Final ([#8191](https://github.com/opensearch-project/OpenSearch/pull/8191))
 
 ### Changed
 - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836))
@@ -126,12 +130,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Deprecated
 
 ### Removed
-- Remove COMPRESSOR variable from CompressorFactory and use DEFLATE_COMPRESSOR instead ([7907](https://github.com/opensearch-project/OpenSearch/pull/7907))
+- Remove `COMPRESSOR` variable from `CompressorFactory` and use `DEFLATE_COMPRESSOR` instead ([7907](https://github.com/opensearch-project/OpenSearch/pull/7907))
 
 ### Fixed
--  Fixing error: adding a new/forgotten parameter to the configuration for checking the config on startup in plugins/repository-s3 #7924
+- Fixing error: adding a new/forgotten parameter to the configuration for checking the config on startup in plugins/repository-s3 #7924
 - Enforce 512 byte document ID limit in bulk updates ([#8039](https://github.com/opensearch-project/OpenSearch/pull/8039))
 - With only GlobalAggregation in request causes unnecessary wrapping with MultiCollector ([#8125](https://github.com/opensearch-project/OpenSearch/pull/8125))
+- Fix mapping char_filter when mapping a hashtag ([#7591](https://github.com/opensearch-project/OpenSearch/pull/7591))
 
 ### Security
 

build.gradle

@@ -46,6 +46,7 @@ import org.gradle.plugins.ide.eclipse.model.EclipseJdt
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 import org.gradle.api.Project;
 import org.gradle.process.ExecResult;
+import org.opensearch.gradle.CheckCompatibilityTask
 
 import static org.opensearch.gradle.util.GradleUtils.maybeConfigure
 
@@ -643,3 +644,15 @@ tasks.withType(TestTaskReports).configureEach {
 tasks.named(JavaBasePlugin.CHECK_TASK_NAME) {
   dependsOn tasks.named('testAggregateTestReport', TestReport)
 }
+
+tasks.register('checkCompatibility', CheckCompatibilityTask) {
+  description('Checks the compatibility with child components')
+}
+
+allprojects { project ->
+  project.afterEvaluate {
+    if (project.tasks.findByName('publishToMavenLocal')) {
+      checkCompatibility.dependsOn(project.tasks.publishToMavenLocal)
+    }
+  }
+}

buildSrc/build.gradle

@@ -122,6 +122,7 @@ dependencies {
   api 'org.jruby.jcodings:jcodings:1.0.58'
   api 'org.jruby.joni:joni:2.1.48'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
+  api "org.ajoberstar.grgit:grgit-core:5.2.0"
 
   testFixturesApi "junit:junit:${props.getProperty('junit')}"
   testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"

buildSrc/src/main/groovy/org/opensearch/gradle/CheckCompatibilityTask.groovy

@@ -0,0 +1,108 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gradle
+
+import groovy.json.JsonSlurper
+import org.ajoberstar.grgit.Grgit
+import org.ajoberstar.grgit.operation.BranchListOp
+import org.gradle.api.DefaultTask
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.Internal
+import org.gradle.api.tasks.TaskAction
+import org.gradle.internal.os.OperatingSystem
+
+import java.nio.file.Paths
+
+class CheckCompatibilityTask extends DefaultTask {
+
+    static final String REPO_URL = 'https://raw.githubusercontent.com/opensearch-project/opensearch-plugins/main/plugins/.meta'
+
+    @Input
+    List repositoryUrls = project.hasProperty('repositoryUrls') ? project.property('repositoryUrls').split(',') : getRepoUrls()
+
+    @Input
+    String ref = project.hasProperty('ref') ? project.property('ref') : 'main'
+
+    @Internal
+    List failedComponents = []
+
+    @Internal
+    List gitFailedComponents = []
+
+    @Internal
+    List compatibleComponents = []
+
+    @TaskAction
+    void checkCompatibility() {
+        logger.info("Checking compatibility for: $repositoryUrls for $ref")
+        repositoryUrls.parallelStream().forEach { repositoryUrl ->
+            def tempDir = File.createTempDir()
+            try {
+                if (cloneAndCheckout(repositoryUrl, tempDir)) {
+                    if (repositoryUrl.toString().endsWithAny('notifications', 'notifications.git')) {
+                        tempDir = Paths.get(tempDir.getAbsolutePath(), 'notifications')
+                    }
+                    project.exec {
+                        workingDir = tempDir
+                        def stdout = new ByteArrayOutputStream()
+                        executable = (OperatingSystem.current().isWindows()) ? 'gradlew.bat' : './gradlew'
+                        args 'assemble'
+                        standardOutput stdout
+                    }
+                    compatibleComponents.add(repositoryUrl)
+                } else {
+                    logger.lifecycle("Skipping compatibility check for $repositoryUrl")
+                }
+            } catch (ex) {
+                failedComponents.add(repositoryUrl)
+                logger.info("Gradle assemble failed for $repositoryUrl", ex)
+            } finally {
+                tempDir.deleteDir()
+            }
+        }
+        if (!failedComponents.isEmpty()) {
+            logger.lifecycle("Incompatible components: $failedComponents")
+            logger.info("Compatible components: $compatibleComponents")
+        }
+        if (!gitFailedComponents.isEmpty()) {
+            logger.lifecycle("Components skipped due to git failures: $gitFailedComponents")
+            logger.info("Compatible components: $compatibleComponents")
+        }
+        if (!compatibleComponents.isEmpty()) {
+            logger.lifecycle("Compatible components: $compatibleComponents")
+        }
+    }
+
+    protected static List getRepoUrls() {
+        def json = new JsonSlurper().parse(REPO_URL.toURL())
+        def labels = json.projects.values()
+        return labels as List
+    }
+
+    protected boolean cloneAndCheckout(repoUrl, directory) {
+        try {
+            def grgit = Grgit.clone(dir: directory, uri: repoUrl)
+            def remoteBranches = grgit.branch.list(mode: BranchListOp.Mode.REMOTE)
+            String targetBranch = 'origin/' + ref
+            if (remoteBranches.find { it.name == targetBranch } == null) {
+                gitFailedComponents.add(repoUrl)
+                logger.info("$ref does not exist for $repoUrl. Skipping the compatibility check!!")
+                return false
+            } else {
+                logger.info("Checking out $targetBranch")
+                grgit.checkout(branch: targetBranch)
+                return true
+            }
+        } catch (ex) {
+            logger.error('Exception occurred during GitHub operations', ex)
+            gitFailedComponents.add(repoUrl)
+            return false
+        }
+    }
+}

buildSrc/version.properties

@@ -29,7 +29,7 @@ jakarta_annotation = 1.3.5
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.5.0
 
-netty             = 4.1.93.Final
+netty             = 4.1.94.Final
 joda              = 2.12.2
 
 # client dependencies
@@ -53,7 +53,7 @@ bouncycastle=1.70
 randomizedrunner  = 2.7.1
 junit             = 4.13.2
 hamcrest          = 2.1
-mockito           = 5.2.0
+mockito           = 5.4.0
 objenesis         = 3.2
 bytebuddy         = 1.14.3
 

modules/transport-netty4/build.gradle

@@ -127,7 +127,6 @@ thirdPartyAudit {
     'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status',
     'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper',
     'com.aayushatharva.brotli4j.encoder.BrotliEncoderChannel',
-    'com.aayushatharva.brotli4j.encoder.Encoder',
     'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
     'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
     // classes are missing

modules/transport-netty4/licenses/netty-buffer-4.1.94.Final.jar.sha1

@@ -0,0 +1 @@
+eec248b26f16e888688e5bb37b7eeda76b78d2f7

modules/transport-netty4/licenses/netty-codec-4.1.94.Final.jar.sha1

@@ -0,0 +1 @@
+c70ef20ca338558147887df60f46341bc47f6900

modules/transport-netty4/licenses/netty-codec-http-4.1.94.Final.jar.sha1

@@ -0,0 +1 @@
+9e5404764092c1f6305ad5719078f46ab228d587

modules/transport-netty4/licenses/netty-codec-http2-4.1.94.Final.jar.sha1

@@ -0,0 +1 @@
+f651595784d6cca4cbca6a8ad74c48fceed6cea8

modules/transport-netty4/licenses/netty-common-4.1.94.Final.jar.sha1

@@ -0,0 +1 @@
+ad4ecf779ebc794cd351f57792f56ea01387b868

modules/transport-netty4/licenses/netty-handler-4.1.94.Final.jar.sha1

@@ -0,0 +1 @@
+cd9121ce24d6d3f2898946d04b0ef3ec548b00b4

modules/transport-netty4/licenses/netty-resolver-4.1.94.Final.jar.sha1

@@ -0,0 +1 @@
+e96f649e8e9dcb29a1f8e95328b99c9eb6cf76c2

modules/transport-netty4/licenses/netty-transport-4.1.94.Final.jar.sha1

@@ -0,0 +1 @@
+ec783a737f96991a87b1d5794e2f9eb2024d708a

modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.94.Final.jar.sha1

@@ -0,0 +1 @@
+3fa5f9d04b6b782d869d6e0657d896eeadca5866