rebase main

Signed-off-by: Kaituo Li <kaituo@amazon.com>

.github/labeler.yml

@@ -1,4 +1,4 @@
-backport 2.x:
+backport 1.x:
   - "*"
   - "*/*"
   - "*/**/*"

.github/workflows/backport.yml

@@ -7,6 +7,7 @@ on:
 
 jobs:
   backport:
+    if: github.event.pull_request.merged == true
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -25,4 +26,5 @@ jobs:
         uses: VachaShah/backport@v2.2.0
         with:
           github_token: ${{ steps.github_app_token.outputs.token }}
-          branch_name: backport/backport-${{ github.event.number }}
+          head_template: backport/backport-<%= number %>-to-<%= base %>
+          failure_labels: backport-failed

.github/workflows/test_build_multi_platform.yml

@@ -46,7 +46,7 @@ jobs:
 
       - name: Build and Run Tests
         run: |
-          ./gradlew build
+          ./gradlew build -x spotlessJava
       - name: Publish to Maven Local
         run: |
           ./gradlew publishToMavenLocal
@@ -91,7 +91,7 @@ jobs:
         run: |
           chown -R 1000:1000 `pwd`
           su `id -un 1000` -c "./gradlew assemble &&
-                               ./gradlew build &&
+                               ./gradlew build -x spotlessJava &&
                                ./gradlew publishToMavenLocal &&
                                ./gradlew integTest -PnumNodes=3"
       - name: Upload Coverage Report
@@ -127,7 +127,7 @@ jobs:
           ./gradlew assemble
       - name: Build and Run Tests
         run: |
-          ./gradlew build
+          ./gradlew build -x spotlessJava
       - name: Publish to Maven Local
         run: |
           ./gradlew publishToMavenLocal

build.gradle

@@ -35,7 +35,7 @@ buildscript {
         js_resource_folder = "src/test/resources/job-scheduler"
         common_utils_version = System.getProperty("common_utils.version", opensearch_build)
         job_scheduler_version = System.getProperty("job_scheduler.version", opensearch_build)
-        bwcVersionShort = "2.10.0"
+        bwcVersionShort = "1.3.2"
         bwcVersion = bwcVersionShort + ".0"
         bwcOpenSearchADDownload = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + bwcVersionShort + '/latest/linux/x64/tar/builds/' +
                 'opensearch/plugins/opensearch-anomaly-detection-' + bwcVersion + '.zip'
@@ -125,10 +125,10 @@ dependencies {
     implementation group: 'com.yahoo.datasketches', name: 'sketches-core', version: '0.13.4'
     implementation group: 'com.yahoo.datasketches', name: 'memory', version: '0.12.2'
     implementation group: 'commons-lang', name: 'commons-lang', version: '2.6'
-    implementation group: 'org.apache.commons', name: 'commons-pool2', version: '2.11.1'
-    implementation 'software.amazon.randomcutforest:randomcutforest-serialization:3.8.0'
-    implementation 'software.amazon.randomcutforest:randomcutforest-parkservices:3.8.0'
-    implementation 'software.amazon.randomcutforest:randomcutforest-core:3.8.0'
+    implementation group: 'org.apache.commons', name: 'commons-pool2', version: '2.12.0'
+    implementation 'software.amazon.randomcutforest:randomcutforest-serialization:4.0.0'
+    implementation 'software.amazon.randomcutforest:randomcutforest-parkservices:4.0.0'
+    implementation 'software.amazon.randomcutforest:randomcutforest-core:4.0.0'
 
     // we inherit jackson-core from opensearch core
     implementation "com.fasterxml.jackson.core:jackson-databind:2.16.1"
@@ -149,6 +149,9 @@ dependencies {
         exclude group: 'org.ow2.asm', module: 'asm-tree'
     }
 
+    // used for output encoding of config descriptions
+    implementation group: 'org.owasp.encoder' , name: 'encoder', version: '1.2.3'
+
     testImplementation group: 'pl.pragmatists', name: 'JUnitParams', version: '1.1.1'
     testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.9.0'
     testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.3'
@@ -538,7 +541,7 @@ List<Provider<RegularFile>> plugins = [
 
 // Creates 2 test clusters with 3 nodes of the old version. 
 2.times {i -> 
-    task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) {
+    task "${baseName}#oldVersionClusterTask$i"(type: RestIntegTestTask) {
         useCluster testClusters."${baseName}$i"
         filter {
             includeTestsMatching "org.opensearch.ad.bwc.*IT"
@@ -554,7 +557,7 @@ List<Provider<RegularFile>> plugins = [
 // Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version 
 // This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
 // This is also used as a one third upgraded cluster for a rolling upgrade.
-task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
+task "${baseName}#mixedClusterTask"(type: RestIntegTestTask) {
     useCluster testClusters."${baseName}0"
     dependsOn "${baseName}#oldVersionClusterTask0"
     doFirst {
@@ -573,7 +576,7 @@ task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
 // Upgrades the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded.
 // This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
 // This is used for rolling upgrade.
-task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
+task "${baseName}#twoThirdsUpgradedClusterTask"(type: RestIntegTestTask) {
     dependsOn "${baseName}#mixedClusterTask"
     useCluster testClusters."${baseName}0"
     doFirst {
@@ -592,7 +595,7 @@ task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTas
 // Upgrades the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded.
 // This results in a fully upgraded cluster.
 // This is used for rolling upgrade.
-task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
+task "${baseName}#rollingUpgradeClusterTask"(type: RestIntegTestTask) {
     dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
     useCluster testClusters."${baseName}0"
     doFirst {
@@ -611,7 +614,7 @@ task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask)
 
 // Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version 
 // at the same time resulting in a fully upgraded cluster.
-task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) {
+task "${baseName}#fullRestartClusterTask"(type: RestIntegTestTask) {
     dependsOn "${baseName}#oldVersionClusterTask1"
     useCluster testClusters."${baseName}1"
     doFirst {
@@ -627,7 +630,7 @@ task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) {
 }
 
 // A bwc test suite which runs all the bwc tasks combined.
-task bwcTestSuite(type: StandaloneRestIntegTestTask) {
+task bwcTestSuite(type: RestIntegTestTask) {
     exclude '**/*Test*'
     exclude '**/*IT*'
     dependsOn tasks.named("${baseName}#mixedClusterTask")

src/main/java/org/opensearch/ad/ml/CheckpointDao.java

@@ -744,7 +744,8 @@ private Optional<ThresholdedRandomCutForest> convertToTRCF(Optional<RandomCutFor
         if (kllThreshold.isPresent()) {
             scores = kllThreshold.get().extractScores();
         }
-        return Optional.of(new ThresholdedRandomCutForest(rcf.get(), anomalyRate, scores));
+        // last parameter is lastShingledInput. Since we don't know it, use all 0 double array
+        return Optional.of(new ThresholdedRandomCutForest(rcf.get(), anomalyRate, scores, new double[rcf.get().getDimensions()]));
     }
 
     /**

src/main/java/org/opensearch/ad/task/ADTaskManager.java

@@ -18,20 +18,7 @@
 import static org.opensearch.ad.constant.ADCommonMessages.NO_ELIGIBLE_NODE_TO_RUN_DETECTOR;
 import static org.opensearch.ad.constant.ADCommonName.DETECTION_STATE_INDEX;
 import static org.opensearch.ad.indices.ADIndexManagement.ALL_AD_RESULTS_INDEX_PATTERN;
-import static org.opensearch.ad.model.ADTask.COORDINATING_NODE_FIELD;
 import static org.opensearch.ad.model.ADTask.DETECTOR_ID_FIELD;
-import static org.opensearch.ad.model.ADTask.ERROR_FIELD;
-import static org.opensearch.ad.model.ADTask.ESTIMATED_MINUTES_LEFT_FIELD;
-import static org.opensearch.ad.model.ADTask.EXECUTION_END_TIME_FIELD;
-import static org.opensearch.ad.model.ADTask.EXECUTION_START_TIME_FIELD;
-import static org.opensearch.ad.model.ADTask.INIT_PROGRESS_FIELD;
-import static org.opensearch.ad.model.ADTask.IS_LATEST_FIELD;
-import static org.opensearch.ad.model.ADTask.LAST_UPDATE_TIME_FIELD;
-import static org.opensearch.ad.model.ADTask.PARENT_TASK_ID_FIELD;
-import static org.opensearch.ad.model.ADTask.STATE_FIELD;
-import static org.opensearch.ad.model.ADTask.STOPPED_BY_FIELD;
-import static org.opensearch.ad.model.ADTask.TASK_PROGRESS_FIELD;
-import static org.opensearch.ad.model.ADTask.TASK_TYPE_FIELD;
 import static org.opensearch.ad.model.ADTaskType.ALL_HISTORICAL_TASK_TYPES;
 import static org.opensearch.ad.model.ADTaskType.HISTORICAL_DETECTOR_TASK_TYPES;
 import static org.opensearch.ad.model.ADTaskType.REALTIME_TASK_TYPES;
@@ -52,6 +39,19 @@
 import static org.opensearch.timeseries.constant.CommonName.TASK_ID_FIELD;
 import static org.opensearch.timeseries.model.TaskState.NOT_ENDED_STATES;
 import static org.opensearch.timeseries.model.TaskType.taskTypeToString;
+import static org.opensearch.timeseries.model.TimeSeriesTask.COORDINATING_NODE_FIELD;
+import static org.opensearch.timeseries.model.TimeSeriesTask.ERROR_FIELD;
+import static org.opensearch.timeseries.model.TimeSeriesTask.ESTIMATED_MINUTES_LEFT_FIELD;
+import static org.opensearch.timeseries.model.TimeSeriesTask.EXECUTION_END_TIME_FIELD;
+import static org.opensearch.timeseries.model.TimeSeriesTask.EXECUTION_START_TIME_FIELD;
+import static org.opensearch.timeseries.model.TimeSeriesTask.INIT_PROGRESS_FIELD;
+import static org.opensearch.timeseries.model.TimeSeriesTask.IS_LATEST_FIELD;
+import static org.opensearch.timeseries.model.TimeSeriesTask.LAST_UPDATE_TIME_FIELD;
+import static org.opensearch.timeseries.model.TimeSeriesTask.PARENT_TASK_ID_FIELD;
+import static org.opensearch.timeseries.model.TimeSeriesTask.STATE_FIELD;
+import static org.opensearch.timeseries.model.TimeSeriesTask.STOPPED_BY_FIELD;
+import static org.opensearch.timeseries.model.TimeSeriesTask.TASK_PROGRESS_FIELD;
+import static org.opensearch.timeseries.model.TimeSeriesTask.TASK_TYPE_FIELD;
 import static org.opensearch.timeseries.settings.TimeSeriesSettings.NUM_MIN_SAMPLES;
 import static org.opensearch.timeseries.util.ExceptionUtil.getErrorMessage;
 import static org.opensearch.timeseries.util.ExceptionUtil.getShardsFailure;

src/test/java/org/opensearch/ad/ODFERestTestCase.java

@@ -35,6 +35,7 @@
 import org.apache.http.HttpHost;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
 import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.message.BasicHeader;
@@ -187,9 +188,8 @@ protected static void configureHttpsClient(RestClientBuilder builder, Settings s
             String password = Optional
                 .ofNullable(System.getProperty("password"))
                 .orElseThrow(() -> new RuntimeException("password is missing"));
-            BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-            final AuthScope anyScope = new AuthScope(null, -1);
-            credentialsProvider.setCredentials(anyScope, new UsernamePasswordCredentials(userName, password));
+            CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+            credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password));
             try {
                 return httpClientBuilder
                     .setDefaultCredentialsProvider(credentialsProvider)

src/test/java/org/opensearch/ad/bwc/ADBackwardsCompatibilityIT.java

@@ -8,6 +8,9 @@
 
 package org.opensearch.ad.bwc;
 
+import static org.opensearch.ad.rest.ADRestTestUtils.DetectorType.MULTI_CATEGORY_HC_DETECTOR;
+import static org.opensearch.ad.rest.ADRestTestUtils.DetectorType.SINGLE_CATEGORY_HC_DETECTOR;
+import static org.opensearch.ad.rest.ADRestTestUtils.DetectorType.SINGLE_ENTITY_DETECTOR;
 import static org.opensearch.ad.rest.ADRestTestUtils.countADResultOfDetector;
 import static org.opensearch.ad.rest.ADRestTestUtils.countDetectors;
 import static org.opensearch.ad.rest.ADRestTestUtils.createAnomalyDetector;
@@ -21,9 +24,6 @@
 import static org.opensearch.ad.rest.ADRestTestUtils.stopHistoricalAnalysis;
 import static org.opensearch.ad.rest.ADRestTestUtils.stopRealtimeJob;
 import static org.opensearch.ad.rest.ADRestTestUtils.waitUntilTaskDone;
-import static org.opensearch.ad.rest.ADRestTestUtils.DetectorType.MULTI_CATEGORY_HC_DETECTOR;
-import static org.opensearch.ad.rest.ADRestTestUtils.DetectorType.SINGLE_CATEGORY_HC_DETECTOR;
-import static org.opensearch.ad.rest.ADRestTestUtils.DetectorType.SINGLE_ENTITY_DETECTOR;
 import static org.opensearch.timeseries.util.RestHandlerUtils.ANOMALY_DETECTOR_JOB;
 import static org.opensearch.timeseries.util.RestHandlerUtils.HISTORICAL_ANALYSIS_TASK;
 import static org.opensearch.timeseries.util.RestHandlerUtils.REALTIME_TASK;
@@ -167,7 +167,7 @@ public void testBackwardsCompatibility() throws Exception {
                 case MIXED:
                     // TODO: We have no way to specify whether send request to old node or new node now.
                     // Add more test later when it's possible to specify request node.
-                    Assert.assertTrue(pluginNames.contains("opensearch-anomaly-detection"));
+                    Assert.assertTrue(pluginNames.contains("opensearch-time-series-analytics"));
                     Assert.assertTrue(pluginNames.contains("opensearch-job-scheduler"));
 
                     // Create single entity detector and start realtime job