Backport/backport 211 to 1.x

CHANGELOG.md

@@ -6,16 +6,17 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ### Added
 - Add workflows and scripts for automating model tracing and uploading process by @thanawan-atc in ([#209](https://github.com/opensearch-project/opensearch-py-ml/pull/209))
 - Add workflow and scripts for automating model listing updating process by @thanawan-atc in ([#210](https://github.com/opensearch-project/opensearch-py-ml/pull/210))
+- Add script to trigger ml-models-release jenkins workflow with generic webhook by @thanawan-atc in ([#211](https://github.com/opensearch-project/opensearch-py-ml/pull/211))
 
-
 ### Changed
-
+- Modify ml-models.JenkinsFile so that it takes model format into account and can be triggered with generic webhook by @thanawan-atc in ([#211](https://github.com/opensearch-project/opensearch-py-ml/pull/211))
 
 ### Fixed
 - Enable make_model_config_json to add model description to model config file by @thanawan-atc in ([#203](https://github.com/opensearch-project/opensearch-py-ml/pull/203))
 - Correct demo_ml_commons_integration.ipynb by @thanawan-atc in ([#208](https://github.com/opensearch-project/opensearch-py-ml/pull/208))
 - Handle the case when the model max length is undefined in tokenizer by @thanawan-atc in ([#219](https://github.com/opensearch-project/opensearch-py-ml/pull/219))
 
+
 ## [1.1.0]
 
 ### Added
@@ -32,6 +33,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Search task added @Nurlanprog ([#177](https://github.com/opensearch-project/opensearch-py-ml/pull/177/files))
 - adding jupyter notebook based documentation for metrics correlation algorithm by @AlibiZhenis ([#186](https://github.com/opensearch-project/opensearch-py-ml/pull/186))
 
+
 ### Changed
 - Update jenkins file to use updated docker image ([#189](https://github.com/opensearch-project/opensearch-py-ml/pull/189))
 - Updated documentation @dhrubo-os ([#98](https://github.com/opensearch-project/opensearch-py-ml/pull/98))

jenkins/ml-models.JenkinsFile

@@ -26,19 +26,53 @@ pipeline {
             description: 'Version number of the model',
             trim: true
         )
-        }
+        choice(
+            name: 'FORMAT',
+            description: 'Tracing format of the model',
+            /* The first choice will be the default choice, and we need it to be empty string
+               so that we can update model listing without specifying 'FORMAT' in JENKINS_PARAM. */
+            choices: ['', 'BOTH', 'TORCH_SCRIPT', 'ONNX'] // We set empty string to be the f
+        )
+    }
+    triggers {
+        GenericTrigger(
+            genericVariables: [
+                [key: 'BASE_DOWNLOAD_PATH', value: '$.BASE_DOWNLOAD_PATH'],
+                [key: 'VERSION', value: '$.VERSION'],
+                [key: 'FORMAT', value: '$.FORMAT']
+            ],
+            token: 'JENKINS_ML_MODELS_RELEASE_GENERIC_WEBHOOK_TOKEN',
+            causeString: 'Triggered by GitHub Actions Workflow',
+            printContributedVariables: true,
+            printPostContent: true
+        )
+    }
     environment {
-        ARTIFACT_PATH = "${BASE_DOWNLOAD_PATH}/${VERSION}/"
+        ARTIFACT_PATH = "${env.BASE_DOWNLOAD_PATH == "ml-models/model_listing" ? "${env.BASE_DOWNLOAD_PATH}" : "${env.BASE_DOWNLOAD_PATH}/${env.VERSION}"}"
         UPLOAD_PATH = "models/ml-models"
-        BUCKET_NAME = credentials('ml-models-bucket-name')
     }
     stages{
         stage('Parameters Check') {
             steps {
                 script {
-                    if(BASE_DOWNLOAD_PATH.isEmpty() || VERSION.isEmpty()) {
+                    echo "BASE_DOWNLOAD_PATH: ${env.BASE_DOWNLOAD_PATH}"
+                    echo "VERSION: ${env.VERSION}"
+                    echo "FORMAT: ${env.FORMAT}"
+                    echo "ARTIFACT_PATH: ${env.ARTIFACT_PATH}"
+                    echo "UPLOAD_PATH: ${env.UPLOAD_PATH}"
+                    if (env.BASE_DOWNLOAD_PATH == "ml-models/model_listing") {
+                        echo "Proceeding to download the ml-models/model_listing."
+                    } else if (env.BASE_DOWNLOAD_PATH.isEmpty()) {
                         currentBuild.result = 'ABORTED'
-                        error('Parameters cannot be empty! Please provide the correct values.')
+                        error('BASE_DOWNLOAD_PATH cannot be empty! Please provide the correct values.')
+                    } else if (env.VERSION.isEmpty()) {
+                        currentBuild.result = 'ABORTED'
+                        error('VERSION cannot be empty! Please provide the correct values.')
+                    } else if (env.FORMAT.isEmpty()) {
+                        currentBuild.result = 'ABORTED'
+                        error('FORMAT cannot be empty! Please provide the correct values.')
+                    } else {
+                        echo "Proceeding to download the ml-models."
                     }
                     if(BASE_DOWNLOAD_PATH.endsWith('/')) {
                         currentBuild.result = 'ABORTED'
@@ -50,15 +84,49 @@ pipeline {
         stage('Download the artifacts') {
             steps {
                 script {
-                    downloadFromS3(
-                        assumedRoleName: 'get_models',
-                        roleAccountNumberCred: 'ml-models-aws-account-number',
-                        downloadPath: "${ARTIFACT_PATH}",
-                        bucketName: "${BUCKET_NAME}",
-                        localPath: "${WORKSPACE}/artifacts",
-                        force: true,
-                        region: 'us-west-2'
-                    )
+                    if (env.BASE_DOWNLOAD_PATH == "ml-models/model_listing") {
+                        downloadFromS3(
+                            assumedRoleName: 'get_models',
+                            roleAccountNumberCred: 'ml-models-aws-account-number',
+                            downloadPath: "${env.ARTIFACT_PATH}/",
+                            bucketName: "${env.BUCKET_NAME}",
+                            localPath: "${env.WORKSPACE}/artifacts",
+                            force: true,
+                            region: 'us-west-2'
+                        )
+                    } else {
+                        if (env.FORMAT == "TORCH_SCRIPT") {
+                            downloadFromS3(
+                                assumedRoleName: 'get_models',
+                                roleAccountNumberCred: 'ml-models-aws-account-number',
+                                downloadPath: "${env.ARTIFACT_PATH}/torch_script/",
+                                bucketName: "${env.BUCKET_NAME}",
+                                localPath: "${env.WORKSPACE}/artifacts",
+                                force: true,
+                                region: 'us-west-2'
+                            )
+                        } else if (env.FORMAT == "ONNX") {
+                            downloadFromS3(
+                                assumedRoleName: 'get_models',
+                                roleAccountNumberCred: 'ml-models-aws-account-number',
+                                downloadPath: "${env.ARTIFACT_PATH}/onnx/",
+                                bucketName: "${env.BUCKET_NAME}",
+                                localPath: "${env.WORKSPACE}/artifacts",
+                                force: true,
+                                region: 'us-west-2'
+                            )
+                        } else if (env.FORMAT == "BOTH") {
+                            downloadFromS3(
+                                assumedRoleName: 'get_models',
+                                roleAccountNumberCred: 'ml-models-aws-account-number',
+                                downloadPath: "${env.ARTIFACT_PATH}/",
+                                bucketName: "${env.BUCKET_NAME}",
+                                localPath: "${env.WORKSPACE}/artifacts",
+                                force: true,
+                                region: 'us-west-2'
+                            )
+                        }
+                    }
                 }
             }
         }
@@ -81,7 +149,7 @@ pipeline {
         always {
             script {
                 postCleanup()
-                }
             }
         }
     }
+}

utils/model_uploader/trigger_ml_models_release.sh

@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: Apache-2.0
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+# Any modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+
+# This program is run by "Model Auto-tracing & Uploading" 
+# & "Model Listing Uploading" workflow (See model_uploader.yml 
+# & model_listing_uploader.yml) to trigger ml-models-release 
+# Jenkins workflow.
+
+JENKINS_TRIGGER_TOKEN=$1
+JENKINS_PARAMS=$2
+JENKINS_URL="https://build.ci.opensearch.org"
+
+TIMEPASS=0
+TIMEOUT=3600
+RESULT="null"
+
+JENKINS_REQ=$(curl -s -XPOST \
+             -H "Authorization: Bearer $JENKINS_TRIGGER_TOKEN" \
+             -H "Content-Type: application/json" \
+             "$JENKINS_URL/generic-webhook-trigger/invoke" \
+             --data "$JENKINS_PARAMS")
+
+echo "Trigger ml-models-release Jenkins workflows"
+echo $JENKINS_PARAMS
+echo $JENKINS_REQ
+
+QUEUE_URL=$(echo $JENKINS_REQ | jq --raw-output '.jobs."ml-models-release".url')
+echo "QUEUE_URL: $QUEUE_URL"
+echo "Wait for jenkins to start workflow" && sleep 15
+
+echo "Check if queue exist in Jenkins after triggering"
+if [ -z "$QUEUE_URL" ] || [ "$QUEUE_URL" != "null" ]; then
+    WORKFLOW_URL=$(curl -s -XGET ${JENKINS_URL}/${QUEUE_URL}api/json | jq --raw-output .executable.url)
+    echo "WORKFLOW_URL: $WORKFLOW_URL"
+    echo "Use queue information to find build number in Jenkins if available"
+    if [ -z "$WORKFLOW_URL" ] || [ "$WORKFLOW_URL" != "null" ]; then
+        RUNNING="true"
+        echo "Waiting for Jenkins to complete the run"
+        while [ "$RUNNING" = "true" ] && [ "$TIMEPASS" -le "$TIMEOUT" ]; do
+            echo "Still running, wait for another 5 seconds before checking again, max timeout $TIMEOUT"
+            echo "Jenkins Workflow URL: $WORKFLOW_URL"
+            TIMEPASS=$(( TIMEPASS + 5 )) && echo time pass: $TIMEPASS
+            sleep 5
+            RUNNING=$(curl -s -XGET ${WORKFLOW_URL}api/json | jq --raw-output .building)
+        done
+
+        if [ "$RUNNING" = "true" ]; then
+            echo "Timed out"
+            RESULT="TIMEOUT"
+        else
+            echo "Completed the run, checking the results now......"
+            RESULT=$(curl -s -XGET ${WORKFLOW_URL}api/json | jq --raw-output .result)
+        fi
+    fi
+fi
+
+echo "Please check jenkins url for logs: $WORKFLOW_URL"
+echo "Result: $RESULT"
+if [ "$RESULT" != "SUCCESS" ]; then
+    exit 1
+fi