Create a Jenkinsfile to run scan on OpenSearch-projects repos

tools/vulnerability-scan/Jenkinsfile

@@ -0,0 +1,48 @@
+pipeline {
+    agent none
+    triggers {
+        cron('H 3 * * *')
+    }
+
+    stages {
+        stage('Run the scan') {
+            agent {
+                docker {
+                    label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
+                    image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028'
+                    args '-e LANG=en_US.UTF-8'
+                    alwaysPull true
+                }
+            }
+            steps {
+                script {
+                    withCredentials([string(credentialsId: 'whitesource-scan-apikey', variable: 'wss_apikey')]) {
+                        echo 'Executing WhiteSource Scan script'
+                        scan()
+                    }
+                }
+            }
+            post() {
+                always {
+                    cleanWs disableDeferredWipeout: true, deleteDirs: true
+                }
+            }
+        }
+    }
+    post() {
+        success {
+            echo "WhiteSource scan has been completed."
+        }
+        failure {
+            echo "WhiteSource scan failed."
+        }
+    }
+}
+
+void scan() {
+    git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main'
+    sh '''
+        cd ./tools/vulnerability-scan/
+        ./wss-scan.sh
+    '''
+}

tools/vulnerability-scan/wss-scan.config

@@ -1,3 +1,3 @@
 baseDirPath=$(pwd)
 gitBasePath=https://github.com/opensearch-project/
-gitRepos=OpenSearch,OpenSearch-Dashboards,alerting,alerting-dashboards-plugin,anomaly-detection,anomaly-detection-dashboards-plugin,asynchronous-search,common-utils,dashboards-notebooks,dashboards-reports,dashboards-visualizations,data-prepper,index-management,index-management-dashboards-plugin,job-scheduler,k-NN,opensearch-cli,performance-analyzer,performance-analyzer-rca,perftop,security,security-dashboards-plugin,sql,trace-analytics
+gitRepos=OpenSearch-Dashboards,alerting,alerting-dashboards-plugin,anomaly-detection,anomaly-detection-dashboards-plugin,asynchronous-search,common-utils,cross-cluster-replication,dashboards-reports,dashboards-visualizations,data-prepper,index-management,index-management-dashboards-plugin,job-scheduler,k-NN,opensearch-java,opensearch-js,opensearch-py,opensearch-dsl-py,performance-analyzer,perftop,security,security-dashboards-plugin,sql,trace-analytics

tools/vulnerability-scan/wss-scan.sh

@@ -31,14 +31,14 @@ fi
 if [ ! -f "wss-unified-agent.jar" ]
 then
   # Download the WhiteSource Agent 
-  wget -q https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar
-  # The version 20.9.2.1 has been tested and can be used if a specific version is required
-  #wget -q  https://github.com/whitesource/unified-agent-distribution/releases/download/v20.9.2.1/wss-unified-agent.jar
+  curl https://unified-agent.s3.amazonaws.com/wss-unified-agent-21.11.1.jar --output wss-unified-agent.jar
 fi
 
 # scan the config file for the user configurations
 # wss-scan.config has to be present in the same working directory as the script
-source wss-scan.config
+echo "Run before source"
+. ./wss-scan.config
+echo $gitRepos
 
 # change comma to whitespace
 gitRepos=${gitRepos//,/$'\n'} 
@@ -50,7 +50,6 @@ rm -rf $basepath
 
 mkdir -p $basepath
 
-
 # clone the desired Repos for scanning 
 for repo in $gitRepos
 do
@@ -59,7 +58,8 @@ do
 done
 
 echo -n > info.txt
-
+target='OpenSearch'
+mvn -v; node -v; npm -v; yarn -v
 
 # scan the Repos using the WhiteSource Unified Agent
 for repo in $gitRepos
@@ -68,38 +68,12 @@ do
   if [ -d "$repo_path" ]
   then
     echo "Scanning repo: "$gitBasePath$repo " Project: " $repo
-    java -jar wss-unified-agent.jar -c wss-unified-agent.config -d $repo_path -apiKey $wss_apikey -product OpenSearch -project $repo > ${repo}.log
+    java -jar wss-unified-agent.jar -c wss-unified-agent.config -d $repo_path -apiKey $wss_apikey -product "$target" -project $repo
   else
     echo "Scanning failed for repo: "$gitBasePath$repo " Project: " $repo
   fi
 done
 
-
-
-# mail function to send the scan details to the desired recepient 
-mail_format_func()
-{
-
-echo "<html><body><table border=1 cellspacing=0 cellpadding=3>" > output.md
-while IFS= read -r line
-do
-# setting comma as the delimiter
-
-  IFS=','
-  read -ra val <<< "$line"
-  echo "<tr>" >> output.md
-  for ln in "${val[@]}" 
-  do
-    echo "${ln//[[:space:]]/}"
-    echo "<td>"${ln//[[:space:]]/}"</td>" >> output.md
-  done
-  echo "</tr>" >> output.md
-done < info.txt
-echo "</table></body></html>" >> output.md
-
-}
-
-mail_format_func
-
 # remove the WhiteSource unified Jar 
 rm "wss-unified-agent.jar" 
+echo "WhiteSource vulnerability scan completed"

tools/vulnerability-scan/wss-unified-agent.config

@@ -75,6 +75,37 @@ forceUpdate.failBuildOnPolicyViolation=false
 resolveAllDependencies=false
 #excludeDependenciesFromNodes=.*commons-io.*,.*maven-model
 
+resolveAllDependencies=false
+archiveExtractionDepth=7
+followSymbolicLinks=true
+commandTimeout=1900
+gradle.resolveDependencies=true
+gradle.aggregateModules=true
+gradle.preferredEnvironment=wrapper
+maven.resolveDependencies=true
+maven.runPreStep=true
+maven.aggregateModules=true
+maven.ignoredScopes=None
+html.resolveDependencies=true
+npm.resolveDependencies=true
+npm.runPreStep=true
+npm.yarnProject=true
+go.resolveDependencies=true
+go.collectDependenciesAtRuntime=true
+go.dependencyManager=
+python.resolveDependencies=true
+python.ignoreSourceFiles=true
+python.runPipenvPreStep=true
+python.pipenvDevDependencies=true
+python.requirementsFileIncludes=dev-requirements.txt
+python.resolveSetupPyFiles=true
+python.installVirtualenv=true
+ruby.resolveDependencies=true
+ruby.ignoreSourceFiles=false
+ruby.installMissingGems=true
+ruby.runBundleInstall=true
+ruby.overwriteGemFile=true
+
 #npm.resolveDependencies=false
 #npm.ignoreSourceFiles=false
 #npm.includeDevDependencies=true
@@ -137,24 +168,7 @@ resolveAllDependencies=false
 #maven.m2RepositoryPath=
 #maven.downloadMissingDependencies=false
 #maven.additionalArguments=
-#maven.projectNameFromDependencyFile=true										 
-
-resolveAllDependencies=false
-archiveExtractionDepth=7
-followSymbolicLinks=true
-gradle.resolveDependencies=true
-gradle.aggregateModules=true
-maven.resolveDependencies=true
-maven.runPreStep=true
-maven.aggregateModules=true
-maven.ignoredScopes=None
-html.resolveDependencies=true
-npm.resolveDependencies=true
-npm.runPreStep=true
-npm.yarnProject=true
-go.collectDependenciesAtRuntime=true
-go.dependencyManager=modules
-go.resolveDependencies=true
+#maven.projectNameFromDependencyFile=true										  
 
 #gradle.ignoredScopes=
 #gradle.resolveDependencies=true
@@ -245,8 +259,8 @@ go.resolveDependencies=true
 ###########################################################################################
 # Includes/Excludes Glob patterns - Please use only one exclude line and one include line #
 ###########################################################################################
-includes=**/*.cc **/*.zip **/*.cpp **/*.c **/*.swf **/*.tgz **/*.h **/*.js **/*.hpp **/*.py **/*.gzip **/*.cs **/*.rb **/*.exe **/*.gz **/*.pl **/*.cxx **/*.c++ **/*.hxx **/*.jar **/*.java
-#includes=**/*.m **/*.mm  **/*.js **/*.php
+includes=**/*.cc **/*.zip **/*.cpp **/*.c **/*.swf **/*.tgz **/*.h **/*.js **/*.hpp **/*.py **/*.gzip **/*.cs **/*.rb **/*.exe **/*.gz **/*.pl **/*.cxx **/*.c++ **/*.hxx **/*.jar **/*.java **/*.go **/*.mod **/*.sum **/*.rb 
+#includes=**/*.m **/*.mm  **/*.js **/*.php **/*.rb
 #includes=**/*.jar
 #includes=**/*.gem **/*.rb
 #includes=**/*.dll **/*.cs **/*.nupkg