diff --git a/k8s-custom-pipelines.yml b/k8s-custom-pipelines.yml index 7b510de285..6a840b390b 100644 --- a/k8s-custom-pipelines.yml +++ b/k8s-custom-pipelines.yml @@ -1,318 +1,301 @@ -resources: - repositories: - - repository: K8sPartnerExtensionTest - type: git - endpoint: AzureReposConnection - name: One/compute-HybridMgmt-K8sPartnerExtensionTest - ref: joinnis/scc-testing - -trigger: - batch: true - branches: - include: - - k8s-configuration -pr: - branches: - include: - - k8s-configuration - -stages: -- stage: BuildTestPublishExtension - displayName: "Build, Test, and Publish Extension" - variables: - K8S_EXTENSION_REPO_PATH: $(Agent.BuildDirectory)/s/compute-HybridMgmt-K8sPartnerExtensionTest - CLI_REPO_PATH: $(Agent.BuildDirectory)/s/azure-cli-extensions - EXTENSION_NAME: "k8s-configuration" - EXTENSION_FILE_NAME: "k8s_configuration" - SUBSCRIPTION_ID: "15c06b1b-01d6-407b-bb21-740b8617dea3" - RESOURCE_GROUP: "K8sPartnerExtensionTest" - BASE_CLUSTER_NAME: "k8s-configuration-cluster" - jobs: - - job: K8sConfigurationTestSuite - displayName: "Run the Test Suite" - pool: - vmImage: 'ubuntu-16.04' - steps: - - checkout: self - - checkout: K8sPartnerExtensionTest - - - bash: | - echo "Installing helm3" - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 get_helm.sh - ./get_helm.sh - - echo "Installing kubectl" - curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" - chmod +x ./kubectl - sudo mv ./kubectl /usr/local/bin/kubectl - kubectl version --client - displayName: "Setup the VM with helm3 and kubectl" - - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' - inputs: - versionSpec: 3.6 - - bash: | - set -ev - - # prepare and activate virtualenv - pip install virtualenv - python3 -m venv env/ - source env/bin/activate - - # clone azure-cli - pip install azdev - - ls $(CLI_REPO_PATH) - - azdev --version - azdev setup -r $(CLI_REPO_PATH) -e $(EXTENSION_NAME) - azdev extension build $(EXTENSION_NAME) - - workingDirectory: $(CLI_REPO_PATH) - displayName: "Setup and Build $(EXTENSION_NAME) with azdev" - - - bash: | - K8S_CONFIG_VERSION=$(ls ${EXTENSION_FILE_NAME}* | cut -d "-" -f2) - echo "##vso[task.setvariable variable=K8S_CONFIG_VERSION]$K8S_CONFIG_VERSION" - cp * $(K8S_EXTENSION_REPO_PATH)/extensions - workingDirectory: $(CLI_REPO_PATH)/dist - displayName: "Copy the Built .whl to Extension Test Path" - - - bash: | - RAND_STR=$RANDOM - AKS_CLUSTER_NAME="${BASE_CLUSTER_NAME}-${RAND_STR}-aks" - ARC_CLUSTER_NAME="${BASE_CLUSTER_NAME}-${RAND_STR}-arc" - - JSON_STRING=$(jq -n \ - --arg SUB_ID "$SUBSCRIPTION_ID" \ - --arg RG "$RESOURCE_GROUP" \ - --arg AKS_CLUSTER_NAME "$AKS_CLUSTER_NAME" \ - --arg ARC_CLUSTER_NAME "$ARC_CLUSTER_NAME" \ - --arg K8S_CONFIG_VERSION "$K8S_CONFIG_VERSION" \ - '{subscriptionId: $SUB_ID, resourceGroup: $RG, aksClusterName: $AKS_CLUSTER_NAME, arcClusterName: $ARC_CLUSTER_NAME, extensionVersion: {"k8s-configuration": $K8S_CONFIG_VERSION, connectedk8s: "1.0.0"}}') - echo $JSON_STRING > settings.json - cat settings.json - workingDirectory: $(K8S_EXTENSION_REPO_PATH) - displayName: "Generate a settings.json file" - - - bash : | - echo "Downloading the kind script" - curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.9.0/kind-linux-amd64 - chmod +x ./kind - ./kind create cluster - displayName: "Create and Start the Kind cluster" - - - task: AzureCLI@2 - displayName: Bootstrap - inputs: - azureSubscription: AzureResourceConnection - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - .\Bootstrap.ps1 -CI - workingDirectory: $(K8S_EXTENSION_REPO_PATH) - - - task: AzureCLI@2 - displayName: Run the Test Suite - inputs: - azureSubscription: AzureResourceConnection - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - .\Test.ps1 -Type Configuration -CI - workingDirectory: $(K8S_EXTENSION_REPO_PATH) - continueOnError: true - - - task: PublishTestResults@2 - inputs: - testResultsFormat: 'JUnit' - testResultsFiles: '**/TestResults.xml' - failTaskOnFailedTests: true - condition: succeededOrFailed() - - - task: AzureCLI@2 - displayName: Cleanup - inputs: - azureSubscription: AzureResourceConnection - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - .\Cleanup.ps1 -CI - workingDirectory: $(K8S_EXTENSION_REPO_PATH) - condition: succeededOrFailed() - - - job: BuildPublishExtension - pool: - vmImage: 'ubuntu-16.04' - displayName: "Build and Publish the Extension Artifact" - variables: - CLI_REPO_PATH: $(Agent.BuildDirectory)/s - EXTENSION_NAME: "k8s-configuration" - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' - inputs: - versionSpec: 3.6 - - bash: | - set -ev - echo "Building extension ${EXTENSION_NAME}..." - - # prepare and activate virtualenv - pip install virtualenv - python3 -m venv env/ - source env/bin/activate - - # clone azure-cli - pip install azdev - - ls $(CLI_REPO_PATH) - - azdev --version - azdev setup -r $(CLI_REPO_PATH) -e $(EXTENSION_NAME) - azdev extension build $(EXTENSION_NAME) - workingDirectory: $(CLI_REPO_PATH) - displayName: "Setup and Build Extension with azdev" - - task: PublishBuildArtifacts@1 - inputs: - pathToPublish: $(CLI_REPO_PATH)/dist - -- stage: AzureCLIOfficial - displayName: "Azure Official CLI Code Checks" - dependsOn: [] - jobs: - - job: CheckLicenseHeader - displayName: "Check License" - pool: - vmImage: 'ubuntu-16.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' - inputs: - versionSpec: 3.6 - - bash: | - set -ev - - # prepare and activate virtualenv - python -m venv env/ - - chmod +x ./env/bin/activate - source ./env/bin/activate - - # clone azure-cli - git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli - - pip install -q azdev - - azdev setup -c ../azure-cli -r ./ - - azdev --version - az --version - - azdev verify license - - - job: StaticAnalysis - displayName: "Static Analysis" - pool: - vmImage: 'ubuntu-16.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' - inputs: - versionSpec: 3.6 - - bash: pip install wheel==0.30.0 pylint==1.9.5 flake8==3.5.0 requests - displayName: 'Install wheel, pylint, flake8, requests' - - bash: python scripts/ci/source_code_static_analysis.py - displayName: "Static Analysis" - - - job: IndexVerify - displayName: "Verify Extensions Index" - pool: - vmImage: 'ubuntu-16.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7' - inputs: - versionSpec: 3.7 - - bash: | - #!/usr/bin/env bash - set -ev - pip install wheel==0.30.0 requests packaging - export CI="ADO" - python ./scripts/ci/test_index.py -v - displayName: "Verify Extensions Index" - - - job: SourceTests - displayName: "Integration Tests, Build Tests" - pool: - vmImage: 'ubuntu-16.04' - strategy: - matrix: - Python36: - python.version: '3.6' - Python38: - python.version: '3.8' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - - bash: pip install wheel==0.30.0 - displayName: 'Install wheel==0.30.0' - - bash: ./scripts/ci/test_source.sh - displayName: 'Run integration test and build test' - env: - ADO_PULL_REQUEST_LATEST_COMMIT: $(System.PullRequest.SourceCommitId) - ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) - - - job: LintModifiedExtensions - displayName: "CLI Linter on Modified Extensions" - pool: - vmImage: 'ubuntu-16.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' - inputs: - versionSpec: 3.6 - - bash: | - set -ev - - # prepare and activate virtualenv - pip install virtualenv - python -m virtualenv venv/ - source ./venv/bin/activate - - # clone azure-cli - git clone --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli - - pip install azdev - - azdev --version - - azdev setup -c ../azure-cli -r ./ -e k8s-configuration - - # overwrite the default AZURE_EXTENSION_DIR set by ADO - AZURE_EXTENSION_DIR=~/.azure/cliextensions az --version - - AZURE_EXTENSION_DIR=~/.azure/cliextensions azdev linter --include-whl-extensions k8s-configuration - displayName: "CLI Linter on Modified Extension" - env: - ADO_PULL_REQUEST_LATEST_COMMIT: $(System.PullRequest.SourceCommitId) - ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) - - - job: IndexRefDocVerify - displayName: "Verify Ref Docs" - pool: - vmImage: 'ubuntu-16.04' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7' - inputs: - versionSpec: 3.7 - - bash: pip install wheel==0.30.0 - displayName: 'Install wheel==0.30.0' - - task: Bash@3 - displayName: "Verify Extension Ref Docs" - inputs: - targetType: 'filePath' - filePath: scripts/ci/test_index_ref_doc.sh +resources: + repositories: + - repository: K8sPartnerExtensionTest + type: git + endpoint: AzureReposConnection + name: One/compute-HybridMgmt-K8sPartnerExtensionTest + ref: joinnis/scc-testing + +trigger: + batch: true + branches: + include: + - k8s-configuration +pr: + branches: + include: + - k8s-configuration + +stages: +- stage: BuildTestPublishExtension + displayName: "Build, Test, and Publish Extension" + variables: + K8S_EXTENSION_REPO_PATH: $(Agent.BuildDirectory)/s/compute-HybridMgmt-K8sPartnerExtensionTest + CLI_REPO_PATH: $(Agent.BuildDirectory)/s/azure-cli-extensions + EXTENSION_NAME: "k8s-configuration" + EXTENSION_FILE_NAME: "k8s_configuration" + SUBSCRIPTION_ID: "15c06b1b-01d6-407b-bb21-740b8617dea3" + RESOURCE_GROUP: "K8sPartnerExtensionTest" + BASE_CLUSTER_NAME: "k8s-configuration-cluster" + jobs: + - job: K8sConfigurationTestSuite + displayName: "Run the Test Suite" + pool: + vmImage: 'ubuntu-16.04' + steps: + - checkout: self + - checkout: K8sPartnerExtensionTest + + - bash: | + echo "Installing helm3" + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 get_helm.sh + ./get_helm.sh + + echo "Installing kubectl" + curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x ./kubectl + sudo mv ./kubectl /usr/local/bin/kubectl + kubectl version --client + displayName: "Setup the VM with helm3 and kubectl" + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' + inputs: + versionSpec: 3.6 + - bash: | + set -ev + + # prepare and activate virtualenv + pip install virtualenv + python3 -m venv env/ + source env/bin/activate + + # clone azure-cli + pip install azdev + + ls $(CLI_REPO_PATH) + + azdev --version + azdev setup -r $(CLI_REPO_PATH) -e $(EXTENSION_NAME) + azdev extension build $(EXTENSION_NAME) + + workingDirectory: $(CLI_REPO_PATH) + displayName: "Setup and Build $(EXTENSION_NAME) with azdev" + + - bash: | + K8S_CONFIG_VERSION=$(ls ${EXTENSION_FILE_NAME}* | cut -d "-" -f2) + echo "##vso[task.setvariable variable=K8S_CONFIG_VERSION]$K8S_CONFIG_VERSION" + cp * $(K8S_EXTENSION_REPO_PATH)/extensions + workingDirectory: $(CLI_REPO_PATH)/dist + displayName: "Copy the Built .whl to Extension Test Path" + + - bash: | + RAND_STR=$RANDOM + AKS_CLUSTER_NAME="${BASE_CLUSTER_NAME}-${RAND_STR}-aks" + ARC_CLUSTER_NAME="${BASE_CLUSTER_NAME}-${RAND_STR}-arc" + + JSON_STRING=$(jq -n \ + --arg SUB_ID "$SUBSCRIPTION_ID" \ + --arg RG "$RESOURCE_GROUP" \ + --arg AKS_CLUSTER_NAME "$AKS_CLUSTER_NAME" \ + --arg ARC_CLUSTER_NAME "$ARC_CLUSTER_NAME" \ + --arg K8S_CONFIG_VERSION "$K8S_CONFIG_VERSION" \ + '{subscriptionId: $SUB_ID, resourceGroup: $RG, aksClusterName: $AKS_CLUSTER_NAME, arcClusterName: $ARC_CLUSTER_NAME, extensionVersion: {"k8s-configuration": $K8S_CONFIG_VERSION, connectedk8s: "1.0.0"}}') + echo $JSON_STRING > settings.json + cat settings.json + workingDirectory: $(K8S_EXTENSION_REPO_PATH) + displayName: "Generate a settings.json file" + + - bash : | + echo "Downloading the kind script" + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.9.0/kind-linux-amd64 + chmod +x ./kind + ./kind create cluster + displayName: "Create and Start the Kind cluster" + + - task: AzureCLI@2 + displayName: Bootstrap + inputs: + azureSubscription: AzureResourceConnection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + .\Bootstrap.ps1 -CI + workingDirectory: $(K8S_EXTENSION_REPO_PATH) + + - task: AzureCLI@2 + displayName: Run the Test Suite + inputs: + azureSubscription: AzureResourceConnection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + .\Test.ps1 -Type Configuration -CI + workingDirectory: $(K8S_EXTENSION_REPO_PATH) + continueOnError: true + + - task: PublishTestResults@2 + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: '**/TestResults.xml' + failTaskOnFailedTests: true + condition: succeededOrFailed() + + - task: AzureCLI@2 + displayName: Cleanup + inputs: + azureSubscription: AzureResourceConnection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + .\Cleanup.ps1 -CI + workingDirectory: $(K8S_EXTENSION_REPO_PATH) + condition: succeededOrFailed() + + - job: BuildPublishExtension + pool: + vmImage: 'ubuntu-16.04' + displayName: "Build and Publish the Extension Artifact" + variables: + CLI_REPO_PATH: $(Agent.BuildDirectory)/s + EXTENSION_NAME: "k8s-configuration" + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' + inputs: + versionSpec: 3.6 + - bash: | + set -ev + echo "Building extension ${EXTENSION_NAME}..." + + # prepare and activate virtualenv + pip install virtualenv + python3 -m venv env/ + source env/bin/activate + + # clone azure-cli + pip install azdev + + ls $(CLI_REPO_PATH) + + azdev --version + azdev setup -r $(CLI_REPO_PATH) -e $(EXTENSION_NAME) + azdev extension build $(EXTENSION_NAME) + workingDirectory: $(CLI_REPO_PATH) + displayName: "Setup and Build Extension with azdev" + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: $(CLI_REPO_PATH)/dist + +- stage: AzureCLIOfficial + displayName: "Azure Official CLI Code Checks" + dependsOn: [] + jobs: + - job: CheckLicenseHeader + displayName: "Check License" + pool: + vmImage: 'ubuntu-16.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' + inputs: + versionSpec: 3.6 + - bash: | + set -ev + + # prepare and activate virtualenv + python -m venv env/ + + chmod +x ./env/bin/activate + source ./env/bin/activate + + # clone azure-cli + git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli + + pip install -q azdev + + azdev setup -c ../azure-cli -r ./ + + azdev --version + az --version + + azdev verify license + + - job: StaticAnalysis + displayName: "Static Analysis" + pool: + vmImage: 'ubuntu-16.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' + inputs: + versionSpec: 3.6 + - bash: pip install wheel==0.30.0 pylint==1.9.5 flake8==3.5.0 requests + displayName: 'Install wheel, pylint, flake8, requests' + - bash: python scripts/ci/source_code_static_analysis.py + displayName: "Static Analysis" + + - job: IndexVerify + displayName: "Verify Extensions Index" + pool: + vmImage: 'ubuntu-16.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.7' + inputs: + versionSpec: 3.7 + - bash: | + #!/usr/bin/env bash + set -ev + pip install wheel==0.30.0 requests packaging + export CI="ADO" + python ./scripts/ci/test_index.py -v + displayName: "Verify Extensions Index" + + - job: SourceTests + displayName: "Integration Tests, Build Tests" + pool: + vmImage: 'ubuntu-16.04' + strategy: + matrix: + Python36: + python.version: '3.6' + Python38: + python.version: '3.8' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + - bash: pip install wheel==0.30.0 + displayName: 'Install wheel==0.30.0' + - bash: ./scripts/ci/test_source.sh + displayName: 'Run integration test and build test' + env: + ADO_PULL_REQUEST_LATEST_COMMIT: $(System.PullRequest.SourceCommitId) + ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) + + - job: LintModifiedExtensions + displayName: "CLI Linter on Modified Extensions" + pool: + vmImage: 'ubuntu-16.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' + inputs: + versionSpec: 3.6 + - bash: | + set -ev + + # prepare and activate virtualenv + pip install virtualenv + python -m virtualenv venv/ + source ./venv/bin/activate + + # clone azure-cli + git clone --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli + + pip install azdev + + azdev --version + + azdev setup -c ../azure-cli -r ./ -e k8s-configuration + + # overwrite the default AZURE_EXTENSION_DIR set by ADO + AZURE_EXTENSION_DIR=~/.azure/cliextensions az --version + + AZURE_EXTENSION_DIR=~/.azure/cliextensions azdev linter --include-whl-extensions k8s-configuration + displayName: "CLI Linter on Modified Extension" + env: + ADO_PULL_REQUEST_LATEST_COMMIT: $(System.PullRequest.SourceCommitId) + ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch)