From 2c1b99f9f767a202c758e7764b281ca9107fb141 Mon Sep 17 00:00:00 2001 From: Shilong Liu Date: Fri, 27 May 2022 12:28:52 +0800 Subject: [PATCH] [ci] Change artifact reference pipeline to common lib pipeline. (#2294) * [ci] Change artifact reference pipeline to common lib pipeline. --- .../build-docker-sonic-vs-template.yml | 24 +++-- .azure-pipelines/build-template.yml | 95 ++++++++----------- .../test-docker-sonic-vs-template.yml | 18 ++-- azure-pipelines.yml | 9 +- 4 files changed, 68 insertions(+), 78 deletions(-) diff --git a/.azure-pipelines/build-docker-sonic-vs-template.yml b/.azure-pipelines/build-docker-sonic-vs-template.yml index b0a6562d3a..ff0ff6c0cb 100644 --- a/.azure-pipelines/build-docker-sonic-vs-template.yml +++ b/.azure-pipelines/build-docker-sonic-vs-template.yml @@ -36,40 +36,47 @@ jobs: inputs: source: specific project: build - pipeline: 9 + pipeline: Azure.sonic-swss-common artifact: ${{ parameters.swss_common_artifact_name }} runVersion: 'latestFromBranch' runBranch: 'refs/heads/$(BUILD_BRANCH)' + path: $(Build.ArtifactStagingDirectory)/download displayName: "Download sonic swss common deb packages" - task: DownloadPipelineArtifact@2 inputs: source: specific project: build - pipeline: 12 + pipeline: Azure.sonic-sairedis artifact: ${{ parameters.sairedis_artifact_name }} runVersion: 'latestFromBranch' runBranch: 'refs/heads/$(BUILD_BRANCH)' + path: $(Build.ArtifactStagingDirectory)/download displayName: "Download sonic sairedis deb packages" - task: DownloadPipelineArtifact@2 inputs: artifact: ${{ parameters.swss_artifact_name }} - displayName: "Download sonic swss artifact" + path: $(Build.ArtifactStagingDirectory)/download + displayName: "Download pre-stage built ${{ parameters.swss_artifact_name }}" - task: DownloadPipelineArtifact@2 inputs: source: specific project: build - pipeline: 1 + pipeline: Azure.sonic-buildimage.official.vs artifact: sonic-buildimage.vs runVersion: 'latestFromBranch' runBranch: 'refs/heads/$(BUILD_BRANCH)' - displayName: "Download sonic buildimage" + path: $(Build.ArtifactStagingDirectory)/download + patterns: '**/target/docker-sonic-vs.gz' + displayName: "Download sonic-buildimage docker-sonic-vs" - script: | + set -ex echo $(Build.DefinitionName).$(Build.BuildNumber) - docker load < ../target/docker-sonic-vs.gz + docker load < $(Build.ArtifactStagingDirectory)/download/target/docker-sonic-vs.gz + mkdir -p .azure-pipelines/docker-sonic-vs/debs - cp -v ../*.deb .azure-pipelines/docker-sonic-vs/debs + cp -v $(Build.ArtifactStagingDirectory)/download/*.deb .azure-pipelines/docker-sonic-vs/debs pushd .azure-pipelines @@ -78,7 +85,8 @@ jobs: popd docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz - + rm -rf $(Build.ArtifactStagingDirectory)/download + displayName: "Build docker-sonic-vs" - publish: $(Build.ArtifactStagingDirectory)/ artifact: ${{ parameters.artifact_name }} displayName: "Archive sonic docker vs image" diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index f6690731b2..f8040bca56 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -23,12 +23,6 @@ parameters: - name: sonic_slave type: string -- name: buildimage_artifact_name - type: string - -- name: buildimage_pipeline - type: number - - name: sairedis_artifact_name type: string @@ -46,6 +40,9 @@ parameters: type: boolean default: false +- name: common_lib_artifact_name + type: string + jobs: - job: displayName: ${{ parameters.arch }} @@ -79,77 +76,63 @@ jobs: inputs: source: specific project: build - pipeline: 9 + pipeline: Azure.sonic-swss-common artifact: ${{ parameters.swss_common_artifact_name }} runVersion: 'latestFromBranch' runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: '$(Build.SourcesDirectory)/${{ parameters.swss_common_artifact_name }}' + path: $(Build.ArtifactStagingDirectory)/download + patterns: | + libswsscommon_1.0.0_${{ parameters.arch }}.deb + libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb displayName: "Download sonic swss common deb packages" - task: DownloadPipelineArtifact@2 inputs: source: specific project: build - pipeline: 12 + pipeline: Azure.sonic-sairedis artifact: ${{ parameters.sairedis_artifact_name }} runVersion: 'latestFromBranch' runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: '$(Build.SourcesDirectory)/${{ parameters.sairedis_artifact_name }}' + path: $(Build.ArtifactStagingDirectory)/download + patterns: | + libsaivs_*.deb + libsaivs-dev_*.deb + libsairedis_*.deb + libsairedis-dev_*.deb + libsaimetadata_*.deb + libsaimetadata-dev_*.deb + syncd-vs_*.deb displayName: "Download sonic sairedis deb packages" - task: DownloadPipelineArtifact@2 - ${{ if eq(parameters.buildimage_pipeline, 141) }}: - continueOnError: True inputs: source: specific project: build - pipeline: ${{ parameters.buildimage_pipeline }} - artifact: ${{ parameters.buildimage_artifact_name }} + pipeline: Azure.sonic-buildimage.common_libs + artifact: ${{ parameters.common_lib_artifact_name }} runVersion: 'latestFromBranch' runBranch: 'refs/heads/$(BUILD_BRANCH)' - path: '$(Build.SourcesDirectory)/${{ parameters.buildimage_artifact_name }}' - displayName: "Download sonic buildimage deb packages" - - script: | - buildimage_artifact_downloaded=n - [ -d "$(Build.SourcesDirectory)/${{ parameters.buildimage_artifact_name }}/target" ] && buildimage_artifact_downloaded=y - echo "buildimage_artifact_downloaded=$buildimage_artifact_downloaded" - echo "##vso[task.setvariable variable=buildimage_artifact_downloaded]$buildimage_artifact_downloaded" - condition: eq(${{ parameters.buildimage_pipeline }}, 141) - displayName: "Check if sonic buildimage deb packages downloaded" - - task: DownloadPipelineArtifact@2 - condition: and(eq(variables.buildimage_artifact_downloaded, 'n'), eq(${{ parameters.buildimage_pipeline }}, 141)) - inputs: - source: specific - project: build - pipeline: ${{ parameters.buildimage_pipeline }} - artifact: 'sonic-buildimage.marvell-armhf1' - runVersion: specific - runId: 80637 - path: '$(Build.SourcesDirectory)/${{ parameters.buildimage_artifact_name }}' - displayName: "Download sonic buildimage deb packages from 80637" + path: $(Build.ArtifactStagingDirectory)/download + patterns: | + target/debs/buster/libnl-3-200_*.deb + target/debs/buster/libnl-3-dev_*.deb + target/debs/buster/libnl-genl-3-200_*.deb + target/debs/buster/libnl-genl-3-dev_*.deb + target/debs/buster/libnl-route-3-200_*.deb + target/debs/buster/libnl-route-3-dev_*.deb + target/debs/buster/libnl-nf-3-200_*.deb + target/debs/buster/libnl-nf-3-dev_*.deb + displayName: "Download common libs" - script: | - cd $(Build.SourcesDirectory)/${{ parameters.buildimage_artifact_name }} - sudo dpkg -i target/debs/buster/libnl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb - sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb - cd $(Build.SourcesDirectory)/${{ parameters.swss_common_artifact_name }} - sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb - sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb - cd $(Build.SourcesDirectory)/${{ parameters.sairedis_artifact_name }} - sudo dpkg -i libsaivs_*.deb - sudo dpkg -i libsaivs-dev_*.deb - sudo dpkg -i libsairedis_*.deb - sudo dpkg -i libsairedis-dev_*.deb - sudo dpkg -i libsaimetadata_*.deb - sudo dpkg -i libsaimetadata-dev_*.deb - sudo dpkg -i syncd-vs_*.deb - workingDirectory: $(Pipeline.Workspace) + set -ex + cd download + sudo dpkg -i $(find target/debs/buster -type f) + sudo dpkg -i $(ls *.deb) + cd .. + rm -rf download + workingDirectory: $(Build.ArtifactStagingDirectory) displayName: "Install libnl3, sonic swss common and sairedis" - script: | - set -x + set -ex tar czf pytest.tgz tests cp -r pytest.tgz $(Build.ArtifactStagingDirectory)/ if [ '${{ parameters.archive_gcov }}' == True ]; then diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml index 2ba42d458b..fc1527f72c 100644 --- a/.azure-pipelines/test-docker-sonic-vs-template.yml +++ b/.azure-pipelines/test-docker-sonic-vs-template.yml @@ -31,25 +31,26 @@ jobs: - task: DownloadPipelineArtifact@2 inputs: artifact: docker-sonic-vs - displayName: "Download docker sonic vs image" - + path: $(Build.ArtifactStagingDirectory)/download + displayName: "Download pre-stage built docker-sonic-vs" - task: DownloadPipelineArtifact@2 inputs: source: specific project: build - pipeline: 9 + pipeline: Azure.sonic-swss-common artifact: sonic-swss-common.amd64.ubuntu20_04 runVersion: 'latestFromBranch' runBranch: 'refs/heads/$(BUILD_BRANCH)' + path: $(Build.ArtifactStagingDirectory)/download displayName: "Download sonic swss common deb packages" - script: | - set -x + set -ex sudo .azure-pipelines/build_and_install_module.sh sudo apt-get install -y libhiredis0.14 - sudo dpkg -i --force-confask,confnew ../libswsscommon_1.0.0_amd64.deb || apt-get install -f - sudo dpkg -i ../python3-swsscommon_1.0.0_amd64.deb + sudo dpkg -i --force-confask,confnew $(Build.ArtifactStagingDirectory)/download/libswsscommon_1.0.0_amd64.deb || apt-get install -f + sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/python3-swsscommon_1.0.0_amd64.deb # install packages for vs test sudo apt-get install -y net-tools bridge-utils vlan @@ -58,8 +59,8 @@ jobs: displayName: "Install dependencies" - script: | - set -x - sudo docker load -i ../docker-sonic-vs.gz + set -ex + sudo docker load -i $(Build.ArtifactStagingDirectory)/download/docker-sonic-vs.gz docker ps ip netns list uname -a @@ -72,6 +73,7 @@ jobs: else sudo py.test -v --force-flaky --junitxml=tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) fi + rm -rf $(Build.ArtifactStagingDirectory)/download displayName: "Run vs tests" - task: PublishTestResults@2 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8dda0580f6..166905654e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -42,8 +42,7 @@ stages: parameters: arch: amd64 sonic_slave: sonic-slave-buster - buildimage_artifact_name: sonic-buildimage.vs - buildimage_pipeline: 142 + common_lib_artifact_name: common-lib swss_common_artifact_name: sonic-swss-common sairedis_artifact_name: sonic-sairedis artifact_name: sonic-swss @@ -60,8 +59,7 @@ stages: timeout: 240 pool: sonicbld-armhf sonic_slave: sonic-slave-buster-armhf - buildimage_artifact_name: sonic-buildimage.marvell-armhf - buildimage_pipeline: 141 + common_lib_artifact_name: common-lib.armhf swss_common_artifact_name: sonic-swss-common.armhf sairedis_artifact_name: sonic-sairedis.armhf artifact_name: sonic-swss.armhf @@ -73,9 +71,8 @@ stages: timeout: 240 pool: sonicbld-arm64 sonic_slave: sonic-slave-buster-arm64 + common_lib_artifact_name: common-lib.arm64 swss_common_artifact_name: sonic-swss-common.arm64 - buildimage_artifact_name: sonic-buildimage.centec-arm64 - buildimage_pipeline: 140 sairedis_artifact_name: sonic-sairedis.arm64 artifact_name: sonic-swss.arm64 archive_gcov: false