Skip to content

Commit

Permalink
improve build workflow files for nightly and release, and report resu…
Browse files Browse the repository at this point in the history
…lts to testmo
  • Loading branch information
dhuang authored and dhuang committed Aug 18, 2023
1 parent 2eabf29 commit 91cca8c
Show file tree
Hide file tree
Showing 5 changed files with 232 additions and 68 deletions.
56 changes: 17 additions & 39 deletions .github/workflows/build-nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,43 +2,21 @@ name: build-nightly
run-name: ${{ github.workflow }} is to create nightly wheel file for pypi
on:
push:
branches:
- 'master'
schedule:
- cron: '0 0 * * *'
workflow_dispatch:
# branches:
# - 'main'
#schedule:
# - cron: '0 0 * * *'
#workflow_dispatch:


jobs:
build-nightly:
runs-on: ubuntu-22.04
permissions:
id-token: write
contents: read
steps:
- uses: aws-actions/configure-aws-credentials@v2
with:
role-to-assume: ${{ secrets.AWS_WEBIDENTITY_FOR_GITHUB_ACTIONS }}
aws-region: us-east-1
- uses: actions/checkout@v3
- run: |
pwd
sudo apt-get install python3-pip
pip3 --version
sudo pip3 install virtualenv
virtualenv venv
source venv/bin/activate
make -B build
deactivate
ls dist/
aws s3 cp dist/*nightly*.whl s3://nm-github-actions/${{ github.event.repository.name }}/
todaytime=`date +%Y%m%d`
date '+%Y%m%d-%k:%M:%S' | tee log_${GITHUB_REF_NAME}_nightly_${todaytime}_${GITHUB_SHA:0:7}
aws s3 cp log_${GITHUB_REF_NAME}_nightly_${todaytime}_${GITHUB_SHA:0:7} s3://nm-github-actions/${{ github.event.repository.name }}/
oldDate=`date --date='-2 month' +%Y%m%d`
oldWhl=`(aws s3 ls s3://nm-github-actions/${{ github.event.repository.name }}/ | grep nightly | grep "${oldDate}") || echo "notfound"`
if [[ "${oldWhl}" != 'notfound' ]]; then
for oldwhl in $(echo "${oldWhl}" | awk '{print $4}')
do
echo "Remove old build ${oldwhl}"
aws s3 rm s3://nm-github-actions/${{ github.event.repository.name }}/${oldwhl}
done
fi

BUILD-SPARSEZOO_NIGHTLY:

uses: ./.github/workflows/util.yml
with:
runs_on: ubuntu-22.04
run_id: ${{ github.run_id }}
build_type: nightly
testmo_project_id: 9
secrets: inherit
44 changes: 44 additions & 0 deletions .github/workflows/build-nightly.yml.bak
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
name: build-nightly
run-name: ${{ github.workflow }} is to create nightly wheel file for pypi
on:
push:
branches:
- 'master'
schedule:
- cron: '0 0 * * *'
workflow_dispatch:
jobs:
build-nightly:
runs-on: ubuntu-22.04
permissions:
id-token: write
contents: read
steps:
- uses: aws-actions/configure-aws-credentials@v2
with:
role-to-assume: ${{ secrets.AWS_WEBIDENTITY_FOR_GITHUB_ACTIONS }}
aws-region: us-east-1
- uses: actions/checkout@v3
- run: |
pwd
sudo apt-get install python3-pip
pip3 --version
sudo pip3 install virtualenv
virtualenv venv
source venv/bin/activate
make -B build
deactivate
ls dist/
aws s3 cp dist/*nightly*.whl s3://nm-github-actions/${{ github.event.repository.name }}/
todaytime=`date +%Y%m%d`
date '+%Y%m%d-%k:%M:%S' | tee log_${GITHUB_REF_NAME}_nightly_${todaytime}_${GITHUB_SHA:0:7}
aws s3 cp log_${GITHUB_REF_NAME}_nightly_${todaytime}_${GITHUB_SHA:0:7} s3://nm-github-actions/${{ github.event.repository.name }}/
oldDate=`date --date='-2 month' +%Y%m%d`
oldWhl=`(aws s3 ls s3://nm-github-actions/${{ github.event.repository.name }}/ | grep nightly | grep "${oldDate}") || echo "notfound"`
if [[ "${oldWhl}" != 'notfound' ]]; then
for oldwhl in $(echo "${oldWhl}" | awk '{print $4}')
do
echo "Remove old build ${oldwhl}"
aws s3 rm s3://nm-github-actions/${{ github.event.repository.name }}/${oldwhl}
done
fi
43 changes: 14 additions & 29 deletions .github/workflows/build-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,33 +2,18 @@ name: build-release
run-name: ${{ github.workflow }} is to create release wheel file for pypi
on:
push:
branches:
- 'release/[0-9]+.[0-9]+'
workflow_dispatch:
# branches:
# - 'release/[0-9]+.[0-9]+'
# workflow_dispatch:

jobs:
build-release:
runs-on: ubuntu-22.04
permissions:
id-token: write
contents: read
steps:
- uses: aws-actions/configure-aws-credentials@v2
with:
role-to-assume: ${{ secrets.AWS_WEBIDENTITY_FOR_GITHUB_ACTIONS }}
aws-region: us-east-1
- uses: actions/checkout@v3
- run: |
pwd
sudo apt-get install python3-pip
pip3 --version
sudo pip3 install virtualenv
virtualenv venv
source venv/bin/activate
sed -i 's/is_release = False/is_release = True/g' utils/neuralmagic/version.py
make -B build
deactivate
ls dist/
aws s3 cp dist/*.whl s3://nm-github-actions/${{ github.event.repository.name }}/
todaytime=`date +%Y%m%d`
date '+%Y%m%d-%k:%M:%S' | tee log_${GITHUB_REF_NAME/\//-}_release_${todaytime}_${GITHUB_SHA:0:7}
aws s3 cp log_${GITHUB_REF_NAME/\//-}_release_${todaytime}_${GITHUB_SHA:0:7} s3://nm-github-actions/${{ github.event.repository.name }}/

BUILD-SPARSEZOO_RELEASE:

uses: ./.github/workflows/util.yml
with:
runs_on: ubuntu-22.04
run_id: ${{ github.run_id }}
build_type: release
testmo_project_id: 9
secrets: inherit
34 changes: 34 additions & 0 deletions .github/workflows/build-release.yml.bak
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
name: build-release
run-name: ${{ github.workflow }} is to create release wheel file for pypi
on:
push:
branches:
- 'release/[0-9]+.[0-9]+'
workflow_dispatch:
jobs:
build-release:
runs-on: ubuntu-22.04
permissions:
id-token: write
contents: read
steps:
- uses: aws-actions/configure-aws-credentials@v2
with:
role-to-assume: ${{ secrets.AWS_WEBIDENTITY_FOR_GITHUB_ACTIONS }}
aws-region: us-east-1
- uses: actions/checkout@v3
- run: |
pwd
sudo apt-get install python3-pip
pip3 --version
sudo pip3 install virtualenv
virtualenv venv
source venv/bin/activate
sed -i 's/is_release = False/is_release = True/g' utils/neuralmagic/version.py
make -B build
deactivate
ls dist/
aws s3 cp dist/*.whl s3://nm-github-actions/${{ github.event.repository.name }}/
todaytime=`date +%Y%m%d`
date '+%Y%m%d-%k:%M:%S' | tee log_${GITHUB_REF_NAME/\//-}_release_${todaytime}_${GITHUB_SHA:0:7}
aws s3 cp log_${GITHUB_REF_NAME/\//-}_release_${todaytime}_${GITHUB_SHA:0:7} s3://nm-github-actions/${{ github.event.repository.name }}/
123 changes: 123 additions & 0 deletions .github/workflows/util.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
name: report-to-testmo
on:
workflow_call:
inputs:
runs_on:
description: "runner label specifying instance running the job"
type: string
required: true
run_id:
description: "run id provided by GHA"
required: true
type: string
build_type:
description: "build type: nightly or release"
type: string
required: true
testmo_project_id:
description: "testmo project id"
type: string
required: true

jobs:

BUILD:
runs-on: ${{ inputs.runs_on }}
outputs:
status: ${{ steps.build.outputs.status }}
commitid: ${{ steps.build.outputs.commitid }}
permissions:
id-token: write
contents: read
steps:

- name: repo checkout
uses: actions/checkout@v3

- name: s3
uses: aws-actions/configure-aws-credentials@v2
with:
role-to-assume: ${{ secrets.AWS_WEBIDENTITY_FOR_GITHUB_ACTIONS }}
aws-region: us-east-1

- name: build
id: build
run: |
pwd
sudo apt-get -y install python3-pip
pip3 --version
sudo pip3 install virtualenv
virtualenv venv
source venv/bin/activate
pip install -e .
if [[ "${{ inputs.build_type }}" = release ]]; then
sed -i 's/is_release = False/is_release = True/g' utils/neuralmagic/version.py
fi
status=$(make -B build || echo 'FAILED')
deactivate
echo "=========== Build log ==========="
echo "${status}"
echo "commitid=${GITHUB_SHA:0:7}" >> "$GITHUB_OUTPUT"
echo "=========== Build status ==========="
if [[ "${status}" = "FAILED" ]]; then
echo "${{ github.event.repository.name }} build failed"
echo "status=failed" >> "$GITHUB_OUTPUT"
exit 1
else
echo "${{ github.event.repository.name }} build success"
fi
echo "=========== Generated build ==========="
ls dist/
echo "=========== Copy build to S3 ==========="
aws s3 cp dist/*.whl s3://nm-github-actions/${{ github.event.repository.name }}/
if [ $? -eq 0 ]; then
echo "ok: copied to s3://nm-github-actions/${{ github.event.repository.name }}/"
echo "status=success" >> "$GITHUB_OUTPUT"
else
echo "failed: copied to s3://nm-github-actions/${{ github.event.repository.name }}/"
echo "status=failed" >> "$GITHUB_OUTPUT"
exit 1
fi
oldDate=`date --date='-2 month' +%Y%m%d`
oldWhl=`(aws s3 ls s3://nm-github-actions/${{ github.event.repository.name }}/ | grep nightly | grep "${oldDate}") || echo "notfound"`
if [[ "${oldWhl}" != 'notfound' ]]; then
for oldwhl in $(echo "${oldWhl}" | awk '{print $4}')
do
echo "Remove old build ${oldwhl} in S3"
aws s3 rm s3://nm-github-actions/${{ github.event.repository.name }}/${oldwhl}
done
fi
TESTMO:
if: success() || failure()
needs: BUILD
runs-on: ${{ inputs.runs_on }}
steps:

- id: report
run: |
echo "node: $(node -v)"
echo "npm: $(npm -v)"
echo "Installing testmo cli..."
sudo npm install -g @testmo/testmo-cli
export TESTMO_TOKEN=${{ secrets.TESTMO_TEST_TOKEN }}
TESTMO_URL="https://neuralmagic.testmo.net"
todaytime=`date +%Y%m%d`
name="${{ github.event.repository.name }} ${{ inputs.build_type }} ${todaytime} ${{ needs.BUILD.outputs.commitid }} RunID:${{ inputs.run_id }}"
echo "========== Build info ==========="
echo "name: ${name}"
echo "build status: ${{ needs.BUILD.outputs.status }}"
echo "========== Report to testmo ==========="
echo "testmo automation:run:submit \\"
echo " --instance ${TESTMO_URL} \\"
echo " --project-id ${{ inputs.testmo_project_id }} \\"
echo " --name ${name} \\"
echo " --source ${{ github.event.repository.name }} \\"
echo " --results result.xml"
testmo automation:run:submit \
--instance "${TESTMO_URL}" \
--project-id ${{ inputs.testmo_project_id }} \
--name "${name}" \
--source "${{ github.event.repository.name }}" \
--results result.xml \
-- cat result.xml

0 comments on commit 91cca8c

Please sign in to comment.