[LDC] Set up CI / package generation with GitHub Actions and Cirrus CI

GH Actions: Windows x86 & x64, macOS x64 & ARM64, Linux x64, Android ARMv7-A & AArch64 Cirrus CI: Linux AArch64 2 packages per platform (enabled/disabled assertions), except for cross- compiled Android. Tagged builds are uploaded to the corresponding GitHub release, untagged builds to the GitHub 'CI' release.
ldc-developers · Nov 30, 2023 · be7fee9 · be7fee9
1 parent 5ded841
commit be7fee9
Show file tree

Hide file tree

Showing 7 changed files with 656 additions and 0 deletions.
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -0,0 +1,108 @@
+environment:
+  CIRRUS_CLONE_DEPTH: 50
+  CLANG_VERSION: '15.0.3' # 15.0.6 requires a more recent libstdc++.so.6 than shipped with Ubuntu 20
+  # no MLIR
+  LLVM_PROJECTS_TO_ENABLE: bolt;compiler-rt;lld
+  # keep these in sync with .github/workflows/ldc-release.yml for Linux x64:
+  LLVM_TARGETS_TO_BUILD: all
+  LLVM_EXPERIMENTAL_TARGETS_TO_BUILD: SPIRV
+  BASE_CMAKE_FLAGS: -DLLVM_ENABLE_UNWIND_TABLES=OFF -DLLVM_ENABLE_TERMINFO=OFF -DLLVM_ENABLE_LIBEDIT=OFF -DCOMPILER_RT_INCLUDE_TESTS=OFF -DLLVM_INCLUDE_TESTS=OFF -DCOMPILER_RT_USE_LIBCXX=OFF
+  EXTRA_CMAKE_FLAGS: -DLLVM_BINUTILS_INCDIR=/usr/include -DLLVM_STATIC_LINK_CXX_STDLIB=ON
+
+task:
+  name: Ubuntu 20.04 $TASK_NAME_SUFFIX
+  compute_engine_instance:
+    image_project: ubuntu-os-cloud
+    image: family/ubuntu-2004-lts-arm64
+    platform: linux
+    architecture: arm64
+    cpu: 4
+    memory: 8G
+    disk: 20
+  timeout_in: 120m
+  environment:
+    PARALLELISM: 4
+    GITHUB_TOKEN: ENCRYPTED[79b00f4f535df0fdfe3ceea87f9d4ec2ba7628e0c60730b99bcd91c7888af275409f02b2f1bc90015c2cb9cc4fb2dd12]
+    matrix:
+      - TASK_NAME_SUFFIX: aarch64
+        LLVM_ENABLE_ASSERTIONS: 'OFF'
+      - TASK_NAME_SUFFIX: aarch64 withAsserts
+        LLVM_ENABLE_ASSERTIONS: 'ON'
+  install_prerequisites_script: |
+    export DEBIAN_FRONTEND=noninteractive
+    apt-get -q update
+    apt-get -yq install git-core curl xz-utils ninja-build g++ python3 pkg-config binutils-dev libxml2-dev zlib1g-dev p7zip-full
+    # install a more recent version of CMake
+    curl -fL --retry 3 --max-time 300 -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-linux-aarch64.tar.gz
+    mkdir cmake_bin
+    tar -xf cmake.tar.gz --strip 1 -C cmake_bin
+    rm cmake.tar.gz
+    # download & extract clang
+    curl -fL --retry 3 --max-time 300 -o clang.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-$CLANG_VERSION/clang+llvm-$CLANG_VERSION-aarch64-linux-gnu.tar.xz
+    mkdir clang_bin
+    tar -xf clang.tar.xz --strip 1 -C clang_bin
+    rm clang.tar.xz
+    # make lld the default linker
+    ln -sf "$PWD/clang_bin/bin/ld.lld" /usr/bin/ld
+    ld --version
+  clone_submodules_script: |
+    git submodule update --init --depth $CIRRUS_CLONE_DEPTH
+  build_script: |
+    export PATH="$PWD/cmake_bin/bin:$PATH"
+    export CC="$PWD/clang_bin/bin/clang"
+    export CXX="$PWD/clang_bin/bin/clang++"
+    nproc
+    free
+    cmake --version
+    ninja --version
+    mkdir build
+    cd build
+    cmake -G Ninja ../llvm \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DCMAKE_INSTALL_PREFIX=$PWD/../install \
+      -DLLVM_ENABLE_PROJECTS="$LLVM_PROJECTS_TO_ENABLE" \
+      -DLLVM_TARGETS_TO_BUILD="$LLVM_TARGETS_TO_BUILD" \
+      -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD="$LLVM_EXPERIMENTAL_TARGETS_TO_BUILD" \
+      -DLLVM_ENABLE_ASSERTIONS=$LLVM_ENABLE_ASSERTIONS \
+      $BASE_CMAKE_FLAGS $EXTRA_CMAKE_FLAGS
+    ninja -j$PARALLELISM install
+    cd ..
+  pack_artifact_script: |
+    if [[ "${CIRRUS_TAG:-}" == ldc-v* ]]; then
+      artifactID=${CIRRUS_TAG:5}
+    else
+      artifactID=${CIRRUS_CHANGE_IN_REPO:0:8}
+    fi
+    assertsSuffix=""
+    if [[ "$LLVM_ENABLE_ASSERTIONS" == "ON" ]]; then
+      assertsSuffix="-withAsserts"
+    fi
+    artifactName=llvm-$artifactID-linux-aarch64$assertsSuffix
+    mv install $artifactName
+    chmod -R go=rX $artifactName
+    artifact=$artifactName.tar.xz
+    tar -cf - --owner=0 --group=0 $artifactName | 7za a $artifact -si -txz -mx9 -mmt$PARALLELISM
+    ls -lh $artifact
+  # Upload to GitHub release (only for branches and tags, no PRs)
+  upload_to_github_script: |
+    if [[ "${CIRRUS_TAG:-}" == ldc-v* ]]; then
+      releaseTag=$CIRRUS_TAG
+    elif [[ "${CIRRUS_TAG:-}" == "" && "${CIRRUS_PR:-}" == "" ]]; then
+      releaseTag=CI
+    else
+      echo 'Skipping upload'
+      exit 0
+    fi
+    releaseID="$(set -eo pipefail; curl -fsS https://api.github.com/repos/ldc-developers/llvm-project/releases/tags/"$releaseTag" | grep '^  "id":' | head -n1 || echo "<error>")"
+    if [[ "$releaseID" == "<error>" ]]; then
+      echo "Error: no GitHub release found for tag '$releaseTag'" >&2
+      exit 1
+    fi
+    releaseID=${releaseID:8:-1}
+    artifact=$(ls llvm-*.tar.xz)
+    echo "Uploading $artifact to GitHub release $releaseTag ($releaseID)..."
+    curl -fsS \
+      -H "Authorization: token $GITHUB_TOKEN" \
+      -H "Content-Type: application/octet-stream" \
+      --data-binary @$artifact \
+      https://uploads.github.com/repos/ldc-developers/llvm-project/releases/$releaseID/assets?name=$artifact
diff --git a/.github/actions/1-setup/action.yml b/.github/actions/1-setup/action.yml
@@ -0,0 +1,73 @@
+name: Install prerequisites
+inputs:
+  clang_version:
+    required: true
+  arch:
+    required: true
+runs:
+  using: composite
+  steps:
+
+    - name: 'Linux: Install required apt packages'
+      if: runner.os == 'Linux'
+      shell: bash
+      run: |
+        set -eux
+        export DEBIAN_FRONTEND=noninteractive
+        sudo apt-get -q update
+        sudo apt-get -yq install \
+          git-core cmake g++ binutils-dev python p7zip-full unzip
+
+    - name: 'Linux: Download & extract clang' # into ../clang
+      if: runner.os == 'Linux'
+      shell: bash
+      run: |
+        set -eux
+        cd ..
+        version='${{ inputs.clang_version }}'
+        curl -fL --retry 3 --max-time 300 -o clang.tar.xz \
+          https://github.com/llvm/llvm-project/releases/download/llvmorg-$version/clang+llvm-$version-x86_64-linux-gnu-ubuntu-18.04.tar.xz
+        mkdir clang
+        tar -xf clang.tar.xz --strip 1 -C clang
+        rm clang.tar.xz
+        clang/bin/clang --version
+
+        # use it as C(++) compiler for future steps
+        echo "CC=$PWD/clang/bin/clang" >> $GITHUB_ENV
+        echo "CXX=$PWD/clang/bin/clang++" >> $GITHUB_ENV
+
+        # make bundled lld the default linker
+        sudo ln -sf "$PWD/clang/bin/ld.lld" /usr/bin/ld
+        ld --version
+    - name: 'Windows: Install clang'
+      if: runner.os == 'Windows'
+      shell: bash
+      run: |
+        set -eux
+        cd ..
+        curl -fL --retry 3 --max-time 300 -o clang.exe \
+          https://github.com/llvm/llvm-project/releases/download/llvmorg-${{ inputs.clang_version }}/LLVM-${{ inputs.clang_version }}-win64.exe
+        ./clang.exe //S # double-slash for bash
+        rm clang.exe
+
+        # C:\Program Files\LLVM\bin should already be in PATH
+        clang-cl --version
+
+        # use it as C(++) compiler for future steps
+        echo "CC=clang-cl" >> $GITHUB_ENV
+        echo "CXX=clang-cl" >> $GITHUB_ENV
+
+        if [[ '${{ inputs.arch }}' == x86 ]]; then
+          # make CMake configure 64-bit clang-cl for 32-bit code emission
+          echo "CFLAGS=-m32" >> $GITHUB_ENV
+          echo "CXXFLAGS=-m32" >> $GITHUB_ENV
+          echo "ASMFLAGS=-m32" >> $GITHUB_ENV
+        fi
+
+    - name: Install ninja
+      uses: seanmiddleditch/gha-setup-ninja@v3
+
+    - name: 'Windows: Set LDC_VSDIR env variable'
+      if: runner.os == 'Windows'
+      shell: bash
+      run: echo "LDC_VSDIR=$(vswhere -latest -property installationPath)" >> $GITHUB_ENV
diff --git a/.github/actions/2-build/action.yml b/.github/actions/2-build/action.yml
@@ -0,0 +1,62 @@
+name: Build & install LLVM incl. LLD, compiler-rt, MLIR, BOLT
+inputs:
+  arch:
+    required: false # Windows only
+  enable_projects:
+    required: true
+  targets_to_build:
+    required: false
+    default: all
+  experimental_targets_to_build:
+    required: false
+    default: ''
+  with_asserts:
+    required: true
+  extra_cmake_flags:
+    required: false
+    default: ''
+runs:
+  using: composite
+  steps:
+
+    - if: runner.os != 'Windows'
+      shell: bash
+      run: |
+        set -eux
+        cd ..
+        cmake --version
+        ninja --version
+        mkdir build
+        cd build
+        cmake -G Ninja ../llvm-project/llvm \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DCMAKE_INSTALL_PREFIX=$PWD/../install \
+          -DLLVM_ENABLE_PROJECTS="${{ inputs.enable_projects }}" \
+          -DLLVM_TARGETS_TO_BUILD="${{ inputs.targets_to_build }}" \
+          -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD="${{ inputs.experimental_targets_to_build }}" \
+          -DLLVM_ENABLE_ASSERTIONS=${{ inputs.with_asserts == 'true' && 'ON' || 'OFF' }} \
+          ${{ inputs.extra_cmake_flags }}
+        ninja install
+
+    # Windows: invoke CMake & ninja in MSVC env
+    - if: runner.os == 'Windows'
+      shell: cmd
+      run: |
+        call "%LDC_VSDIR%\Common7\Tools\VsDevCmd.bat" -arch=${{ inputs.arch }} || exit /b
+        echo on
+        cd .. || exit /b
+        cmake --version || exit /b
+        ninja --version || exit /b
+        mkdir build || exit /b
+        cd build || exit /b
+        cmake -G Ninja ..\llvm-project\llvm ^
+          -DCMAKE_BUILD_TYPE=Release ^
+          "-DCMAKE_INSTALL_PREFIX=%CD%\..\install" ^
+          "-DLLVM_ENABLE_PROJECTS=${{ inputs.enable_projects }}" ^
+          "-DLLVM_TARGETS_TO_BUILD=${{ inputs.targets_to_build }}" ^
+          "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=${{ inputs.experimental_targets_to_build }}" ^
+          -DLLVM_ENABLE_ASSERTIONS=${{ inputs.with_asserts == 'true' && 'ON' || 'OFF' }} ^
+          ${{ inputs.extra_cmake_flags }}
+        if %errorlevel% neq 0 exit /b %errorlevel%
+
+        ninja install || exit /b
diff --git a/.github/actions/3-package/action.yml b/.github/actions/3-package/action.yml
@@ -0,0 +1,93 @@
+name: Create package & upload artifact(s)
+inputs:
+  arch:
+    required: true
+  os:
+    required: false
+    default: '' # native
+  with_asserts:
+    required: true
+runs:
+  using: composite
+  steps:
+
+    - name: Pack installation dir
+      shell: bash
+      run: |
+        set -euxo pipefail
+        cd ..
+        mkdir artifacts
+
+        if [[ '${{ github.ref }}' = refs/tags/ldc-v* ]]; then
+          artifactID='${{ github.ref }}'
+          artifactID="${artifactID:15}"
+        else
+          artifactID='${{ github.sha }}'
+          artifactID="${artifactID:0:8}"
+        fi
+
+        os='${{ inputs.os }}'
+        if [[ "$os" == '' ]]; then
+          if [[ '${{ runner.os }}' == Linux ]]; then
+            os=linux
+          elif [[ '${{ runner.os }}' == macOS ]]; then
+            os=osx
+          elif [[ '${{ runner.os }}' == Windows ]]; then
+            os=windows
+          else
+            echo "Error: unknown OS '${{ runner.os }}'"
+            exit 1
+          fi
+        fi
+
+        if [[ "$os" == windows ]]; then
+          # on Windows, the lld symlinks are copies
+          rm install/bin/{ld.lld,ld64.lld*,lld-link,wasm-ld}.exe
+          mv install/bin/lld.exe install/bin/lld-link.exe
+        fi
+
+        assertsSuffix="${{ inputs.with_asserts == 'true' && '-withAsserts' || '' }}"
+
+        artifactName="llvm-$artifactID-$os-${{ inputs.arch }}$assertsSuffix"
+        mv install $artifactName
+        if [[ '${{ runner.os }}' == Windows ]]; then
+          cd $artifactName
+          7z a -mx=9 ../artifacts/$artifactName.7z * >/dev/null
+          cd ..
+        else
+          chmod -R go=rX $artifactName
+          if [[ '${{ runner.os }}' == macOS ]]; then
+            sudo chown -R root:wheel $artifactName
+            tar -cJf artifacts/$artifactName.tar.xz --options='compression-level=9' $artifactName
+          else
+            tar -cf - --owner=0 --group=0 $artifactName | 7za a artifacts/$artifactName.tar.xz -si -txz -mx9
+          fi
+        fi
+
+        # export ARTIFACT_{ID,NAME}
+        echo "ARTIFACT_ID=$artifactID" >> $GITHUB_ENV
+        echo "ARTIFACT_NAME=$os-${{ inputs.arch }}$assertsSuffix" >> $GITHUB_ENV
+
+    - name: 'Linux: Pack source dir'
+      if: runner.os == 'Linux' && inputs.os == '' && inputs.with_asserts == 'false'
+      shell: bash
+      run: |
+        set -euxo pipefail
+        git clean -dffx
+        git submodule foreach git clean -dffx
+        artifactName="llvm-$ARTIFACT_ID.src"
+        # just the LLVM src dir + bolt + compiler-rt + lld + mlir
+        mv compiler-rt llvm/projects
+        mv bolt lld mlir llvm/tools
+        chmod -R go=rX llvm
+        tar -cf - --exclude-vcs --owner=0 --group=0 --transform="s,^llvm,$artifactName," llvm | 7za a ../artifacts/$artifactName.tar.xz -si -txz -mx9
+
+    - name: 'Move artifacts dir for uploading'
+      shell: bash
+      run: mv ../artifacts ./
+
+    - name: Upload artifact(s)
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ env.ARTIFACT_NAME }}
+        path: artifacts/