Add GitHub workflows with CUDA and MKL (#700)

* Add GitHub workflows with CUDA * Install MKL on Windows * Upload CPU-only executables from Windows and Ubuntu as artifacts * Clean up workflows
marian-nmt · Aug 21, 2020 · a21e48f · a21e48f
1 parent c4c4b79
commit a21e48f
Show file tree

Hide file tree

Showing 13 changed files with 489 additions and 56 deletions.
diff --git a/.github/workflows/build-windows-2019-cpu.yml b/.github/workflows/build-windows-2019-cpu.yml
diff --git a/.github/workflows/build-macos-10.15-cpu.yml → .github/workflows/macos-cpu.yml b/.github/workflows/build-macos-10.15-cpu.yml → .github/workflows/macos-cpu.yml
@@ -1,4 +1,4 @@
-name: macos-10.5-cpu
+name: macOS CPU-only
 
 on:
   push:

diff --git a/.github/workflows/build-ubuntu-18.04-cpu.yml → .github/workflows/ubuntu-cpu.yml b/.github/workflows/build-ubuntu-18.04-cpu.yml → .github/workflows/ubuntu-cpu.yml
@@ -1,4 +1,4 @@
-name: ubuntu-18.04-cpu
+name: Ubuntu 18.04 CPU-only
 
 on:
   push:
@@ -7,7 +7,7 @@ on:
     branches: [ master ]
 
 jobs:
-  build:
+  build-and-upload:
 
     runs-on: ubuntu-18.04
 
@@ -41,7 +41,8 @@ jobs:
       run: |
         mkdir -p build
         cd build
-        cmake .. -DCOMPILE_CPU=on -DCOMPILE_CUDA=off -DCOMPILE_EXAMPLES=on -DCOMPILE_SERVER=on -DCOMPILE_TESTS=on \
+        cmake .. -DCMAKE_BUILD_TYPE=Release \
+          -DCOMPILE_CPU=on -DCOMPILE_CUDA=off -DCOMPILE_EXAMPLES=on -DCOMPILE_SERVER=on -DCOMPILE_TESTS=on \
           -DUSE_FBGEMM=on -DUSE_SENTENCEPIECE=on \
           -DBOOST_ROOT=$BOOST_ROOT_1_69_0 -DBOOST_INCLUDEDIR=$BOOST_ROOT_1_69_0/include -DBOOST_LIBRARYDIR=$BOOST_ROOT_1_69_0/lib \
           -DBoost_ARCHITECTURE=-x64
@@ -62,3 +63,13 @@ jobs:
         ./marian-scorer --version
         ./spm_encode --version
 
+    - name: Prepare archive
+      working-directory: build
+      run: tar zcvf marian-ubuntu-release-static.tar.gz marian*
+
+    # Marian is built with FBGEMM, so there are some restrictions on what CPUs the executables can be run
+    - name: Upload archive
+      uses: actions/upload-artifact@v2
+      with:
+        name: marian-ubuntu-release-static.tar.gz
+        path: build/marian-ubuntu-release-static.tar.gz
diff --git a/.github/workflows/ubuntu-gpu.yml b/.github/workflows/ubuntu-gpu.yml
@@ -0,0 +1,84 @@
+name: Ubuntu CPU+CUDA
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        include:
+          # Ubuntu 20.04 supports CUDA 11+
+          # Ubuntu 18.04 supports CUDA 10.1+
+          - os: ubuntu-18.04
+            cuda: "10.2"
+            gcc: 8
+          # Ubuntu 16.04 supports CUDA 8+
+          - os: ubuntu-16.04
+            cuda: "10.2"
+            gcc: 7
+          - os: ubuntu-16.04
+            cuda: 9.2
+            gcc: 7
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    # The following packages are already installed on GitHub-hosted runners: build-essential openssl libssl-dev
+    # No need to install libprotobuf{17,10,9v5} on Ubuntu {20,18,16}.04 because it is installed together with libprotobuf-dev
+    - name: Install dependencies
+      run: sudo apt-get install -y libgoogle-perftools-dev libprotobuf-dev protobuf-compiler
+
+    # https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
+    - name: Install MKL
+      run: |
+        wget -qO- "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB" | sudo apt-key add -
+        sudo sh -c "echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list"
+        sudo apt-get update -o Dir::Etc::sourcelist="/etc/apt/sources.list.d/intel-mkl.list"
+        sudo apt-get install -y --no-install-recommends intel-mkl-64bit-2020.0-088
+
+    # The script simplifies installation of different versions of CUDA
+    - name: Install CUDA
+      run: ./scripts/ci/install_cuda_ubuntu.sh ${{ matrix.cuda }}
+
+    # Boost is already installed on GitHub-hosted runners in a non-standard location
+    # https://github.com/actions/virtual-environments/issues/687#issuecomment-610471671
+    - name: Configure CMake
+      run: |
+        mkdir -p build
+        cd build
+        CC=/usr/bin/gcc-${{ matrix.gcc }} CXX=/usr/bin/g++-${{ matrix.gcc }} CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }} \
+        cmake .. \
+          -DCOMPILE_CPU=on -DCOMPILE_CUDA=on -DCOMPILE_EXAMPLES=on -DCOMPILE_SERVER=on -DCOMPILE_TESTS=on \
+          -DUSE_FBGEMM=on -DUSE_SENTENCEPIECE=on \
+          -DBOOST_ROOT=$BOOST_ROOT_1_69_0 -DBOOST_INCLUDEDIR=$BOOST_ROOT_1_69_0/include -DBOOST_LIBRARYDIR=$BOOST_ROOT_1_69_0/lib \
+          -DBoost_ARCHITECTURE=-x64 \
+          -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-${{ matrix.cuda }}
+
+    - name: Compile
+      working-directory: build
+      run: make -j2
+
+    # Unit tests are not run because GitHub-hosted runners do not have GPUs
+    # TODO: add a flag to CMake to compile unit tests only on CPU
+    #- name: Run unit tests
+      #working-directory: build
+      #run: make test
+
+    - name: Print versions
+      working-directory: build
+      run: |
+        ./marian --version
+        ./marian-decoder --version
+        ./marian-scorer --version
+        ./spm_encode --version
+
diff --git a/.github/workflows/windows-cpu.yml b/.github/workflows/windows-cpu.yml
@@ -0,0 +1,66 @@
+name: Windows CPU-only
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build-and-upload:
+
+    runs-on: windows-2019
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Download MKL
+      run: |
+        Invoke-WebRequest -Uri https://romang.blob.core.windows.net/mariandev/ci/mkl-2020.1-windows-static.zip -TimeoutSec 600 -OutFile mkl.zip
+        Expand-Archive -Force mkl.zip ${{ github.workspace }}/mkl
+        # Set MKLROOT environment variables so that CMake can find MKL.
+        # GITHUB_WORKSPACE is an environment variable available on all GitHub-hosted runners
+        echo "::set-env name=MKLROOT::$env:GITHUB_WORKSPACE/mkl"
+      shell: powershell
+
+    - name: Prepare vcpkg
+      uses: lukka/run-vcpkg@v2
+      with:
+        vcpkgArguments: protobuf
+        vcpkgGitCommitId: 6185aa76504a5025f36754324abf307cc776f3da
+        vcpkgDirectory: ${{ github.workspace }}/vcpkg/
+        vcpkgTriplet: x64-windows-static
+
+    # Build with a simplified CMake settings JSON file
+    - name: Run CMake
+      uses: lukka/run-cmake@v2
+      with:
+        buildDirectory: ${{ github.workspace }}/build/
+        cmakeAppendedArgs: -G Ninja
+        cmakeListsOrSettingsJson: CMakeSettingsJson
+        # JSON file must be in the same directory as the main CMakeLists.txt
+        cmakeSettingsJsonPath: ${{ github.workspace }}/_CMakeSettingsCI_CPU.json
+        useVcpkgToolchainFile: true
+
+    - name: Run unit tests
+      working-directory: build/Release/
+      run: ctest
+
+    #- name: Print versions
+      #working-directory: build/Release/
+      #run: |
+        #.\marian.exe --version
+        #.\marian-decoder.exe --version
+        #.\marian-scorer.exe --version
+        #.\spm_encode.exe --version
+      #shell: powershell
+
+    # Marian is built with FBGEMM, so there are some restrictions on what CPUs the executables can be run
+    - name: Upload archive
+      uses: actions/upload-artifact@v2
+      with:
+        name: marian-windows-release-static
+        path: build/Release/marian*.exe
diff --git a/.github/workflows/windows-gpu.yml b/.github/workflows/windows-gpu.yml
@@ -0,0 +1,63 @@
+name: Windows CPU+CUDA
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: windows-2019
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Install CUDA
+      run: |
+        .\scripts\ci\install_cuda_windows.ps1 "10.2"
+        # Set path to CUDA for subsequent steps so that CMake can find it
+        echo "::set-env name=CUDA_PATH::$env:CUDA_PATH"
+        echo "::add-path::$env:CUDA_PATH/bin"
+      shell: powershell
+
+    - name: Download MKL
+      run: |
+        Invoke-WebRequest -Uri https://romang.blob.core.windows.net/mariandev/ci/mkl-2020.1-windows-static.zip -TimeoutSec 600 -OutFile mkl.zip
+        Expand-Archive -Force mkl.zip ${{ github.workspace }}/mkl
+        # Set MKLROOT environment variables so that CMake can find MKL.
+        # GITHUB_WORKSPACE is an environment variable available on all GitHub-hosted runners
+        echo "::set-env name=MKLROOT::$env:GITHUB_WORKSPACE/mkl"
+      shell: powershell
+
+    - name: Prepare vcpkg
+      uses: lukka/run-vcpkg@v2
+      with:
+        vcpkgArguments: protobuf
+        vcpkgGitCommitId: 6185aa76504a5025f36754324abf307cc776f3da
+        vcpkgDirectory: ${{ github.workspace }}/vcpkg/
+        vcpkgTriplet: x64-windows-static
+
+    # Build with a simplified CMake settings JSON file.
+    # On Windows+CUDA we compile with COMPILE_CUDA=on and USE_NCCL=off
+    - name: Run CMake
+      uses: lukka/run-cmake@v2
+      with:
+        buildDirectory: ${{ github.workspace }}/build/
+        cmakeAppendedArgs: -G Ninja
+        cmakeListsOrSettingsJson: CMakeSettingsJson
+        # JSON file must be in the same directory as the main CMakeLists.txt
+        cmakeSettingsJsonPath: ${{ github.workspace }}/_CMakeSettingsCI_GPU.json
+        useVcpkgToolchainFile: true
+
+    - name: Print versions
+      working-directory: build/Debug/
+      run: |
+        .\marian.exe --version
+        .\marian-decoder.exe --version
+        .\marian-scorer.exe --version
+        .\spm_encode.exe --version
diff --git a/_CMakeSettingsCI_CPU.json b/_CMakeSettingsCI_CPU.json
@@ -0,0 +1,28 @@
+{
+  "configurations": [
+    {
+      "name": "Release",
+      "generator": "Ninja",
+      "configurationType": "Release",
+      "inheritEnvironments": [ "msvc_x64" ],
+      "cmakeCommandArgs": "",
+      "buildCommandArgs": "-v",
+      "ctestCommandArgs": "",
+      "variables": [
+        { "name": "OPENSSL_USE_STATIC_LIBS:BOOL", "value": "TRUE" },
+        { "name": "OPENSSL_MSVC_STATIC_RT:BOOL", "value": "TRUE" },
+
+        { "name": "COMPILE_CUDA:BOOL", "value": "FALSE" },
+        { "name": "COMPILE_CPU:BOOL", "value": "TRUE" },
+        { "name": "COMPILE_EXAMPLES:BOOL", "value": "FALSE" },
+        { "name": "COMPILE_SERVER:BOOL", "value": "FALSE" },
+        { "name": "COMPILE_TESTS:BOOL", "value": "TRUE" },
+
+        { "name": "USE_FBGEMM:BOOL", "value": "TRUE" },
+        { "name": "USE_MPI:BOOL", "value": "FALSE" },
+        { "name": "USE_SENTENCEPIECE:BOOL", "value": "TRUE" },
+        { "name": "USE_STATIC_LIBS:BOOL", "value": "TRUE" }
+      ]
+    }
+  ]
+}
diff --git a/CMakeSettingsCI.json → _CMakeSettingsCI_GPU.json b/CMakeSettingsCI.json → _CMakeSettingsCI_GPU.json
@@ -12,14 +12,15 @@
         { "name": "OPENSSL_USE_STATIC_LIBS:BOOL", "value": "TRUE" },
         { "name": "OPENSSL_MSVC_STATIC_RT:BOOL", "value": "TRUE" },
 
-        { "name": "COMPILE_CUDA:BOOL", "value": "FALSE" },
+        { "name": "COMPILE_CUDA:BOOL", "value": "TRUE" },
         { "name": "COMPILE_CPU:BOOL", "value": "TRUE" },
         { "name": "COMPILE_EXAMPLES:BOOL", "value": "FALSE" },
         { "name": "COMPILE_SERVER:BOOL", "value": "FALSE" },
         { "name": "COMPILE_TESTS:BOOL", "value": "TRUE" },
 
         { "name": "USE_FBGEMM:BOOL", "value": "TRUE" },
         { "name": "USE_MPI:BOOL", "value": "FALSE" },
+        { "name": "USE_NCCL:BOOL", "value": "FALSE" },
         { "name": "USE_SENTENCEPIECE:BOOL", "value": "TRUE" },
         { "name": "USE_STATIC_LIBS:BOOL", "value": "TRUE" }
       ]
@@ -36,14 +37,15 @@
         { "name": "OPENSSL_MSVC_STATIC_RT:BOOL", "value": "TRUE" },
         { "name": "OPENSSL_USE_STATIC_LIBS:BOOL", "value": "TRUE" },
 
-        { "name": "COMPILE_CUDA:BOOL", "value": "FALSE" },
+        { "name": "COMPILE_CUDA:BOOL", "value": "TRUE" },
         { "name": "COMPILE_CPU:BOOL", "value": "TRUE" },
         { "name": "COMPILE_EXAMPLES:BOOL", "value": "FALSE" },
         { "name": "COMPILE_SERVER:BOOL", "value": "FALSE" },
         { "name": "COMPILE_TESTS:BOOL", "value": "TRUE" },
 
         { "name": "USE_FBGEMM:BOOL", "value": "TRUE" },
         { "name": "USE_MPI:BOOL", "value": "FALSE" },
+        { "name": "USE_NCCL:BOOL", "value": "FALSE" },
         { "name": "USE_SENTENCEPIECE:BOOL", "value": "TRUE" },
         { "name": "USE_STATIC_LIBS:BOOL", "value": "TRUE" }
       ]

diff --git a/scripts/ci/README.md b/scripts/ci/README.md
@@ -0,0 +1 @@
+Scripts for continuous integration.