From f7ce771baa6784ef7e0618a5d3d038513e1169f7 Mon Sep 17 00:00:00 2001 From: Sandeep Datta Date: Thu, 8 Aug 2024 14:04:55 -0700 Subject: [PATCH] Removed sccache. --- .github/actions/build/action.yml.j2 | 8 -- continuous_integration/scripts/setup-utils | 79 +------------ continuous_integration/scripts/vault-s3-init | 111 ------------------- 3 files changed, 2 insertions(+), 196 deletions(-) delete mode 100755 continuous_integration/scripts/vault-s3-init diff --git a/.github/actions/build/action.yml.j2 b/.github/actions/build/action.yml.j2 index 698b7a4..4bb640e 100644 --- a/.github/actions/build/action.yml.j2 +++ b/.github/actions/build/action.yml.j2 @@ -53,14 +53,6 @@ runs: ls -lahR ${{ env.ARTIFACTS_DIR }} <% endif %> - - if: github.repository_owner == 'nvidia' - name: Get AWS credentials for sccache bucket - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: us-east-2 - role-duration-seconds: 28800 # 8 hours - role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-nv-legate - - if: ${{ inputs.use-container }} name: Build (in container) shell: bash --noprofile --norc -xeuo pipefail {0} diff --git a/continuous_integration/scripts/setup-utils b/continuous_integration/scripts/setup-utils index 13e5233..06ae51c 100755 --- a/continuous_integration/scripts/setup-utils +++ b/continuous_integration/scripts/setup-utils @@ -27,7 +27,7 @@ install_darwin_tools() { export READLINK=greadlink brew update - brew install cmake coreutils git gnu-getopt gnu-sed jq ninja wget sccache + brew install cmake coreutils git gnu-getopt gnu-sed jq ninja wget install_darwin_mamba; } @@ -79,47 +79,6 @@ install_from_apt() { apt-get -q install -y wget curl jq sudo ninja-build vim numactl rsync } -install_sccache_linux() { - set -xeuo pipefail - - ARCH=$(uname -m) - - if [ "$ARCH" == "x86_64" ]; then - SCCACHE_ARCH="x86_64-unknown-linux-musl" - elif [ "$ARCH" == "aarch64" ]; then - SCCACHE_ARCH="aarch64-unknown-linux-musl" - else - echo "Unsupported architecture: $ARCH" - exit 1 - fi - - SCCACHE_VERSION="v0.8.1" - SCCACHE_URL="https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}.tar.gz" - - wget -q "$SCCACHE_URL" - tar -xf "sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}.tar.gz" - sudo mv "sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}/sccache" /usr/bin/sccache -} - - -maybe_install_sccache_linux() { - set -xeuo pipefail - - if ! command -v sccache &> /dev/null; then - echo "sccache not found, proceeding with installation." - install_sccache_linux - else - sccache_version=$(sccache --version 2>&1 | awk '/sccache/ {print $2}') - if [[ -z "$sccache_version" ]] || ! version_greater_equal "$sccache_version" "0.5.4"; then - echo "sccache version less than 0.5.4, proceeding with installation." - install_sccache_linux - else - echo "sccache version is 0.5.4 or greater, no need to install." - fi - fi -} - - install_cmake() { set -xeuo pipefail @@ -143,7 +102,6 @@ install_linux_tools() { export READLINK=readlink install_from_apt; - maybe_install_sccache_linux; install_cmake; mkdir -p /tmp/out /tmp/env_yaml @@ -272,39 +230,6 @@ setup_build_env() { prep_git; } -sccache_stop_server_and_show_stats() { - set -xeuo pipefail - sccache --stop-server || true && sccache --show-stats; -} - -init_sccache() { - set -xeuo pipefail - - export SCCACHE_REGION="us-east-2" - export SCCACHE_BUCKET="rapids-sccache-east" - export SCCACHE_S3_KEY_PREFIX=legate-cunumeric-dev - export VAULT_HOST=https://vault.ops.k8s.rapids.ai - CMAKE_C_COMPILER_LAUNCHER=$(which sccache) - export CMAKE_C_COMPILER_LAUNCHER - export CMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER} - export CMAKE_CUDA_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER} - - echo AWS_REGION="${AWS_REGION:-}" - echo AWS_SESSION_TOKEN="${AWS_SESSION_TOKEN:-}" - echo AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-}" - echo AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-}" - - mkdir -p ~/.cache; - - local secrets_dir="$REPO_DIR/.creds" - - if [ -d "$secrets_dir" ] && [ "$(ls -A "$secrets_dir")" ]; then - vault-s3-init; - else - sccache_stop_server_and_show_stats - fi -} - init_build_env() { set -x; @@ -319,7 +244,7 @@ init_build_env() { cd "$PREBUILD_DIR" setup_build_env; - init_sccache; + cd "$REPO_DIR"; if [[ -d "${BUILD_DIR}" ]]; then diff --git a/continuous_integration/scripts/vault-s3-init b/continuous_integration/scripts/vault-s3-init deleted file mode 100755 index 1209bd6..0000000 --- a/continuous_integration/scripts/vault-s3-init +++ /dev/null @@ -1,111 +0,0 @@ -#! /usr/bin/env bash - -set -xeuo pipefail; - -get_vault_token() { - set -eo pipefail - local VAULT_HOST="$1"; - local user_org="$2"; - local gh_token="$3"; - - local vault_token=null; - - vault_token="$( \ - curl -s \ - -X POST \ - -H "Content-Type: application/json" \ - -d "{\"token\": \"$gh_token\"}" \ - "$VAULT_HOST/v1/auth/github-${user_org}/login" \ - | jq -r '.auth.client_token' \ - )"; - - echo "vault_token='$vault_token'"; -} - -vault_s3_init() { - set -eo pipefail - # Attempt to retrieve temporary AWS credentials from a vault - # instance using GitHub OAuth. - - eval "export $(find "$REPO_DIR/.creds" -type f -exec bash -c 'echo $(basename $0)=$(<$0)' {} \;)"; - - if [[ -z "${VAULT_HOST:-}" ]]; then return; fi - if [[ -z "${SCCACHE_BUCKET:-}" ]]; then return; fi - if [[ -z "${GH_TOKEN:-}" ]]; then return; fi - - echo "" - echo "Attempting to use your GitHub account to authenticate"; - echo "with vault at '${VAULT_HOST}'."; - echo "" - - local vault_token=null; - local user_orgs=nv-legate; - - # Attempt to authenticate with GitHub - eval "$(get_vault_token "${VAULT_HOST}" ${user_orgs} "$GH_TOKEN")"; - - if [[ "${vault_token:-null}" == null ]]; then - echo "Your GitHub user was not recognized by vault. Exiting." >&2; - return; - fi - - echo "Successfully authenticated with vault!"; - - local ttl="${VAULT_S3_TTL:-"43200s"}"; - local uri="${VAULT_S3_URI:-"v1/aws/creds/devs"}"; - - # Generate temporary AWS creds - aws_creds="$( \ - curl -s \ - -X GET \ - -H "X-Vault-Token: $vault_token" \ - -H "Content-Type: application/json" \ - "${VAULT_HOST}/$uri?ttl=$ttl" \ - | jq -r '.data' \ - )"; - - AWS_ACCESS_KEY_ID="$(echo "$aws_creds" | jq -r '.access_key')"; - export AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY="$(echo "$aws_creds" | jq -r '.secret_key')"; - export AWS_SECRET_ACCESS_KEY - - if [[ "${AWS_ACCESS_KEY_ID:-null}" == null ]]; then - echo "Failed to generate temporary AWS S3 credentials. Exiting." >&2; - return; - fi - - if [[ "${AWS_SECRET_ACCESS_KEY:-null}" == null ]]; then - echo "Failed to generate temporary AWS S3 credentials. Exiting." >&2; - return; - fi - - # Generate AWS config files - mkdir -p ~/.aws; - - date '+%s' > ~/.aws/stamp; - - cat < ~/.aws/config -[default] -${SCCACHE_BUCKET:+bucket=$SCCACHE_BUCKET} -${SCCACHE_REGION:+region=$SCCACHE_REGION} -EOF - - cat < ~/.aws/credentials -[default] -aws_access_key_id=$AWS_ACCESS_KEY_ID -aws_secret_access_key=$AWS_SECRET_ACCESS_KEY -EOF - - chmod 0600 ~/.aws/{config,credentials}; - - echo "Successfully generated temporary AWS S3 credentials!"; - - # Stop server and reset sccache stats. - sccache --stop-server || true - - # Wait for AWS credentials to propagate - sleep 10 - sccache --show-stats -} - -(vault_s3_init "$@");