diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 4bc4d8a4d92..208ba0b3555 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -121,7 +121,8 @@ jobs: - name: Install depot tools shell: bash run: | - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git + git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git + touch depot_tools/.disable_auto_update echo "${GITHUB_WORKSPACE}/depot_tools" >> $GITHUB_PATH - name: Setup gclient diff --git a/.github/workflows/custom-actions/build-packager/action.yaml b/.github/workflows/custom-actions/build-packager/action.yaml index 6c20a74344b..0556ae86b13 100644 --- a/.github/workflows/custom-actions/build-packager/action.yaml +++ b/.github/workflows/custom-actions/build-packager/action.yaml @@ -52,12 +52,30 @@ runs: echo "::endgroup::" fi + - name: Force Python 2 to support ancient build system (non-Linux only) + if: runner.os != 'Linux' + uses: actions/setup-python@v2 + with: + python-version: '2.x' + + - name: Force Python 2 to support ancient build system (Linux only) + if: runner.os == 'Linux' + shell: bash + run: | + echo "::group::Install python2" + sudo apt install -y python2 + sudo ln -sf python2 /usr/bin/python + echo "::endgroup::" + - name: Install depot tools shell: bash run: | echo "::group::Install depot_tools" - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git + git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git + touch depot_tools/.disable_auto_update echo "${GITHUB_WORKSPACE}/depot_tools" >> $GITHUB_PATH + # Bypass VPYTHON included by depot_tools. Prefer the system installation. + echo "VPYTHON_BYPASS=manually managed python not supported by chrome operations" >> $GITHUB_ENV echo "::endgroup::" - name: Build ninja (arm only) diff --git a/.github/workflows/github_release.yaml b/.github/workflows/github_release.yaml index 05faa4267ef..0d018670c6c 100644 --- a/.github/workflows/github_release.yaml +++ b/.github/workflows/github_release.yaml @@ -200,7 +200,8 @@ jobs: - name: Install depot tools shell: bash run: | - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git + git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git + touch depot_tools/.disable_auto_update echo "${GITHUB_WORKSPACE}/depot_tools" >> $GITHUB_PATH - name: Setup gclient diff --git a/Dockerfile b/Dockerfile index 59be6b0e5ab..7a395a5030e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,14 +3,21 @@ FROM alpine:3.11 as builder # Install utilities, libraries, and dev tools. RUN apk add --no-cache \ bash curl \ - bsd-compat-headers linux-headers \ + bsd-compat-headers c-ares-dev linux-headers \ build-base git ninja python2 python3 +# Default to python2 because our build system is ancient. +RUN ln -sf python2 /usr/bin/python + # Install depot_tools. WORKDIR / -RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN touch depot_tools/.disable_auto_update ENV PATH $PATH:/depot_tools +# Bypass VPYTHON included by depot_tools. Prefer the system installation. +ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" + # Alpine uses musl which does not have mallinfo defined in malloc.h. Define the # structure to workaround a Chromium base bug. RUN sed -i \ @@ -18,9 +25,6 @@ RUN sed -i \ /usr/include/malloc.h ENV GYP_DEFINES='musl=1' -# Bypass VPYTHON included by depot_tools, which no longer works in Alpine. -ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" - # Build shaka-packager from the current directory, rather than what has been # merged. WORKDIR shaka_packager diff --git a/docs/source/build_instructions.md b/docs/source/build_instructions.md index 340a668a05e..416c0184963 100644 --- a/docs/source/build_instructions.md +++ b/docs/source/build_instructions.md @@ -56,14 +56,18 @@ GYP_MSVS_OVERRIDE_PATH="C:/Program Files (x86)/Microsoft Visual Studio/2019/Comm ## Install `depot_tools` -### Linux and Mac - -Clone the `depot_tools` repository from Chromium: +Clone a particular branch of the `depot_tools` repository from Chromium: ```shell -git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +touch depot_tools/.disable_auto_update ``` +The latest version of depot_tools will not work, so please use that branch! + + +### Linux and Mac + Add `depot_tools` to the end of your PATH (you will probably want to put this in your `~/.bashrc` or `~/.zshrc`). Assuming you cloned `depot_tools` to `/path/to/depot_tools`: @@ -74,17 +78,8 @@ export PATH="$PATH:/path/to/depot_tools" ### Windows -Download the -[depot_tools bundle](https://storage.googleapis.com/chrome-infra/depot_tools.zip) -and extract it somewhere. - -**WARNING: DO NOT** use drag-n-drop or copy-n-paste extract from Explorer, -this will not extract the hidden “.git” folder which is necessary for -depot_tools to autoupdate itself. You can use “Extract all…” from the context -menu though. - Add depot_tools to the start of your PATH (must be ahead of any installs of -Python). Assuming you unzipped the bundle to C:\src\depot_tools, open: +Python). Assuming you cloned the repo to C:\src\depot_tools, open: Control Panel → System and Security → System → Advanced system settings diff --git a/packager/testing/dockers/Alpine_Dockerfile b/packager/testing/dockers/Alpine_Dockerfile index 2ce7850a499..df32f525cfd 100644 --- a/packager/testing/dockers/Alpine_Dockerfile +++ b/packager/testing/dockers/Alpine_Dockerfile @@ -6,11 +6,18 @@ RUN apk add --no-cache \ bsd-compat-headers c-ares-dev linux-headers \ build-base git ninja python2 python3 +# Default to python2 because our build system is ancient. +RUN ln -sf python2 /usr/bin/python + # Install depot_tools. WORKDIR / -RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN touch depot_tools/.disable_auto_update ENV PATH $PATH:/depot_tools +# Bypass VPYTHON included by depot_tools. Prefer the system installation. +ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" + # Alpine uses musl which does not have mallinfo defined in malloc.h. Define the # structure to workaround a Chromium base bug. RUN sed -i \ @@ -18,8 +25,5 @@ RUN sed -i \ /usr/include/malloc.h ENV GYP_DEFINES='musl=1' -# Bypass VPYTHON included by depot_tools, which no longer works in Alpine. -ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" - # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/ArchLinux_Dockerfile b/packager/testing/dockers/ArchLinux_Dockerfile index e5065e9b73f..1b0435687e8 100644 --- a/packager/testing/dockers/ArchLinux_Dockerfile +++ b/packager/testing/dockers/ArchLinux_Dockerfile @@ -6,10 +6,17 @@ RUN pacman -Sy --needed --noconfirm \ c-ares \ gcc git python2 python3 +# Default to python2 because our build system is ancient. +RUN ln -sf python2 /usr/bin/python + # Install depot_tools. WORKDIR / -RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN touch depot_tools/.disable_auto_update ENV PATH /depot_tools:$PATH +# Bypass VPYTHON included by depot_tools. Prefer the system installation. +ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" + # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/CentOS_Dockerfile b/packager/testing/dockers/CentOS_Dockerfile index 327581d3ecb..e2c604b04bd 100644 --- a/packager/testing/dockers/CentOS_Dockerfile +++ b/packager/testing/dockers/CentOS_Dockerfile @@ -1,18 +1,28 @@ FROM centos:8 +# Fix up CentOS repo info, which is outdated and not maintained in DockerHub. +# See https://stackoverflow.com/a/71309215 +RUN cd /etc/yum.repos.d/ +RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* +RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + # Install utilities, libraries, and dev tools. RUN yum install -y \ which \ c-ares-devel libatomic \ gcc-c++ git python2 python3 -# Default to python3. -RUN alternatives --set python /usr/bin/python3 +# Default to python2 because our build system is ancient. +RUN ln -sf python2 /usr/bin/python # Install depot_tools. WORKDIR / -RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN touch depot_tools/.disable_auto_update ENV PATH /depot_tools:$PATH +# Bypass VPYTHON included by depot_tools. Prefer the system installation. +ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" + # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/Debian_Dockerfile b/packager/testing/dockers/Debian_Dockerfile index cba5b5e31d2..12a271740e1 100644 --- a/packager/testing/dockers/Debian_Dockerfile +++ b/packager/testing/dockers/Debian_Dockerfile @@ -7,9 +7,16 @@ RUN apt-get install -y \ libc-ares-dev \ build-essential git python python3 +# Default to python2 because our build system is ancient. +RUN ln -sf python2 /usr/bin/python + # Install depot_tools. -RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN touch depot_tools/.disable_auto_update ENV PATH /depot_tools:$PATH +# Bypass VPYTHON included by depot_tools. Prefer the system installation. +ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" + # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/Fedora_Dockerfile b/packager/testing/dockers/Fedora_Dockerfile index 63319f71c5c..f4edfb531e5 100644 --- a/packager/testing/dockers/Fedora_Dockerfile +++ b/packager/testing/dockers/Fedora_Dockerfile @@ -6,13 +6,17 @@ RUN yum install -y \ c-ares-devel libatomic \ gcc-c++ git python2 -# Default to python3. -RUN alternatives --install /usr/bin/python python /usr/bin/python3 3 +# Default to python2 because our build system is ancient. +RUN ln -sf python2 /usr/bin/python # Install depot_tools. WORKDIR / -RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN touch depot_tools/.disable_auto_update ENV PATH /depot_tools:$PATH +# Bypass VPYTHON included by depot_tools. Prefer the system installation. +ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" + # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/OpenSUSE_Dockerfile b/packager/testing/dockers/OpenSUSE_Dockerfile index 8059a1f920c..99cb654df3e 100644 --- a/packager/testing/dockers/OpenSUSE_Dockerfile +++ b/packager/testing/dockers/OpenSUSE_Dockerfile @@ -6,10 +6,17 @@ RUN zypper in -y \ c-ares-devel \ gcc-c++ git python python3 +# Default to python2 because our build system is ancient. +RUN ln -sf python2 /usr/bin/python + # Install depot_tools. WORKDIR / -RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN touch depot_tools/.disable_auto_update ENV PATH /depot_tools:$PATH +# Bypass VPYTHON included by depot_tools. Prefer the system installation. +ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" + # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/Ubuntu_Dockerfile b/packager/testing/dockers/Ubuntu_Dockerfile index 03b5be72cd6..3e3a14ba8e1 100644 --- a/packager/testing/dockers/Ubuntu_Dockerfile +++ b/packager/testing/dockers/Ubuntu_Dockerfile @@ -7,9 +7,16 @@ RUN apt-get install -y \ libc-ares-dev \ build-essential git python python3 +# Default to python2 because our build system is ancient. +RUN ln -sf python2 /usr/bin/python + # Install depot_tools. -RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN touch depot_tools/.disable_auto_update ENV PATH /depot_tools:$PATH +# Bypass VPYTHON included by depot_tools. Prefer the system installation. +ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" + # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager".