Fix for #45 #46 and more

* set 1.38.33 as ignored due probelm with dependencies * added python3 and python3-pip to slim image * mitigate new build system introduced by upstream project * temporarely removed possibility to compile incoming + master
trzecieu · Aug 29, 2019 · 73317ea · 73317ea
1 parent 785603d
commit 73317ea
Show file tree

Hide file tree

Showing 6 changed files with 108 additions and 19 deletions.
diff --git a/build b/build
@@ -47,6 +47,7 @@ LOG_COMPILATION = "build.log"
 SDK_IGNORED = [
     "1.38.7", # malformed due an incompatibility with EMSDK release
     "1.38.17", # ignored due an SDK error
+    "1.38.33", # broken dependency between emscipten<>emscripten-releases
 ]
 
 TEST_CASES = {
@@ -320,7 +321,8 @@ class DockerHelper:
     def decompose_image_id(image_id):
         print(image_id)
         project, tag = image_id.split(":")
-        return (project, tag)
+        version = re.match(r'.+(\d+\.\d+\.\d+).+', tag).groups()[0]
+        return (project, tag, version)
 
     @staticmethod
     def push_image(image, clean=False):
@@ -420,7 +422,9 @@ def get_builds(tags, update=False, branches=None, releases=False):
 
 def compile_image(image_id, emsdk_cs, no_cache = False):
     t_start = datetime.datetime.now()
-    docker_project, emscripten_sdk = DockerHelper.decompose_image_id(image_id)
+    print("---")
+    print(image_id)
+    docker_project, emscripten_sdk, emscripten_version = DockerHelper.decompose_image_id(image_id)
     log("[INFO] Start compiling [{i}]".format(i=image_id))
     with open("build_{}.log".format(image_id.replace("/","_")), "w") as f:
         cmd = []
@@ -431,7 +435,8 @@ def compile_image(image_id, emsdk_cs, no_cache = False):
         cmd += ["--no-cache"] if no_cache else []
         cmd += [
             "-t", image_id,
-            "--build-arg", "EMSCRIPTEN_SDK=" + emscripten_sdk,
+            # "--build-arg", "EMSCRIPTEN_SDK=" + emscripten_sdk,
+            "--build-arg", "EMSCRIPTEN_VERSION=" + emscripten_version,
             "--build-arg", "EMSDK_CHANGESET=" + emsdk_cs,
             "docker/" + docker_project
         ]
@@ -570,6 +575,11 @@ def create_compilation_sets(variants):
 
 def compile(args):
     """Build images, and push tags to the queue"""
+
+    assert(not args.branches)   # not supported ATM
+    assert(not args.incoming)   # not supported ATM
+    assert(not args.master)     # not supported ATM
+
     variants = get_variants_to_compile(args)
     sets = create_compilation_sets(variants)
 
@@ -586,7 +596,6 @@ def compile(args):
         for i, v in enumerate(s.variants):
             image_id = v.full_name
             dashboard.set_task(image_id, i + 1, len(s.variants))
-
             if compile_image(image_id, args.emsdk_cs, no_cache = args.no_cache) and EmscriptenTester.test_image(image_id) :
                 log("{} - SUCCESS".format(image_id), True)
             else:
@@ -669,6 +678,7 @@ if __name__ == "__main__":
     parser_build.add_argument("--releases", action="store_true", help="Update released SDKs")
     parser_build.add_argument("--fast-fail", action="store_true", help="Stops queue after first failure")
     parser_build.add_argument("--emsdk-cs", default="master", help="Explicitly use given branch/changeset of juj/emsdk")
+    parser_build.add_argument("--no-cache", action="store_true", help="Sets --no-cache for docker build command")
 
     parser_test = subparsers.add_parser("test", help="Test given tag(s) with Emscripten and WebAssembly compatibility")
     parser_test.set_defaults(function=test)
@@ -678,7 +688,6 @@ if __name__ == "__main__":
 
     parser_push = subparsers.add_parser("push", help="Runs a service what will push created images")
     parser_push.add_argument("--clean", action="store_true", help="Remove pushed images")
-    parser_push.add_argument("--no-cache", action="store_true", help="Sets --no-cache for docker build command")
     parser_push.add_argument("--no-block", action="store_true", help="Don't wait if queue is empty")
     parser_push.set_defaults(function=push)
 
@@ -687,4 +696,5 @@ if __name__ == "__main__":
     parser_set_latest.set_defaults(function=set_latest)
 
     args = parser.parse_args()
+
     args.function(args)
diff --git a/docker/trzeci/emscripten-slim/Dockerfile b/docker/trzeci/emscripten-slim/Dockerfile
@@ -3,7 +3,7 @@ FROM debian:stretch AS stage_build
 # ------------------------------ ABOUT THIS IMAGE  -----------------------------
 # This Dockerfile has two major sections:
 # * STAGE BUILD: Which uses required tools to build a static version of Emscripten SDK
-# * STAGE DEPLOY: Which copies folder of Emscripten (/emsd_portable) from previous stage, and installs very based tools to make Emscripten work.
+# * STAGE DEPLOY: Which copies folder of Emscripten (/emsdk_portable) from previous stage, and installs very based tools to make Emscripten work.
 #
 # Compiled Emscripten SDK meant to be ready to go out of the shelf. That is `/emsdk_portable`:
 # - contains every required part of Emscripten SDK
@@ -19,8 +19,8 @@ FROM debian:stretch AS stage_build
 # ------------------------------------------------------------------------------
 # -------------------------------- STAGE BUILD  --------------------------------
 # ------------------------------------------------------------------------------
-
-ARG EMSCRIPTEN_SDK=sdk-tag-1.37.16-64bit
+ARG EMSCRIPTEN_VERSION=1.38.33
+ARG EMSCRIPTEN_SDK=sdk-tag-${EMSCRIPTEN_VERSION}-64bit
 ARG EMSDK_CHANGESET=master
 
 # ------------------------------------------------------------------------------
@@ -44,7 +44,6 @@ RUN echo "## Update and install packages" \
         python-pip \
 &&  echo "## Done"
 
-
 RUN  echo "## Installing CMake" \
     &&  wget https://cmake.org/files/v3.6/cmake-3.6.3-Linux-x86_64.sh -q \
     &&  mkdir /opt/cmake \
@@ -55,18 +54,81 @@ RUN  echo "## Installing CMake" \
 RUN  echo "## Get EMSDK" \
     &&  git clone https://github.com/emscripten-core/emsdk.git ${EMSDK} && cd ${EMSDK} && git reset --hard ${EMSDK_CHANGESET} \
         \
-    # Update emscripten-tags.txt file manually with request version, this will tell emsdk that wanted version to compile is available
-    &&  echo ${EMSCRIPTEN_SDK} | sed  's/.\+-\([0-9]\+\.[0-9]\+\.[0-9]\+\).\+/\1/g' > emscripten-tags.txt \
+    &&  ./emsdk.py update-tags \
+    &&  releases_revision=`cat ./emscripten-releases-tags.txt | sed -n "s/.*${EMSCRIPTEN_VERSION}.*\"\(.*\)\".*/\1/p"` \
+    &&  ( cd releases && git reset --hard ${releases_revision} ) \
 &&  echo "## Done"
 
+# ------------------------------------------------------------------------------
+# This sections aims to mitigate new waterfall build system that has been used in emscripten
+# and the fact that it's not possible anymore to build new SDKs from sources
+
+    RUN  echo "## Compose Fastcomp" \
+        &&  cd ${EMSDK} && mkdir -p zips \
+            \
+        &&  repository=https://github.com/emscripten-core/emscripten-fastcomp \
+        &&  revision=`cat ./releases/DEPS | sed -n "s/.*fastcomp_revision.*'\(.*\)'.*/\1/p"` \
+        &&  touch ./zips/llvm-e${EMSCRIPTEN_VERSION}.tar.gz \
+        &&  git clone $repository clang/tag-e${EMSCRIPTEN_VERSION}/src \
+        &&  (cd clang/tag-e${EMSCRIPTEN_VERSION}/src && git reset --hard ${revision}) \
+            \
+    &&  echo "## Done"
+
+    RUN  echo "## Compose Fastcomp - Clang" \
+        &&  cd ${EMSDK} && mkdir -p zips \
+            \
+        &&  repository=https://github.com/emscripten-core/emscripten-fastcomp-clang \
+        &&  revision=`cat ./releases/DEPS | sed -n "s/.*fastcomp_clang_revision.*'\(.*\)'.*/\1/p"` \
+        &&  touch ./zips/clang-e${EMSCRIPTEN_VERSION}.tar.gz \
+        &&  git clone $repository clang/tag-e${EMSCRIPTEN_VERSION}/src/tools/clang \
+        &&  (cd clang/tag-e${EMSCRIPTEN_VERSION}/src/tools/clang && git reset --hard ${revision}) \
+            \
+    &&  echo "## Done"
+
+    RUN  echo "## Compose Binaryen" \
+        &&  cd ${EMSDK} && mkdir -p zips \
+            \
+        &&  repository=https://github.com/WebAssembly/binaryen \
+        &&  revision=`cat ./releases/DEPS | sed -n "s/.*binaryen_revision.*'\(.*\)'.*/\1/p"` \
+        &&  touch ./zips/binaryen-e${EMSCRIPTEN_VERSION}.tar.gz \
+        &&  git clone $repository binaryen/tag-${EMSCRIPTEN_VERSION} \
+        &&  (cd binaryen/tag-${EMSCRIPTEN_VERSION} && git reset --hard ${revision}) \
+            \
+    &&  echo "## Done"
+
+    RUN  echo "## Compose Emscripten" \
+        &&  cd ${EMSDK} && mkdir -p zips \
+            \
+        &&  repository=https://github.com/emscripten-core/emscripten \
+        &&  revision=`cat ./releases/DEPS | sed -n "s/.*emscripten_revision.*'\(.*\)'.*/\1/p"` \
+        &&  touch ./zips/emscripten-e${EMSCRIPTEN_VERSION}.tar.gz \
+        &&  git clone $repository emscripten/tag-${EMSCRIPTEN_VERSION} \
+        &&  (cd emscripten/tag-${EMSCRIPTEN_VERSION} && git reset --hard ${revision}) \
+            \
+    &&  echo "## Done"
+
+    # patch emsdk in order to install all required tools from repositories
+    # fix for: #45 #46
+    RUN  echo "## Patch EMSDK" \
+        &&  cd ${EMSDK} \
+        &&  printf "\n${EMSCRIPTEN_VERSION}" > legacy-emscripten-tags.txt \
+        &&  printf "\n${EMSCRIPTEN_VERSION}" > legacy-binaryen-tags.txt \
+        &&  sed -i 's/download_even_if_exists = True/download_even_if_exists = False # a hack to install binaryen from repo/' ./emsdk.py \
+        &&  sed -i "s/.*${EMSCRIPTEN_VERSION}.*//g" ./emscripten-releases-tags.txt \
+        &&  echo "\"${EMSCRIPTEN_VERSION}\"" > ./emscripten/tag-${EMSCRIPTEN_VERSION}/emscripten-version.txt \
+    &&  echo "## Done"
+
+# / end of work-around section
+# ------------------------------------------------------------------------------
+
 RUN echo "## Compile Emscripten" \
     &&  cd ${EMSDK} \
         \
     &&  ./emsdk install node-8.9.1-64bit > /dev/null \
     # Compile llvm with dynamic libs support
     # This will create local shared objects that are used on all LLVM tool.
     # It significantly reduces final size of image.
-    &&  LLVM_CMAKE_ARGS=-DLLVM_LINK_LLVM_DYLIB=ON,-DLLVM_LINK_LLVM_DYLIB=ON \
+    &&  LLVM_CMAKE_ARGS=-DLLVM_LINK_LLVM_DYLIB=ON \
         ./emsdk install --build=MinSizeRel ${EMSCRIPTEN_SDK} \
         \
 &&  echo "## Done"
@@ -282,6 +344,8 @@ RUN echo "## Update and install packages" \
         ca-certificates \
         python \
         python-pip \
+        python3 \
+        python3-pip \
     \
     # Standard Cleanup on Debian images
     &&  apt-get -y clean \
@@ -335,6 +399,8 @@ RUN echo "## Internal Testing of image (activated)" \
     &&  npm --version \
     &&  python --version \
     &&  pip --version \
+    &&  python3 --version \
+    &&  pip3 --version \
     \
     &&  em++ --version \
     &&  emcc --version \
@@ -356,6 +422,8 @@ RUN echo "## Internal Testing of image (no activation)" \
     &&  npm --version \
     &&  python --version \
     &&  pip --version \
+    &&  python3 --version \
+    &&  pip3 --version \
     \
     &&  em++ --version \
     &&  emcc --version \

diff --git a/docker/trzeci/emscripten-ubuntu/Dockerfile b/docker/trzeci/emscripten-ubuntu/Dockerfile
@@ -1,5 +1,5 @@
-ARG EMSCRIPTEN_SDK=sdk-tag-1.38.25-64bit
-FROM trzeci/emscripten:${EMSCRIPTEN_SDK} as emscripten_base
+ARG EMSCRIPTEN_VERSION=1.38.33
+FROM trzeci/emscripten:sdk-tag-${EMSCRIPTEN_VERSION}-64bit as emscripten_base
 
 # ------------------------------------------------------------------------------
 # Entries in this section is required to move Emscripten SDK to another base image

diff --git a/docker/trzeci/emscripten/Dockerfile b/docker/trzeci/emscripten/Dockerfile
@@ -1,5 +1,5 @@
-ARG EMSCRIPTEN_SDK=sdk-tag-1.37.16-64bit
-FROM trzeci/emscripten-slim:${EMSCRIPTEN_SDK}
+ARG EMSCRIPTEN_VERSION=1.38.33
+FROM trzeci/emscripten-slim:sdk-tag-${EMSCRIPTEN_VERSION}-64bit
 
 # ------------------------------ ABOUT THIS IMAGE  -----------------------------
 # This image shows one way of extending base image which is `emscripten-slim` by

diff --git a/emscripten-slim.md b/emscripten-slim.md
@@ -190,7 +190,12 @@ docker inspect  --format '{{ index .Config.Labels "org.label-schema.vcs-ref"}}'
 ## History
 <sub>(Please note that following history refers only to the history of this Docker Image and how it was build / what includes. For release notes of emscripten, please follow https://emscripten.org)</sub>
 
-* **1.38.33**: [#44](https://github.com/trzecieu/emscripten-docker/issues/44) Significant refactoring of base image emscripten-slim. Please visit issue, to get extended context and motivation.
+* **1.38.34**: Cumulative change
+[#45](https://github.com/trzecieu/emscripten-docker/issues/45)[#46](https://github.com/trzecieu/emscripten-docker/issues/46) Adopt image to new Emscripten SDK
+  * Starting from 1.38.33 Emscripten has switched project build system to waterfall, which has introduced some complication to building the image. The main problem was that no longer was possible to compile Emscripten from sources, with making sure that we have exactly the same components like Emscripten was released. For the instance WebAssembly/binaryen repo wasn't tagged with Emscripten version anymore.
+  * Added python3, python3-pip package as requested in [WebAssembly/binaryen#2281](https://github.com/WebAssembly/binaryen/pull/2281).
+
+[#44](https://github.com/trzecieu/emscripten-docker/issues/44) Significant refactoring of base image emscripten-slim. Please visit issue, to get extended context and motivation.
   * Improvements:
     * `/emsdk_portable` is fully moveable folder that can be used as a `COPY --from` source of multi stage build
     * `/emsdk_portable/dockerfiles` contains Dockerfile sources used to compile a particular image - so that it should be fairly easy to replicate and inspect content of images
@@ -206,6 +211,7 @@ docker inspect  --format '{{ index .Config.Labels "org.label-schema.vcs-ref"}}'
     * Image no longer preserves folder structure between versions, some tool might be placed in different location between versions
       * Instead it creates symbolic links in fixed locations that match old structure
     * `nodejs` is no longer symlinked (`node` should be used instead)
+* **1.38.33**: Version ignored due problem with dependencies
 * **1.38.26** [#36](https://github.com/trzecieu/emscripten-docker/issues/36) - Reduce image size from 330MB to 189MB
 * **1.38.20** [#34](https://github.com/trzecieu/emscripten-docker/issues/34) - Fix error when `emcc` tries to read internal documentation
 * **1.38.17** Version ignored due problems with [Emscripten]

diff --git a/emscripten.md b/emscripten.md
@@ -188,12 +188,16 @@ docker inspect  --format '{{ index .Config.Labels "org.label-schema.vcs-ref"}}'
 ## History
 <sub>(Please note that following history refers only to the history of this Docker Image and how it was build / what includes. For release notes of emscripten, please follow https://emscripten.org)</sub>
 
-* **1.38.33**: [#44](https://github.com/trzecieu/emscripten-docker/issues/44) Significant refactoring of base image emscripten-slim. Please visit issue, to get extended context and motivation.
+* **1.38.34**: Cumulative change
+[#45](https://github.com/trzecieu/emscripten-docker/issues/45)[#46](https://github.com/trzecieu/emscripten-docker/issues/46) Adopt image to new Emscripten SDK
+  * Starting from 1.38.33 Emscripten has switched project build system to waterfall, which has introduced some complication to building the image. The main problem was that no longer was possible to compile Emscripten from sources, with making sure that we have exactly the same components like Emscripten was released. For the instance WebAssembly/binaryen repo wasn't tagged with Emscripten version anymore.
+
+[#44](https://github.com/trzecieu/emscripten-docker/issues/44) Significant refactoring of base image emscripten-slim. Please visit issue, to get extended context and motivation.
   * Improvements:
     * `/emsdk_portable` is fully moveable folder that can be used as a `COPY --from` source of multi stage build
     * `/emsdk_portable/dockerfiles` contains Dockerfile sources used to compile a particular image - so that it should be fairly easy to replicate and inspect content of images
     * `emsdk` should be fully functional tool now, so that can be used for upgrading bundled emscripten SDK or to install extra tools
-    * Even further size optimization (From ~190MB to ~160MB) by:
+    * Even further size optimization for slim image (From ~190MB to ~160MB) by:
       * stripping out symbols from node.js and emscripten-clang tools
       * change base image to `debiang:stretch-slim`
       * remove non-essential files from emscripten folder
@@ -204,6 +208,7 @@ docker inspect  --format '{{ index .Config.Labels "org.label-schema.vcs-ref"}}'
     * Image no longer preserves folder structure between versions, some tool might be placed in different location between versions
       * Instead it creates symbolic links in fixed locations that match old structure
     * `nodejs` is no longer symlinked (`node` should be used instead)
+* **1.38.33**: Version ignored due problem with dependencies
 * **1.38.30**: [#40](https://github.com/trzecieu/emscripten-docker/issues/40) Fixed image compilation problem caused by JRE backport package
 * **1.38.22**: [#35](https://github.com/trzecieu/emscripten-docker/issues/35) upgrade to `cmake` 3.12.2
 * **1.38.17**: Version ignored due problems with [Emscripten]