fix: fix docker build (#4461)

* always compile extensions if git not present * compile extension if main module is missing * do not print git errors * fix docker builds * re-enable docker on CI * load image into docker after building * report on errors and fail script * remove buildx * force extensions to install all dependencies when NODE_ENV is production, yarn will not install dev dependencies. We need those to build extensions. * revert not using buildx * start up docker in background * update code scan action to run on push to main
microsoft · Oct 22, 2020 · e8d0d55 · e8d0d55
1 parent d18b0fb
commit e8d0d55
Show file tree

Hide file tree

Showing 8 changed files with 349 additions and 79 deletions.
diff --git a/Composer/.dockerignore → .dockerignore b/Composer/.dockerignore → .dockerignore
@@ -14,4 +14,9 @@ Dockerfile
 extensions/localPublish/hostedBots/*
 
 # ignore test
-coverage
+Composer/coverage
+Composer/packages/server/.composer
+
+extensions/.build-cache.json
+extensions/**/lib
+extensions/**/dist
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -2,51 +2,53 @@ name: "Code scanning - action"
 
 on:
   push:
+    branches:
+      - "main"
   pull_request:
+    types: [opened, reopened, synchronize]
   schedule:
-    - cron: '0 19 * * 0'
+    - cron: "0 19 * * 0"
 
 jobs:
   CodeQL-Build:
-
     # CodeQL runs on ubuntu-latest and windows-latest
     runs-on: ubuntu-latest
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-      with:
-        # We must fetch at least the immediate parents so that if this is
-        # a pull request then we can checkout the head.
-        fetch-depth: 2
-
-    # If this run was triggered by a pull request event, then checkout
-    # the head of the pull request instead of the merge commit.
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
-      
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      # Override language selection by uncommenting this and choosing your languages
-      # with:
-      #   languages: go, javascript, csharp, python, cpp, java
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with:
+          # We must fetch at least the immediate parents so that if this is
+          # a pull request then we can checkout the head.
+          fetch-depth: 2
+
+      # If this run was triggered by a pull request event, then checkout
+      # the head of the pull request instead of the merge commit.
+      - run: git checkout HEAD^2
+        if: ${{ github.event_name == 'pull_request' }}
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v1
+        # Override language selection by uncommenting this and choosing your languages
+        # with:
+        #   languages: go, javascript, csharp, python, cpp, java
+
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v1
+
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 https://git.io/JvXDl
+
+      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+      #    and modify them (or add more) to build your code if your project
+      #    uses a compiled language
+
+      #- run: |
+      #   make bootstrap
+      #   make release
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v1
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -0,0 +1,45 @@
+name: Docker Build Validation
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    types: [opened, reopened, synchronize]
+  workflow_dispatch:
+
+jobs:
+  build:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      - name: Build
+        uses: docker/build-push-action@v2
+        with:
+          push: false
+          context: .
+          file: Dockerfile
+          tags: botframework-composer
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          load: true
+      - name: Health check
+        run: |
+          docker run -d --rm -p "5000:5000" botframework-composer
+          sleep 10
+          curl -Is http://localhost:5000 | grep -q "200 OK"
+        shell: bash
+      - name: Clean up
+        if: always()
+        run: docker stop $(docker ps -a -q)
diff --git a/Composer/package.json b/Composer/package.json
@@ -122,8 +122,5 @@
     "tslib": "^2.0.0",
     "typescript": "3.9.2",
     "wsrun": "^5.2.0"
-  },
-  "dependencies": {
-    "globby": "^11.0.1"
   }
 }
diff --git a/Composer/scripts/compileExtensions.js b/Composer/scripts/compileExtensions.js
@@ -9,9 +9,7 @@ const path = require('path');
 // eslint-disable-next-line security/detect-child-process
 const { execSync } = require('child_process');
 
-const glob = require('globby');
-
-const extensionsDir = path.resolve(__dirname, '../../extensions');
+const extensionsDir = process.env.COMPOSER_BUILTIN_EXTENSIONS_DIR || path.resolve(__dirname, '../../extensions');
 const buildCachePath = path.resolve(extensionsDir, '.build-cache.json');
 
 console.log('Compiling extensions in %s', extensionsDir);
@@ -42,16 +40,20 @@ let buildCache = (() => {
   }
 })();
 
-const getLastModified = (files = []) => {
+const getLastModified = (extensionPath) => {
   let last = new Date(0);
 
-  for (const f of files) {
-    // returns last modified date of file in ISO 8601 format
-    const gitTimestamp = execSync(`git log -1 --pretty="%cI" "${f}"`).toString().trim();
+  try {
+    const gitTimestamp = execSync(`git log -1 --pretty="%cI" "${extensionPath}"`, {
+      encoding: 'utf-8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    }).trim();
     const timestamp = new Date(gitTimestamp);
     if (timestamp > last) {
       last = timestamp;
     }
+  } catch (_err) {
+    last = new Date();
   }
 
   return last;
@@ -65,7 +67,17 @@ const writeToCache = (name, lastModified) => {
   fs.writeFileSync(buildCachePath, JSON.stringify(buildCache, null, 2));
 };
 
-const shouldCompile = (name, lastModified) => {
+const missingMain = (extPath, packageJSON) => {
+  const main = packageJSON && packageJSON.main;
+
+  if (main) {
+    return !fs.existsSync(path.join(extPath, main));
+  }
+
+  return true;
+};
+
+const hasChanges = (name, lastModified) => {
   return buildCache[name] ? new Date(buildCache[name]) < lastModified : true;
 };
 
@@ -74,8 +86,8 @@ const compile = (name, extPath) => {
   const hasBuild = packageJSON && packageJSON.scripts && packageJSON.scripts.build;
 
   console.log('[%s] compiling', name);
-  console.log('[%s] yarn --force', name);
-  execSync('yarn --force --frozen-lockfile', { cwd: extPath, stdio: 'inherit' });
+  console.log('[%s] yarn install', name);
+  execSync('yarn --force --production=false', { cwd: extPath, stdio: 'inherit' });
 
   if (hasBuild) {
     console.log('[%s] yarn build', name);
@@ -87,20 +99,30 @@ const compile = (name, extPath) => {
 
 checkComposerLibs();
 
+const errors = [];
+
 for (const entry of allExtensions) {
   if (entry.isDirectory()) {
-    const dir = path.join(extensionsDir, entry.name);
-    const allFiles = glob
-      .sync('**/*', { cwd: dir, gitignore: true, ignore: ['node_modules', 'lib', 'dist'] })
-      .map((f) => path.join(dir, f));
-    const lastModified = getLastModified(allFiles);
-    if (shouldCompile(entry.name, lastModified)) {
+    const extPath = path.join(extensionsDir, entry.name);
+    const packageJSON = JSON.parse(fs.readFileSync(path.join(extPath, 'package.json')));
+    const lastModified = getLastModified(extPath);
+    if (missingMain(extPath, packageJSON) || hasChanges(entry.name, lastModified)) {
       try {
-        compile(entry.name, dir);
+        compile(entry.name, extPath);
         writeToCache(entry.name, lastModified);
       } catch (err) {
+        errors.push({
+          name: entry.name,
+          message: err.message,
+        });
         console.error(err);
       }
     }
   }
 }
+
+if (errors.length > 0) {
+  const formattedErrors = errors.map((e) => `\t- [${e.name}] ${e.message}`).join('\n');
+  console.error(`There was an error compiling these extensions:\n${formattedErrors}`);
+  process.exit(1);
+}
diff --git a/Composer/Dockerfile → Dockerfile b/Composer/Dockerfile → Dockerfile
@@ -12,32 +12,23 @@ FROM node:12-alpine as build
 ARG YARN_ARGS
 
 WORKDIR /src/Composer
-COPY . .
+COPY ./Composer .
+COPY ./extensions ../extensions
 # run yarn install as a distinct layer
 RUN yarn install --frozen-lock-file $YARN_ARGS
 ENV NODE_OPTIONS "--max-old-space-size=4096"
 ENV NODE_ENV "production"
+ENV COMPOSER_BUILTIN_EXTENSIONS_DIR "/src/extensions"
 RUN yarn build:prod $YARN_ARGS
 
-
-
 FROM node:12-alpine as composerbasic
 ARG YARN_ARGS
 
 WORKDIR /app/Composer
 COPY --from=build /src/Composer/yarn.lock .
 COPY --from=build /src/Composer/package.json .
-COPY --from=build /src/Composer/packages/client ./packages/client
-COPY --from=build /src/Composer/packages/adaptive-flow ./packages/adaptive-flow
-COPY --from=build /src/Composer/packages/adaptive-form ./packages/adaptive-form
-COPY --from=build /src/Composer/packages/extension ./packages/extension
-COPY --from=build /src/Composer/packages/extension-client ./packages/extension-client
-COPY --from=build /src/Composer/packages/intellisense ./packages/intellisense
-COPY --from=build /src/Composer/packages/server ./packages/server
-COPY --from=build /src/Composer/packages/lib ./packages/lib
-COPY --from=build /src/Composer/packages/tools ./packages/tools
-COPY --from=build /src/Composer/packages/ui-plugins ./packages/ui-plugins
-COPY --from=build /src/extensions ./extensions
+COPY --from=build /src/Composer/packages ./packages
+COPY --from=build /src/extensions ../extensions
 
 ENV NODE_ENV "production"
 RUN yarn --production --frozen-lockfile --force $YARN_ARGS && yarn cache clean
@@ -90,4 +81,7 @@ ENV DOTNET_RUNNING_IN_CONTAINER=true \
   # Set the invariant mode since icu_libs isn't included (see https://github.com/dotnet/announcements/issues/20)
   DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true
 
+ENV COMPOSER_BUILTIN_EXTENSIONS_DIR "/app/extensions"
+ENV COMPOSER_REMOTE_EXTENSIONS_DIR "/app/remote-extensions"
+ENV COMPOSER_EXTENSION_DATA "/app/extensions.json"
 CMD ["yarn","start:server"]
diff --git a/extensions/azurePublish/yarn.lock b/extensions/azurePublish/yarn.lock
@@ -969,9 +969,6 @@ buffer@^5.1.0, buffer@^5.5.0:
   version "5.6.0"
   resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.6.0.tgz#a31749dc7d81d84db08abf937b6b8c4033f62786"
   integrity sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==
-  dependencies:
-    base64-js "^1.0.2"
-    ieee754 "^1.1.4"
 
 cardinal@^2.1.1:
   version "2.1.1"