Webpack DLLs and HappyPack (mui#135)

Adds a HappyPack and Vendor DLL implementation to improve the development experience.  The Vendor DLL is automatically calculated and build within the development server, reducing any management overhead.  We use a md5 hash against the project's dependencies to know when to rebuild the vendor dll.

.gitignore

@@ -34,3 +34,6 @@ npm-debug.log
 
 # Flow Coverage Report
 flow-coverage/
+
+# Happypack
+.happypack

README.md

@@ -209,6 +209,12 @@ For those of us not wanting to use `flow`. Running this command removes all `flo
 
 __Warning:__ This is a destructive behavior - it modifies your actual source files. Please make sure you commit any existing changes to your src before running this command.
 
+### `npm run build:dlls`
+
+Compiles a vendor library bundle, which improves build and recompile times by caching this code under the assumption it wont change often. Every time a you add a new dependency you must recompile the vendor DLL using the above command.
+
+Dependencies are added by hand in the `dll.config.js` file located inside `tools/webpack`.
+
 ## Troubleshooting ##
 
 ___Q:___ __My project fails to build and execute when I deploy it to my host__

package.json

@@ -77,8 +77,11 @@
     "flow-bin": "0.33.0",
     "flow-coverage-report": "0.1.0",
     "flow-remove-types": "1.0.4",
+    "happypack": "2.2.1",
     "json-loader": "0.5.4",
+    "md5": "2.2.1",
     "node-notifier": "4.6.1",
+    "promisify-node": "0.4.0",
     "react-hot-loader": "3.0.0-beta.6",
     "recursive-readdir": "2.1.0",
     "regenerator-runtime": "0.9.5",

src/universalMiddleware/clientAssets.js

@@ -46,4 +46,14 @@ const assets = Object.keys(assetsBundleFileContents)
     return acc;
   }, { scripts: [], styles: [] });
 
+// When we are in development mode our development server will generate a
+// vendor DLL in order to dramatically reduce our compilation times.  Therefore
+// we need to inject the path to the vendor dll bundle below.
+// @see /tools/development/ensureVendorDLLExists.js
+if (process.env.NODE_ENV === 'development') {
+  const vendorPaths = require('../../tools/config/vendorDLLPaths'); // eslint-disable-line global-require
+
+  assets.scripts.splice(0, 0, vendorPaths.dllWebPath);
+}
+
 export default assets;

src/universalMiddleware/render.js

@@ -23,6 +23,7 @@ function scriptTags(scripts : Array<string>) {
 }
 
 const styles = styleTags(clientAssets.styles);
+
 const scripts = scriptTags(clientAssets.scripts);
 
 /**

tools/.eslintrc

@@ -0,0 +1,10 @@
+{
+  "rules": {
+    "global-require": 0,
+    "no-console": 0,
+    "no-underscore-dangle": 0,
+    "import/no-extraneous-dependencies": 0,
+    "import/no-dynamic-require": 0,
+    "import/newline-after-import": 0
+  }
+}

tools/config/vendorDLLPaths.js

@@ -0,0 +1,20 @@
+const pathResolve = require('path').resolve;
+const appRootPath = require('app-root-path').toString();
+const envVars = require('./envVars');
+
+const dllName = 'vendor';
+const bundleSubDir = '/client/dlls';
+const dllOutputDir = pathResolve(appRootPath, envVars.BUNDLE_OUTPUT_PATH, `.${bundleSubDir}`);
+const dllWebPath = `${bundleSubDir}/${dllName}.js`;
+const dllPath = pathResolve(dllOutputDir, `${dllName}.js`);
+const dllJsonPath = pathResolve(dllOutputDir, `${dllName}.json`);
+const dependenciesHashFilePath = pathResolve(dllOutputDir, 'dependencies_hash');
+
+module.exports = {
+  dllName,
+  dllOutputDir,
+  dllPath,
+  dllJsonPath,
+  dependenciesHashFilePath,
+  dllWebPath,
+};

tools/development/ensureVendorDLLExists.js

@@ -0,0 +1,131 @@
+const webpack = require('webpack');
+const pathExtName = require('path').extname;
+const pathResolve = require('path').resolve;
+const md5 = require('md5');
+const fs = require('fs');
+const promisify = require('promisify-node');
+const recursive = promisify(require('recursive-readdir'));
+const appRootPath = require('app-root-path').toString();
+const vendorDLLPaths = require('../config/vendorDLLPaths');
+const createNotification = require('./createNotification');
+
+// -----------------------------------------------------------------------------
+// PRIVATES
+
+const importRegex = /(from '|require\(')([\w\-_]+)/g;
+
+const {
+  dllName,
+  dllOutputDir,
+  dllJsonPath,
+  dependenciesHashFilePath,
+} = vendorDLLPaths;
+
+// We calculate a hash of the package.json's dependencies, which we can use
+// to determine if dependencies have changed since the last time we built
+// the vendor dll.
+const currentDependenciesHash = md5(
+  JSON.stringify(require(pathResolve(appRootPath, 'package.json')).dependencies)
+);
+
+function webpackConfigFactory(modules) {
+  return {
+    // We only use this for development, so lets always include source maps.
+    devtool: 'inline-source-map',
+    entry: { [dllName]: modules },
+    output: {
+      path: dllOutputDir,
+      filename: `${dllName}.js`,
+      library: dllName,
+    },
+    plugins: [
+      new webpack.DllPlugin({
+        path: dllJsonPath,
+        name: dllName,
+      }),
+    ],
+  };
+}
+
+function buildVendorDLL() {
+  return new Promise((resolve, reject) => {
+    Promise.all([
+      recursive(pathResolve(appRootPath, 'src/client')),
+      recursive(pathResolve(appRootPath, 'src/shared/universal')),
+    ])
+    .then(([clientFiles, universalFiles]) => {
+      const isJsFile = file => pathExtName(file) === '.js';
+      const allJSFiles = [...clientFiles, ...universalFiles].filter(isJsFile);
+      const modules = allJSFiles.reduce((acc, cur) => {
+        const fileContents = fs.readFileSync(cur, 'utf8');
+        let match = importRegex.exec(fileContents);
+        while (match != null) {
+          acc.add(match[2]);
+          match = importRegex.exec(fileContents);
+        }
+        return acc;
+      }, new Set());
+
+      createNotification({
+        title: 'vendorDLL',
+        level: 'info',
+        message: 'Vendor DLL build complete. Check console for module list.',
+      });
+      console.log([...modules]);
+
+      const webpackConfig = webpackConfigFactory([...modules]);
+      const vendorDLLCompiler = webpack(webpackConfig);
+      vendorDLLCompiler.run((err) => {
+        if (err) {
+          reject(err);
+        }
+        // Update the dependency hash
+        if (!fs.existsSync(dllOutputDir)) {
+          fs.mkdirSync(dllOutputDir);
+        }
+        fs.writeFileSync(dependenciesHashFilePath, currentDependenciesHash);
+
+        resolve();
+      });
+    });
+  });
+}
+
+// -----------------------------------------------------------------------------
+// DEFAULT EXPORT
+
+function ensureVendorDLLExists() {
+  return new Promise((resolve, reject) => {
+    if (!fs.existsSync(dependenciesHashFilePath)) {
+      // builddll
+      createNotification({
+        title: 'vendorDLL',
+        level: 'warn',
+        message: 'Generating a new vendor dll for boosted development performance...',
+      });
+      buildVendorDLL().then(resolve).catch(reject);
+    } else {
+      // first check if the md5 hashes match
+      const dependenciesHash = fs.readFileSync(dependenciesHashFilePath, 'utf8');
+      const dependenciesChanged = dependenciesHash !== currentDependenciesHash;
+
+      if (dependenciesChanged) {
+        createNotification({
+          title: 'vendorDLL',
+          level: 'warn',
+          message: 'New vendor dependencies detected. Regenerating the vendor dll...',
+        });
+        buildVendorDLL().then(resolve).catch(reject);
+      } else {
+        createNotification({
+          title: 'vendorDLL',
+          level: 'info',
+          message: 'No changes to existing vendor dependencies. Using the existing vendor dll.',
+        });
+        resolve();
+      }
+    }
+  });
+}
+
+module.exports = ensureVendorDLLExists;

tools/development/index.js

@@ -1,43 +1,56 @@
-/* eslint-disable no-console */
-/* eslint-disable global-require */
-/* eslint-disable no-underscore-dangle */
-/* eslint-disable import/no-extraneous-dependencies */
-/* eslint-disable import/newline-after-import */
-
-const path = require('path');
+const pathResolve = require('path').resolve;
 const chokidar = require('chokidar');
 const webpack = require('webpack');
 const createNotification = require('./createNotification');
 const HotServer = require('./hotServer');
 const HotClient = require('./hotClient');
+const ensureVendorDLLExists = require('./ensureVendorDLLExists');
+const vendorDLLPaths = require('../config/vendorDLLPaths');
 
 class HotDevelopment {
   constructor() {
-    try {
-      const clientConfigFactory = require('../webpack/client.config');
-      const clientConfig = clientConfigFactory({ mode: 'development' });
-      this.clientCompiler = webpack(clientConfig);
-
-      const middlewareConfigFactory = require('../webpack/universalMiddleware.config');
-      const middlewareConfig = middlewareConfigFactory({ mode: 'development' });
-      this.middlewareCompiler = webpack(middlewareConfig);
-
-      const serverConfigFactory = require('../webpack/server.config');
-      const serverConfig = serverConfigFactory({ mode: 'development' });
-      this.serverCompiler = webpack(serverConfig);
-    } catch (err) {
+    ensureVendorDLLExists().then(() => {
+      try {
+        const clientConfigFactory = require('../webpack/client.config');
+        const clientConfig = clientConfigFactory({ mode: 'development' });
+        // Install the vendor DLL plugin.
+        clientConfig.plugins.push(
+          new webpack.DllReferencePlugin({
+            manifest: require(vendorDLLPaths.dllJsonPath),
+          })
+        );
+        this.clientCompiler = webpack(clientConfig);
+
+        const middlewareConfigFactory = require('../webpack/universalMiddleware.config');
+        const middlewareConfig = middlewareConfigFactory({ mode: 'development' });
+        this.middlewareCompiler = webpack(middlewareConfig);
+
+        const serverConfigFactory = require('../webpack/server.config');
+        const serverConfig = serverConfigFactory({ mode: 'development' });
+        this.serverCompiler = webpack(serverConfig);
+      } catch (err) {
+        createNotification({
+          title: 'development',
+          level: 'error',
+          message: 'Webpack configs are invalid, please check the console for more information.',
+        });
+        console.log(err);
+        return;
+      }
+
+      this.prepHotServer();
+      this.prepHotUniversalMiddleware();
+      this.prepHotClient();
+    }).catch((err) => {
       createNotification({
-        title: 'development',
+        title: 'vendorDLL',
         level: 'error',
-        message: 'Webpack configs are invalid, please check the console for more information.',
+        message: 'Unfortunately an error occured whilst trying to build the vendor dll used by the development server. Please check the console for more information.',
       });
-      console.log(err);
-      return;
-    }
-
-    this.prepHotServer();
-    this.prepHotUniversalMiddleware();
-    this.prepHotClient();
+      if (err) {
+        console.log(err);
+      }
+    });
   }
 
   prepHotClient() {
@@ -134,7 +147,7 @@ const hotDevelopment = new HotDevelopment();
 // Any changes to our webpack configs should be notified as requiring a restart
 // of the development tool.
 const watcher = chokidar.watch(
-  path.resolve(__dirname, '../webpack')
+  pathResolve(__dirname, '../webpack')
 );
 watcher.on('ready', () => {
   watcher.on('change', () => {

tools/utils/index.js

@@ -1,3 +1,25 @@
+const CPU_COUNT = require('os').cpus().length;
+const HappyPack = require('happypack');
+
+// This determines how many threads a HappyPack instance can spin up.
+// See the plugins section of the webpack configuration for more.
+const happyPackThreadPool = HappyPack.ThreadPool({ // eslint-disable-line new-cap
+  size: CPU_COUNT >= 4
+    ? Math.round(CPU_COUNT / 2)
+    : 2,
+});
+
+// Generates a HappyPack plugin.
+// @see https://github.com/amireh/happypack/
+function happyPackPlugin({ name, loaders }) {
+  return new HappyPack({
+    id: name,
+    verbose: false,
+    threadPool: happyPackThreadPool,
+    loaders,
+  });
+}
+
 // :: [Any] -> [Any]
 function removeEmpty(x) {
   return x.filter(y => !!y);
@@ -22,4 +44,5 @@ module.exports = {
   removeEmpty,
   ifElse,
   merge,
+  happyPackPlugin,
 };