fix(tests): improve karma config for failing CI server.

 - Add the `clearContext: false` option to hopefully fix the random
   "some tests reloaded" error.
 - Create separate CI karma config for all CI-related changes.
 - improve karma error capture for karma.js
 - reduce disconnectTimeout 500 mSecs
 - Update `test-versions.sh` to use CI Karma config and fix typos.

References #6699.  Closes #7094

config/karma-ci.conf.js

@@ -0,0 +1,23 @@
+var baseKarma = require('./karma.conf.js');
+
+module.exports = function(config) {
+  baseKarma(config);
+
+  // Override defaults with custom CI settings
+  config.set({
+    colors: false,
+    singleRun:true,
+    autoWatch: false,
+    logLevel: config.LOG_DEBUG,
+
+    // Only launch one browser at a time since doing multiple can cause disconnects/issues
+    concurrency: 1,
+
+    browsers: ['Chrome', 'PhantomJS2', 'Firefox'],
+
+    client: {
+      // Do not clear the context as this can cause reload failures with Jasmine
+      clearContext:false
+    }
+  });
+};

config/karma.conf.js

@@ -31,7 +31,6 @@ module.exports = function(config) {
 
   var dependencies = process.env.KARMA_TEST_JQUERY ? ['node_modules/jquery/dist/jquery.js'] : [];
       dependencies = dependencies.concat([
-        'node_modules/phantomjs-polyfill/bind-polyfill.js',
         'node_modules/angular/angular.js',
         'node_modules/angular-animate/angular-animate.js',
         'node_modules/angular-aria/angular-aria.js',
@@ -51,6 +50,8 @@ module.exports = function(config) {
     frameworks: ['jasmine'],
     files: dependencies.concat(testSrc),
 
+    browserDisconnectTimeout:500,
+
     logLevel: config.LOG_DEBUG,
     port: 9876,
     reporters: ['progress'],
@@ -61,9 +62,6 @@ module.exports = function(config) {
     singleRun: true,
     autoWatch: false,
 
-    // Only launch one browser at a time since doing multiple can cause disconnects/issues
-    concurrency: 1,
-
     // Start these browsers, currently available:
     // - Chrome
     // - ChromeCanary
@@ -72,35 +70,7 @@ module.exports = function(config) {
     // - Safari (only Mac; has to be installed with `npm install karma-safari-launcher`)
     // - PhantomJS
     // - IE (only Windows; has to be installed with `npm install karma-ie-launcher`)
-    browsers: ['Firefox', 'PhantomJS2'],
-
-    // you can define custom flags
-    customLaunchers: {
-      Chrome_without_security: {
-        base: 'Chrome',
-        flags: ['--disable-web-security']
-      },
-      PhantomJS_without_security: {
-        base: 'PhantomJS',
-        options : {
-          onResourceRequested : function (request) {
-              console.log('Request ' + JSON.stringify(request, undefined, 4));
-          },
-          onError : function (msg, trace) {
-              console.log(msg);
-              trace.forEach(function(item) {
-                  console.log('  ', item.file, ':', item.line);
-              });
-          }
-        },
-        flags: [
-          '--web-security=no',
-          '--proxy-type=none',
-          '--remote-debugger-port=9000',
-          '--remote-debugger-autorun=yes'
-        ]
-      }
-    }
+    browsers: ['Chrome', 'PhantomJS2']
   });
 
 };

gulp/tasks/karma.js

@@ -4,76 +4,71 @@ var util = require('../util');
 var ROOT = require('../const').ROOT;
 var args = util.args;
 var Server = require('karma').Server;
+
+var karma;    // karma server instance using `new Server()`
 var karmaConfig = {
   logLevel: 'warn',
-  singleRun: true,
-  autoWatch: false,
   configFile: ROOT + '/config/karma.conf.js'
 };
-var karma;    // karma server instance using `new Server()`
 
 // Make full build of JS and CSS
 exports.dependencies = ['build'];
 
 exports.task = function (done) {
   var errorCount = 0;
 
-  /**
-   * For each version of testings (unminified, minified, minified w/ jQuery)
-   * capture the exitCode and update the error count...
-   *
-   * When all versions are done, report any errors that may manifest
-   * [e.g. perhaps in the minified tests]
-   *
-   * NOTE: All versions must pass before the CI server will announce 'success'
-   */
-  function captureError(next,done) {
-    return function(exitCode) {
-      if (exitCode != 0) {
-        gutil.log(gutil.colors.red("Karma exited with the following exit code: " + exitCode));
-        errorCount++;
-      }
-      // Do not process next set of tests if current set had >0 errors.
-      (errorCount > 0) && done() || next();
-    };
-  }
-
+  if ( args.browsers )  karmaConfig.browsers = args.browsers.trim().split(',');
+  if ( args.reporters ) karmaConfig.reporters = args.reporters.trim().split(',');
+  if ( args.config )    karmaConfig.configFile = ROOT + '/' + args.config.trim();
 
-  if ( args.browsers ) {
-    karmaConfig.browsers = args.browsers.trim().split(',');
-  }
 
-  if ( args.reporters ) {
-    karmaConfig.reporters = args.reporters.trim().split(',');
-  }
+  gutil.log(gutil.colors.blue('Running unit tests on unminified source.'));
 
-  gutil.log('Running unit tests on unminified source.');
   karma = new Server( karmaConfig, captureError(testMinified,clearEnv));
   karma.start();
 
   function testMinified() {
-    gutil.log('Running unit tests on minified source.');
+    gutil.log(gutil.colors.blue('Running unit tests on minified source.'));
     process.env.KARMA_TEST_COMPRESSED = true;
 
     karma = new Server(karmaConfig, captureError(testMinifiedJquery,clearEnv));
     karma.start();
   }
 
   function testMinifiedJquery() {
-    gutil.log('Running unit tests on minified source w/ jquery.');
+    gutil.log(gutil.colors.blue('Running unit tests on minified source w/ jquery.'));
     process.env.KARMA_TEST_COMPRESSED = true;
     process.env.KARMA_TEST_JQUERY = true;
 
-    karma = new Server(karmaConfig, clearEnv);
+    karma = new Server(karmaConfig, done);
     karma.start();
   }
 
   function clearEnv() {
-    process.env.KARMA_TEST_COMPRESSED = undefined;
-    process.env.KARMA_TEST_JQUERY = undefined;
+    process.env.KARMA_TEST_COMPRESSED = false;
+    process.env.KARMA_TEST_JQUERY = false;
+  }
 
-    if (errorCount > 0) { process.exit(errorCount); }
-    done();
-    done = function() { };
+  /**
+   * For each version of testings (unminified, minified, minified w/ jQuery)
+   * capture the exitCode and update the error count...
+   *
+   * When all versions are done, report any errors that may manifest
+   * [e.g. perhaps in the minified tests]
+   *
+   * NOTE: All versions must pass before the CI server will announce 'success'
+   */
+  function captureError(next,abort) {
+    return function(exitCode, errorCode) {
+      if (exitCode != 0) {
+        errorCount++;
+        gutil.log(gutil.colors.red("Karma exited with the following exit code: " + exitCode));
+        if ( errorCode ) gutil.log(gutil.colors.red("Karma error code: " + errorCode));
+      }
+
+      if( errorCount > 0) process.exit(errorCount);
+      else                next();
+    };
   }
+
 };

scripts/test-versions.sh

@@ -1,16 +1,16 @@
-# the purpose of this file is to download
+# The purpose of this file is to download
 # assigned AngularJS source files and test
 # them against this build of AngularMaterial.
 
 # This works by pulling in all of the tags
-# form angular.js, finding the highest version
+# from angular.js, finding the highest version
 # numbers for each branch (e.g. 1.3 => 1.3.X where
 # X is the highest patch release). For each
 # detected version it will then copy over each
 # of the source files to the node_modules/angular-X
 # folder and then run `gulp karma` to see if
 # they pass. If there are one or more failed tests
-# then this script will propagate a failed exit code
+# then this script will propagate a failed exit code.
 
 # [INPUT]
 # just run `./scripts/test-versions.sh`
@@ -25,7 +25,7 @@ VERSIONS=(1.3 1.4 1.5 snapshot)
 BROWSERS="Firefox,Chrome,Safari"
 
 #
-# DO NOT EDIT PASSED THIS LINE 
+# DO NOT EDIT PAST THIS LINE
 #
 CDN="https://code.angularjs.org"
 FAILED=false
@@ -53,8 +53,8 @@ if [ ! -e ./tmp/angular.js ]; then
   git clone https://github.com/angular/angular.js ./tmp/angular.js
 fi
 
-# this will gaurantee that we have the latest versions
-# of AngularJS when testing material incase the HEAD
+# this will guarantee that we have the latest versions
+# of AngularJS when testing material in case the HEAD
 # of ./tmp/angular.js is outdated.
 git --git-dir ./tmp/angular.js/.git fetch
 
@@ -109,12 +109,13 @@ for VERSION in "${VERSIONS[@]}"; do
   done
 
   echo "\n"
-  node ./node_modules/gulp/bin/gulp.js karma --reporters='dots' --browsers=$BROWSERS
+  pwd
+  node ./node_modules/gulp/bin/gulp.js karma --config=config/karma-ci.conf.js --reporters='dots' --browsers=$BROWSERS
   LAST_EXIT_CODE=$?
 
   echo "\n\n--- Finished Testing AngularMaterial against AngularJS (${VERSION}) ---"
 
-  if [ $LAST_EXIT_CODE == "1" ]; then
+  if [ $LAST_EXIT_CODE != "0" ]; then
     echo "STATUS: FAILED"
     FAILED=true
   else