[Recorder] soft-record (#7213)

This PR intends to add soft-record, a way to only record what hasn't changed.

To be able to check whether the tests have changed, we check whether a MD5 hash of the test function's source code has changed from the previous time a recording stored this MD5 to the current test execution.

Any new recording will store this MD5 into the recorded file. To only record the tests that have changed, the user must specify the environment variable TEST_MODE, with value "soft-record".

Fixes #5156

.gitignore

@@ -134,7 +134,6 @@ dist
 dist-*
 test-dist
 test-dist*
-browser
 test-browser
 typings
 typedoc
@@ -150,3 +149,9 @@ swagger/*.json
 
 # library-specific ignores
 sdk/keyvault/*/src/**/*.js
+
+# skipping browser tests while also ignoring the browser folders where the browser bundles are
+/**/browser/*.js
+/**/browser/*.js.map
+/**/browser/*.html
+!/test/browser/*.spec.ts

sdk/keyvault/keyvault-certificates/karma.conf.js

@@ -1,7 +1,7 @@
 // https://github.com/karma-runner/karma-chrome-launcher
 process.env.CHROME_BIN = require("puppeteer").executablePath();
 require("dotenv").config({ path: "../.env" });
-const { jsonRecordingFilterFunction, isPlaybackMode } = require("@azure/test-utils-recorder");
+const { jsonRecordingFilterFunction, isPlaybackMode, isSoftRecordMode, isRecordMode } = require("@azure/test-utils-recorder");
 
 module.exports = function(config) {
   config.set({
@@ -28,7 +28,7 @@ module.exports = function(config) {
       // Promise,String.prototype.startsWith,String.prototype.endsWith,String.prototype.repeat,String.prototype.includes,Array.prototype.includes,Object.keys
       "https://cdn.polyfill.io/v2/polyfill.js?features=Promise,String.prototype.startsWith,String.prototype.endsWith,String.prototype.repeat,String.prototype.includes,Array.prototype.includes,Object.keys|always",
       "dist-test/index.browser.js"
-    ].concat(isPlaybackMode() ? ["recordings/browsers/**/*.json"] : []),
+    ].concat((isPlaybackMode() || isSoftRecordMode()) ? ["recordings/browsers/**/*.json"] : []),
 
     exclude: [],
 
@@ -99,7 +99,7 @@ module.exports = function(config) {
     browserDisconnectTolerance: 3,
     browserConsoleLogOptions: {
       // IMPORTANT: COMMENT the following line if you want to print debug logs in your browsers in record mode!!
-      terminal: process.env.TEST_MODE !== "record"
+      terminal: !isRecordMode()
     },
 
     client: {

sdk/keyvault/keyvault-keys/karma.conf.js

@@ -1,7 +1,7 @@
 // https://github.com/karma-runner/karma-chrome-launcher
 process.env.CHROME_BIN = require("puppeteer").executablePath();
 require("dotenv").config({ path: "../.env" });
-const { jsonRecordingFilterFunction, isPlaybackMode } = require("@azure/test-utils-recorder");
+const { jsonRecordingFilterFunction, isPlaybackMode, isSoftRecordMode, isRecordMode } = require("@azure/test-utils-recorder");
 
 module.exports = function(config) {
   config.set({
@@ -28,7 +28,7 @@ module.exports = function(config) {
       // Promise,String.prototype.startsWith,String.prototype.endsWith,String.prototype.repeat,String.prototype.includes,Array.prototype.includes,Object.keys
       "https://cdn.polyfill.io/v2/polyfill.js?features=Promise,String.prototype.startsWith,String.prototype.endsWith,String.prototype.repeat,String.prototype.includes,Array.prototype.includes,Object.keys|always",
       "dist-test/index.browser.js"
-    ].concat(isPlaybackMode() ? ["recordings/browsers/**/*.json"] : []),
+    ].concat((isPlaybackMode() || isSoftRecordMode()) ? ["recordings/browsers/**/*.json"] : []),
 
     exclude: [],
 
@@ -99,7 +99,7 @@ module.exports = function(config) {
     browserDisconnectTolerance: 3,
     browserConsoleLogOptions: {
       // IMPORTANT: COMMENT the following line if you want to print debug logs in your browsers in record mode!!
-      terminal: process.env.TEST_MODE !== "record"
+      terminal: !isRecordMode()
     },
 
     client: {

sdk/keyvault/keyvault-secrets/karma.conf.js

@@ -1,7 +1,7 @@
 // https://github.com/karma-runner/karma-chrome-launcher
 process.env.CHROME_BIN = require("puppeteer").executablePath();
 require("dotenv").config({ path: "../.env" });
-const { jsonRecordingFilterFunction, isPlaybackMode } = require("@azure/test-utils-recorder");
+const { jsonRecordingFilterFunction, isPlaybackMode, isSoftRecordMode, isRecordMode } = require("@azure/test-utils-recorder");
 
 module.exports = function(config) {
   config.set({
@@ -28,7 +28,7 @@ module.exports = function(config) {
       // Promise,String.prototype.startsWith,String.prototype.endsWith,String.prototype.repeat,String.prototype.includes,Array.prototype.includes,Object.keys
       "https://cdn.polyfill.io/v2/polyfill.js?features=Promise,String.prototype.startsWith,String.prototype.endsWith,String.prototype.repeat,String.prototype.includes,Array.prototype.includes,Object.keys|always",
       "dist-test/index.browser.js"
-    ].concat(isPlaybackMode() ? ["recordings/browsers/**/*.json"] : []),
+    ].concat((isPlaybackMode() || isSoftRecordMode()) ? ["recordings/browsers/**/*.json"] : []),
 
     exclude: [],
 
@@ -99,7 +99,7 @@ module.exports = function(config) {
     browserDisconnectTolerance: 3,
     browserConsoleLogOptions: {
       // IMPORTANT: COMMENT the following line if you want to print debug logs in your browsers in record mode!!
-      terminal: process.env.TEST_MODE !== "record"
+      terminal: !isRecordMode()
     },
 
     client: {

sdk/storage/storage-blob/karma.conf.js

@@ -1,7 +1,7 @@
 // https://github.com/karma-runner/karma-chrome-launcher
 process.env.CHROME_BIN = require("puppeteer").executablePath();
 require("dotenv").config({ path: "../.env" });
-const { jsonRecordingFilterFunction, isPlaybackMode } = require("@azure/test-utils-recorder");
+const { jsonRecordingFilterFunction, isPlaybackMode, isSoftRecordMode, isRecordMode } = require("@azure/test-utils-recorder");
 
 module.exports = function(config) {
   config.set({
@@ -33,7 +33,7 @@ module.exports = function(config) {
       // Promise,String.prototype.startsWith,String.prototype.endsWith,String.prototype.repeat,String.prototype.includes,Array.prototype.includes,Object.assign,Object.keys
       "https://cdn.polyfill.io/v2/polyfill.js?features=Symbol,Promise,String.prototype.startsWith,String.prototype.endsWith,String.prototype.repeat,String.prototype.includes,Array.prototype.includes,Object.assign,Object.keys|always",
       "dist-test/index.browser.js"
-    ].concat(isPlaybackMode() ? ["recordings/browsers/**/*.json"] : []),
+    ].concat((isPlaybackMode() || isSoftRecordMode()) ? ["recordings/browsers/**/*.json"] : []),
 
     // list of files / patterns to exclude
     exclude: [],
@@ -119,7 +119,7 @@ module.exports = function(config) {
     browserDisconnectTimeout: 10000,
     browserDisconnectTolerance: 3,
     browserConsoleLogOptions: {
-      terminal: process.env.TEST_MODE !== "record"
+      terminal: !isRecordMode()
     },
 
     client: {

sdk/storage/storage-file-datalake/karma.conf.js

@@ -1,7 +1,7 @@
 // https://github.com/karma-runner/karma-chrome-launcher
 process.env.CHROME_BIN = require("puppeteer").executablePath();
 require("dotenv").config({ path: "../.env" });
-const { jsonRecordingFilterFunction, isPlaybackMode } = require("@azure/test-utils-recorder");
+const { jsonRecordingFilterFunction, isPlaybackMode, isSoftRecordMode, isRecordMode } = require("@azure/test-utils-recorder");
 
 module.exports = function(config) {
   config.set({
@@ -33,7 +33,7 @@ module.exports = function(config) {
       // Promise,String.prototype.startsWith,String.prototype.endsWith,String.prototype.repeat,String.prototype.includes,Array.prototype.includes,Object.assign,Object.keys
       "https://cdn.polyfill.io/v2/polyfill.js?features=Symbol,Promise,String.prototype.startsWith,String.prototype.endsWith,String.prototype.repeat,String.prototype.includes,Array.prototype.includes,Object.assign,Object.keys|always",
       "dist-test/index.browser.js"
-    ].concat(isPlaybackMode() ? ["recordings/browsers/**/*.json"] : []),
+    ].concat((isPlaybackMode() || isSoftRecordMode()) ? ["recordings/browsers/**/*.json"] : []),
 
     // list of files / patterns to exclude
     exclude: [],
@@ -118,7 +118,7 @@ module.exports = function(config) {
     browserDisconnectTimeout: 10000,
     browserDisconnectTolerance: 3,
     browserConsoleLogOptions: {
-      terminal: process.env.TEST_MODE !== "record"
+      terminal: !isRecordMode()
     },
 
     client: {

sdk/storage/storage-file-share/karma.conf.js

@@ -1,7 +1,7 @@
 // https://github.com/karma-runner/karma-chrome-launcher
 process.env.CHROME_BIN = require("puppeteer").executablePath();
 require("dotenv").config({ path: "../.env" });
-const { jsonRecordingFilterFunction, isPlaybackMode } = require("@azure/test-utils-recorder");
+const { jsonRecordingFilterFunction, isPlaybackMode, isSoftRecordMode, isRecordMode } = require("@azure/test-utils-recorder");
 
 module.exports = function(config) {
   config.set({
@@ -33,7 +33,7 @@ module.exports = function(config) {
       // Promise,String.prototype.startsWith,String.prototype.endsWith,String.prototype.repeat,String.prototype.includes,Array.prototype.includes,Object.assign,Object.keys
       "https://cdn.polyfill.io/v2/polyfill.js?features=Symbol,Promise,String.prototype.startsWith,String.prototype.endsWith,String.prototype.repeat,String.prototype.includes,Array.prototype.includes,Object.assign,Object.keys|always",
       "dist-test/index.browser.js"
-    ].concat(isPlaybackMode() ? ["recordings/browsers/**/*.json"] : []),
+    ].concat((isPlaybackMode() || isSoftRecordMode()) ? ["recordings/browsers/**/*.json"] : []),
 
     // list of files / patterns to exclude
     exclude: [],
@@ -119,7 +119,7 @@ module.exports = function(config) {
     browserDisconnectTimeout: 10000,
     browserDisconnectTolerance: 3,
     browserConsoleLogOptions: {
-      terminal: process.env.TEST_MODE !== "record"
+      terminal: !isRecordMode()
     },
 
     client: {

sdk/storage/storage-queue/karma.conf.js

@@ -1,7 +1,7 @@
 // https://github.com/karma-runner/karma-chrome-launcher
 process.env.CHROME_BIN = require("puppeteer").executablePath();
 require("dotenv").config({ path: "../.env" });
-const { jsonRecordingFilterFunction, isPlaybackMode } = require("@azure/test-utils-recorder");
+const { jsonRecordingFilterFunction, isPlaybackMode, isSoftRecordMode, isRecordMode } = require("@azure/test-utils-recorder");
 
 module.exports = function(config) {
   config.set({
@@ -33,7 +33,7 @@ module.exports = function(config) {
       // Promise,String.prototype.startsWith,String.prototype.endsWith,String.prototype.repeat,String.prototype.includes,Array.prototype.includes,Object.assign,Object.keys
       "https://cdn.polyfill.io/v2/polyfill.js?features=Symbol,Promise,String.prototype.startsWith,String.prototype.endsWith,String.prototype.repeat,String.prototype.includes,Array.prototype.includes,Object.assign,Object.keys|always",
       "dist-test/index.browser.js"
-    ].concat(isPlaybackMode() ? ["recordings/browsers/**/*.json"] : []),
+    ].concat((isPlaybackMode() || isSoftRecordMode()) ? ["recordings/browsers/**/*.json"] : []),
 
     // list of files / patterns to exclude
     exclude: [],
@@ -119,7 +119,7 @@ module.exports = function(config) {
     browserDisconnectTimeout: 10000,
     browserDisconnectTolerance: 3,
     browserConsoleLogOptions: {
-      terminal: process.env.TEST_MODE !== "record"
+      terminal: !isRecordMode()
     },
 
     client: {

sdk/test-utils/recorder/CHANGELOG.md

@@ -1,8 +1,9 @@
 ## 2020-02-06
 
 - If any URLs are meant to be replaced in the recordings with `replaceableVariables`, hostname from the URLs will be independently matched and replaced. [#7204](https://github.com/Azure/azure-sdk-for-js/issues/7204)
-
   - Example - ENV_VAR `ACCOUNT_URL=https://azureaccount.com/` is supposed to be replaced with `https://endpoint/`, `azureaccount.com` will be independently matched and replaced with `endpoint` too.
+- Added the "soft-record" mode, which allows users to only record the tests that have changed. [#7213](https://github.com/Azure/azure-sdk-for-js/issues/7213)
+
 
 - [BUG FIX] Tests leveraging `coreHttp.requestOptions.timeout` with empty recordings(no nock scopes with request-responses defined in the recording) fail during the playback mode when executed along with other tests. Fixed the issue by resetting nock's global state. More info - [#7264](https://github.com/Azure/azure-sdk-for-js/issues/7264)
 

sdk/test-utils/recorder/GUIDELINES.md

@@ -8,6 +8,12 @@
   - Tests hit the live-service
   - [Nock](https://www.npmjs.com/package/nock)/[Nise](https://www.npmjs.com/package/nise) are leveraged for recording the request-responses for future use
   - If recordings are already present, forces re-recording
+  - The recorded files contain a hash of the test that executed that recording.
+  - Some recorded files might not have the hash if they were generated by an older version of the recorder.
+- If TEST_MODE = "soft-record",
+  - Tests will be skipped if they haven't changed. `soft-record` mode is isomorphous to `record` mode in all other aspects.
+  - We determine whether a test has changed or not by doing a MD5 hash of the test's source code.
+  - If the hash is not stored in the recordings, they will be re-recorded to include the hash.
 - Else If TEST_MODE = "live",
   - Tests hit the live-service, we don't record the requests/responses
 - Else If TEST_MODE = "playback" (or if the TEST_MODE is neither "record" nor "live"),

sdk/test-utils/recorder/README.md

@@ -142,9 +142,9 @@ import { record, env, delay } from "@azure/test-utils-recorder";
 
 The common recorder provides the following public methods and properties:
 
-- `record`: Which deals with recording and playing back the network requests,
+- `record()`: Which deals with recording and playing back the network requests,
   depending on the value assigned to the `TEST_MODE` environment variable. If
-  `TEST_MODE` equals to `record`, it will automatically store network requests
+  `TEST_MODE` equals to `record` or `soft-record`, it will automatically store network requests
   in a plain text file in the folder `recordings` at the root of your
   repository (which for our example case is the root of the
   `@azure/keyvault/keyvault-keys` repository).
@@ -155,7 +155,7 @@ The common recorder provides the following public methods and properties:
   package in the repo. It also returns an object with a method `stop()`, which
   will allow you to control when you want the recorder to stop re-routing your
   http requests.
-- `Recorder`: The return of `record` is going to be an instance of the
+- `Recorder`: The return of the `record()` method is going to be an instance of the
   `Recorder`, which has some useful functions: `stop`, `skip`, `getUniqueName`,
   and `newDate`. `stop` will stop the recorder from storing a copy of the HTTP
   requests and responses. `skip` will pause the recorder only during the test
@@ -169,7 +169,9 @@ The common recorder provides the following public methods and properties:
   but only while the `TEST_MODE` is not `playback`, since you want to make sure you can run the playback tests
   as fast as possible.
 - `isRecordMode`, which is a shorthand for checking if the environment variable
-  `TEST_MODE` is set to `record`.
+  `TEST_MODE` is set to `record` or `soft-record`.
+- `isSoftRecordMode`, which is a shorthand for checking if the environment
+  variable `TEST_MODE` is set to `soft-record` only.
 - `isPlaybackMode`, which is a shorthand for checking if the environment
   variable `TEST_MODE` is set to `playback`.
 - `setReplaceableVariables`, which will allow you to hide sensitive content
@@ -183,12 +185,12 @@ The common recorder provides the following public methods and properties:
 ### Configuring your project
 
 Having the common recorder as a devDependency means that you'll be able to start
-recording tests right away by using the exported method `record`. We'll get
+recording tests right away by using the exported method `record()`. We'll get
 into the details further down this document. This function will do recordings,
 or will play back previous recordings, depending on an environment variable:
-`TEST_MODE`. If the environment variable `TEST_MODE` is empty, `record` (and most
+`TEST_MODE`. If the environment variable `TEST_MODE` is empty, `record()` (and most
 of the functions provided by test-utils-recorder) won't be doing anything. You'll need
-to set this environment variable to `record` to start recording, and then to
+to set the `TEST_MODE` environment variable to `record` (or `soft-record`) to start recording, and then to
 `playback` to play the recordings back at your code.
 
 #### package.json scripts
@@ -297,7 +299,7 @@ section of our guidelines:
 ### How to record
 
 To record your tests, make sure to set the environment variable `TEST_MODE` to
-`record`, then in your code, call to the `record` function exported from
+`record` or `soft-record`, then in your code, call to the `record()` function exported from
 `@azure/test-utils-recorder`, then call it before the http request you want to
 make. In the following example, we'll invoke the `record()` method before
 authenticating our KeyVault client:
@@ -336,6 +338,9 @@ After running this test with the `TEST_MODE` environment variable set to
 `recordings/node/my_test/recording_before_each_hook.js` with the contents of
 the HTTP request as well as the contents of the HTTP response.
 
+If `TEST_MODE` is set to `soft-record` instead, the recorder will only create
+this recording file if the test has changed from a previous execution.
+
 You'll see in the code above that we're invoking `recorder.stop`. This is so
 that, after each test, we can stop recording and the test file can be
 generated. We recommend creating new recorders on `beforeEach` and stopping the
@@ -391,6 +396,8 @@ request according to their matching copy stored in the recordings.
 
 Once you have your recorded files, to update them after changing one of the tests, simply
 re-run the tests with `TEST_MODE` set to `record`. This will overwrite previously existing files.
+Or re run the tests with `TEST_MODE` set to `soft-record` to only overwrite the files related to
+tests that have changed.
 
 > **Note:** If you rename the file of the test, or the name of the test, the
 > path of the recording will change. Make sure to delete the recordings
@@ -482,7 +489,7 @@ Some HTTP requests might have parameters with sensitive information. To get
 them out of your recordings, you can call to `skipQueryParams` with an array of strings
 where you specify the names of the query parameter you want to remove.
 
-For example, give nthat we find this query parameters in our recordings:
+For example, given that we find this query parameters in our recordings:
 `?sv=2018-11-09&sr=c&sig=<sig>&sktid=<sktid>&skv=2018-11-09&se=2019-08-07T07%3A00%3A00Z&sp=rwdl`,
 if we don't want the parameters "sr", "sig" and "sp" to appear in these files, we can do the following:
 

sdk/test-utils/recorder/karma.conf.js

@@ -96,7 +96,15 @@ module.exports = function(config) {
     // start these browsers
     // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher
     // 'ChromeHeadless', 'Chrome', 'Firefox', 'Edge', 'IE'
-    browsers: ["ChromeHeadless"],
+    // --no-sandbox allows our tests to run in Linux without having to change the system.
+    // --disable-web-security allows us to authenticate from the browser without having to write tests using interactive auth, which would be far more complex.
+    browsers: ["ChromeHeadlessNoSandbox"],
+    customLaunchers: {
+      ChromeHeadlessNoSandbox: {
+        base: "ChromeHeadless",
+        flags: ["--no-sandbox", "--disable-web-security"]
+      }
+    },
 
     // Continuous Integration mode
     // if true, Karma captures browsers, runs the tests and exits
@@ -110,7 +118,8 @@ module.exports = function(config) {
     browserDisconnectTimeout: 10000,
     browserDisconnectTolerance: 3,
     browserConsoleLogOptions: {
-      terminal: process.env.TEST_MODE !== "record"
+      // We would usually hide the logs from the tests, but we don't need to do this inside of the recorder package because we are not recording the tests.
+      // // terminal: process.env.TEST_MODE !== "record"
     },
 
     client: {

sdk/test-utils/recorder/package.json

@@ -71,9 +71,13 @@
     "@rollup/plugin-node-resolve": "^7.0.0",
     "@rollup/plugin-replace": "^2.2.0",
     "@types/fs-extra": "^8.0.0",
+    "@types/chai": "^4.1.6",
+    "@types/md5": "~2.1.33",
     "@types/mocha": "^5.2.5",
     "@types/nise": "^1.4.0",
-    "@types/chai": "^4.1.6",
+    "@types/node": "^8.0.0",
+    "@types/mock-require": "~2.0.0",
+    "@types/mock-fs": "~4.10.0",
     "chai": "^4.2.0",
     "karma": "^4.0.1",
     "karma-chrome-launcher": "^3.0.0",
@@ -88,9 +92,12 @@
     "karma-mocha": "^1.3.0",
     "karma-mocha-reporter": "^2.2.5",
     "karma-remap-istanbul": "^0.6.0",
+    "md5": "^2.2.1",
     "mocha": "^6.2.2",
     "mocha-junit-reporter": "^1.18.0",
     "mocha-multi": "^1.1.3",
+    "mock-fs": "^4.10.4",
+    "mock-require": "^3.0.3",
     "npm-run-all": "^4.1.5",
     "nyc": "^14.0.0",
     "prettier": "^1.16.4",