diff --git a/sdk/test-utils/recorder/CHANGELOG.md b/sdk/test-utils/recorder/CHANGELOG.md index 46e731205c25..930e98c0fb3f 100644 --- a/sdk/test-utils/recorder/CHANGELOG.md +++ b/sdk/test-utils/recorder/CHANGELOG.md @@ -2,6 +2,36 @@ ## 2.0.0 (Unreleased) +## 2022-01-27 + +Add support for the new string sanitizers, including **breaking changes**: + +- Removed the `Sanitizer` class, instead making the `addSanitizers` function in `sanitizer.ts` take in a `HttpClient` and recording ID as parameter. +- Refactored the `addSanitizers` function to call smaller functions for each sanitizer (some of which are a bit FP-style) instead of using if statements + special cases. Hopefully this will make things a bit easier to maintain. +- Some other minor refactors (e.g. extracting duplicated `createRecordingRequest` function into a utility). +- Add support for the string sanitizers in what I think is the most logical way, but there is a **breaking change**: + - When calling `addSanitizers`, instead of specifying `generalRegexSanitizers: [...]` etc., you now specify `generalSanitizers: [...]`. Both regex sanitizers and string sanitizers can be used in this way, for example: + +```ts +recorder.addSanitizers({ + generalSanitizers: [ + { + regex: true, // Regex matching is enabled by setting the 'regex' option to true. + target: ".*regex", + value: "sanitized", + }, + { + // Note that `regex` defaults to false and doesn't need to be specified when working with bare strings. + // In my experience, this is the most common scenario anyway. + target: "Not a regex", + value: "sanitized", + }, + ], +}); +``` + +[#19954](https://github.com/Azure/azure-sdk-for-js/pull/19954) + ## 2022-01-06 - Renaming the package `@azure-tools/test-recorder-new@1.0.0` as `@azure-tools/test-recorder@2.0.0`. diff --git a/sdk/test-utils/recorder/MIGRATION.md b/sdk/test-utils/recorder/MIGRATION.md index 9456dfba50f5..32cc9bb9a4e8 100644 --- a/sdk/test-utils/recorder/MIGRATION.md +++ b/sdk/test-utils/recorder/MIGRATION.md @@ -15,7 +15,7 @@ The new recorder is version 2.0.0 of the `@azure-tools/test-recorder` package. U // ... "devDependencies": { // ... - "@azure-tools/test-recorder": "^2.0.0", + "@azure-tools/test-recorder": "^2.0.0" } } ``` @@ -152,15 +152,20 @@ In this example, the name of the queue used in the recording is randomized. Howe A powerful feature of the legacy recorder was its `customizationsOnRecordings` option, which allowed for arbitrary replacements to be made to recordings. The new recorder's analog to this is the sanitizer functionality. -### GeneralRegexSanitizer +### General sanitizers -For a simple find/replace, a `GeneralRegexSanitizer` can be used. For example: +For a simple find/replace, `generalSanitizers` can be used. For example: ```ts await recorder.addSanitizers({ - generalRegexSanitizers: [ + generalSanitizers: [ { - regex: "find", // This should be a .NET regular expression as it is passed to the .NET proxy tool + target: "find", // With `regex` unspecified, this matches a plaintext string + value: "replace", + }, + { + regex: true, // Enable regex matching + target: "[Rr]egex", // This is a .NET regular expression that will be compiled by the proxy tool. value: "replace", }, // add additional sanitizers here as required @@ -168,7 +173,9 @@ await recorder.addSanitizers({ }); ``` -This example would replace all instances of `find` in the recording with `replace`. +This example has two sanitizers: +- The first sanitizer replaces all instances of "find" in the recording with "replace". +- The second example demonstrates the use of a regular expression for replacement, where anything matching the .NET regular expression `[Rr]egex` (i.e. "Regex" and "regex") would be replaced with "replace". ### ConnectionStringSanitizer diff --git a/sdk/test-utils/recorder/src/recorder.ts b/sdk/test-utils/recorder/src/recorder.ts index a94fbca7e6c6..6f3168df1551 100644 --- a/sdk/test-utils/recorder/src/recorder.ts +++ b/sdk/test-utils/recorder/src/recorder.ts @@ -3,9 +3,7 @@ import { createDefaultHttpClient, - createPipelineRequest, HttpClient, - HttpMethods, Pipeline, PipelinePolicy, PipelineRequest, @@ -27,7 +25,7 @@ import { Test } from "mocha"; import { sessionFilePath } from "./utils/sessionFilePath"; import { SanitizerOptions } from "./utils/utils"; import { paths } from "./utils/paths"; -import { Sanitizer } from "./sanitizer"; +import { addSanitizers, transformsInfo } from "./sanitizer"; import { handleEnvSetup } from "./utils/envSetupForPlayback"; import { Matcher, setMatcher } from "./matcher"; import { @@ -37,6 +35,7 @@ import { WebResource, WebResourceLike, } from "@azure/core-http"; +import { createRecordingRequest } from "./utils/createRecordingRequest"; /** * This client manages the recorder life cycle and interacts with the proxy-tool to do the recording, @@ -55,7 +54,6 @@ export class Recorder { private stateManager = new RecordingStateManager(); private httpClient?: HttpClient; private sessionFile?: string; - private sanitizer?: Sanitizer; private variables: Record; constructor(private testContext?: Test | undefined) { @@ -68,7 +66,6 @@ export class Recorder { "Unable to determine the recording file path, testContext provided is not defined." ); } - this.sanitizer = new Sanitizer(this.url, this.httpClient); } this.variables = {}; } @@ -113,8 +110,12 @@ export class Recorder { */ async addSanitizers(options: SanitizerOptions): Promise { // If check needed because we only sanitize when the recording is being generated, and we need a recording to apply the sanitizers on. - if (isRecordMode() && ensureExistence(this.sanitizer, "this.sanitizer")) { - return this.sanitizer.addSanitizers(options); + if ( + isRecordMode() && + ensureExistence(this.httpClient, "this.httpClient") && + ensureExistence(this.recordingId, "this.recordingId") + ) { + return addSanitizers(this.httpClient, this.url, this.recordingId, options); } } @@ -135,7 +136,7 @@ export class Recorder { const startUri = `${this.url}${isPlaybackMode() ? paths.playback : paths.record}${ paths.start }`; - const req = this._createRecordingRequest(startUri); + const req = createRecordingRequest(startUri, this.sessionFile, this.recordingId); if (ensureExistence(this.httpClient, "TestProxyHttpClient.httpClient")) { const rsp = await this.httpClient.sendRequest({ @@ -153,12 +154,14 @@ export class Recorder { if (isPlaybackMode()) { this.variables = rsp.bodyAsText ? JSON.parse(rsp.bodyAsText) : {}; } - if (ensureExistence(this.sanitizer, "TestProxyHttpClient.sanitizer")) { - // Setting the recordingId in the sanitizer, - // the sanitizers added will take the recording id and only be part of the current test - this.sanitizer.setRecordingId(this.recordingId); - await handleEnvSetup(options.envSetupForPlayback, this.sanitizer); - } + + await handleEnvSetup( + this.httpClient, + this.url, + this.recordingId, + options.envSetupForPlayback + ); + // Sanitizers to be added only in record mode if (isRecordMode() && options.sanitizerOptions) { // Makes a call to the proxy-tool to add the sanitizers for the current recording id @@ -177,7 +180,7 @@ export class Recorder { this.stateManager.state = "stopped"; if (this.recordingId !== undefined) { const stopUri = `${this.url}${isPlaybackMode() ? paths.playback : paths.record}${paths.stop}`; - const req = this._createRecordingRequest(stopUri); + const req = createRecordingRequest(stopUri, undefined, this.recordingId); req.headers.set("x-recording-save", "true"); if (isRecordMode()) { @@ -211,19 +214,16 @@ export class Recorder { } } - /** - * Adds the recording file and the recording id headers to the requests that are sent to the proxy tool. - * These are required to appropriately save the recordings in the record mode and picking them up in playback. - */ - private _createRecordingRequest(url: string, method: HttpMethods = "POST") { - const req = createPipelineRequest({ url, method }); - if (ensureExistence(this.sessionFile, "sessionFile")) { - req.body = JSON.stringify({ "x-recording-file": this.sessionFile }); + async transformsInfo(): Promise { + if (isLiveMode()) { + throw new RecorderError("Cannot call transformsInfo in live mode"); } - if (this.recordingId !== undefined) { - req.headers.set("x-recording-id", this.recordingId); + + if (ensureExistence(this.httpClient, "this.httpClient")) { + return await transformsInfo(this.httpClient, this.url, this.recordingId!); } - return req; + + throw new RecorderError("Expected httpClient to be defined"); } /** diff --git a/sdk/test-utils/recorder/src/sanitizer.ts b/sdk/test-utils/recorder/src/sanitizer.ts index 9039754ec3ed..b3d29440b2f6 100644 --- a/sdk/test-utils/recorder/src/sanitizer.ts +++ b/sdk/test-utils/recorder/src/sanitizer.ts @@ -1,230 +1,305 @@ import { HttpClient } from "@azure/core-rest-pipeline"; -import { createPipelineRequest, HttpMethods } from "@azure/core-rest-pipeline"; import { getRealAndFakePairs } from "./utils/connectionStringHelpers"; +import { createRecordingRequest } from "./utils/createRecordingRequest"; import { paths } from "./utils/paths"; import { + ConnectionStringSanitizer, + ContinuationSanitizer, + FindReplaceSanitizer, getTestMode, + HeaderSanitizer, isRecordMode, + isStringSanitizer, ProxyToolSanitizers, RecorderError, - RegexSanitizer, - sanitizerKeywordMapping, + RemoveHeaderSanitizer, SanitizerOptions, } from "./utils/utils"; /** - * Sanitizer class to handle communication with the proxy-tool relating to the sanitizers adding/resetting, etc. + * Signature of a function that adds a sanitizer of type T. */ -export class Sanitizer { - constructor(private url: string, private httpClient: HttpClient) {} - private recordingId: string | undefined; - - setRecordingId(recordingId: string): void { - this.recordingId = recordingId; - } - - /** - * Returns the html document of all the available transforms in the proxy-tool - */ - async transformsInfo(): Promise { - if (this.recordingId) { - const infoUri = `${this.url}${paths.info}${paths.available}`; - const req = this._createRecordingRequest(infoUri, "GET"); - if (!this.httpClient) { - throw new RecorderError( - `Something went wrong, Sanitizer.httpClient should not have been undefined in ${getTestMode()} mode.` - ); - } - const rsp = await this.httpClient.sendRequest({ - ...req, - allowInsecureConnection: true, - }); - if (rsp.status !== 200) { - throw new RecorderError("Info request failed."); - } - return rsp.bodyAsText; - } else { - throw new RecorderError( - "Bad state, recordingId is not defined when called transformsInfo()." - ); - } - } - - /** - * addSanitizers adds sanitizers to the current recording. Sanitizers will be applied before recordings are saved. - * - * Takes SanitizerOptions as the input, passes on to the proxy-tool. - */ - async addSanitizers(options: SanitizerOptions): Promise { - if (options.connectionStringSanitizers) { - await Promise.all( - options.connectionStringSanitizers.map((replacer) => - this.addConnectionStringSanitizer(replacer.actualConnString, replacer.fakeConnString) - ) - ); - } +type AddSanitizer = ( + httpClient: HttpClient, + url: string, + recordingId: string, + sanitizer: T +) => Promise; +/** + * Given an AddSanitizer function, create an AddSanitizer function that operates on an array of T, adding + * each sanitizer in the array individually. + */ +const pluralize = + (singular: AddSanitizer): AddSanitizer => + async (httpClient: HttpClient, url: string, recordingId: string, sanitizers: T[]) => { await Promise.all( - ( - [ - // The following sanitizers have similar request bodies and this abstraction avoids duplication - "generalRegexSanitizers", - "bodyKeySanitizers", - "bodyRegexSanitizers", - "headerRegexSanitizers", - "uriRegexSanitizers", - ] as const - ).map((prop) => { - const replacers = options[prop]; - if (replacers) { - return Promise.all( - replacers.map((replacer: RegexSanitizer) => { - if ( - // sanitizers where the "regex" is a required attribute - [ - "bodyKeySanitizers", - "bodyRegexSanitizers", - "generalRegexSanitizers", - "uriRegexSanitizers", - ].includes(prop) && - !replacer.regex - ) { - if (!isRecordMode()) return; - throw new RecorderError( - `Attempted to add an invalid sanitizer - ${JSON.stringify(replacer)}` - ); - } - return this.addSanitizer({ - sanitizer: sanitizerKeywordMapping[prop], - body: JSON.stringify(replacer), - }); - }) - ); - } else return; - }) + sanitizers.map((sanitizer) => singular(httpClient, url, recordingId, sanitizer)) ); + }; - await Promise.all( - ( - [ - // The following sanitizers have similar request bodies and this abstraction avoids duplication - "resetSanitizer", - "oAuthResponseSanitizer", - ] as const - ).map((prop) => { - // TODO: Test - if (options[prop]) { - return this.addSanitizer({ - sanitizer: sanitizerKeywordMapping[prop], - body: undefined, - }); - } else return; - }) - ); +/** + * Makes an AddSanitizer function that passes the sanitizer content directly to the test proxy request body. + */ +const makeAddSanitizer = + (sanitizerName: ProxyToolSanitizers): AddSanitizer> => + async ( + httpClient: HttpClient, + url: string, + recordingId: string, + sanitizer: Record + ) => { + await addSanitizer(httpClient, url, recordingId, { + sanitizer: sanitizerName, + body: sanitizer, + }); + }; - if (options.removeHeaderSanitizer) { - this.addSanitizer({ - sanitizer: "RemoveHeaderSanitizer", - body: JSON.stringify({ - headersForRemoval: options.removeHeaderSanitizer.headersForRemoval.toString(), - }), +/** + * Makes an AddSanitizer function that adds the sanitizer if the value is set to true, + * and otherwise makes no request to the server. Used for ResetSanitizer and OAuthResponseSanitizer. + */ +const makeAddBodilessSanitizer = + (sanitizerName: ProxyToolSanitizers): AddSanitizer => + async (httpClient: HttpClient, url: string, recordingId: string, enable: boolean) => { + if (enable) { + await addSanitizer(httpClient, url, recordingId, { + sanitizer: sanitizerName, + body: undefined, }); } + }; - if (options.continuationSanitizers) { - // TODO: Test - await Promise.all( - options.continuationSanitizers.map((replacer) => - this.addSanitizer({ - sanitizer: "ContinuationSanitizer", - body: JSON.stringify({ - ...replacer, - resetAfterFirst: replacer.resetAfterFirst.toString(), - }), - }) - ) - ); - } - - if (options.uriSubscriptionIdSanitizer) { - await this.addSanitizer({ - sanitizer: "UriSubscriptionIdSanitizer", - body: JSON.stringify(options.uriSubscriptionIdSanitizer), +/** + * Makes an AddSanitizer function for a FindReplaceSanitizer, for example a bodySanitizer. + * Depending on the input FindReplaceSanitizer options, either adds a sanitizer named `regexSanitizerName` + * or `stringSanitizerName`. + */ +const makeAddFindReplaceSanitizer = + ( + regexSanitizerName: ProxyToolSanitizers, + stringSanitizerName: ProxyToolSanitizers + ): AddSanitizer => + async ( + httpClient: HttpClient, + url: string, + recordingId: string, + sanitizer: FindReplaceSanitizer + ): Promise => { + if (isStringSanitizer(sanitizer)) { + await addSanitizer(httpClient, url, recordingId, { + sanitizer: stringSanitizerName, + body: { + target: sanitizer.target, + value: sanitizer.value, + }, + }); + } else { + await addSanitizer(httpClient, url, recordingId, { + sanitizer: regexSanitizerName, + body: { + regex: sanitizer.target, + value: sanitizer.value, + groupForReplace: sanitizer.groupForReplace, + }, }); } + }; + +/** + * Internally, + * - connection strings are parsed and + * - each part of the connection string is mapped with its corresponding fake value + * - GeneralStringSanitizer is applied for each of the parts with the real and fake values that are parsed + */ +async function addConnectionStringSanitizer( + httpClient: HttpClient, + url: string, + recordingId: string, + { actualConnString, fakeConnString }: ConnectionStringSanitizer +): Promise { + if (!actualConnString) { + if (!isRecordMode()) return; + throw new RecorderError( + `Attempted to add an invalid sanitizer - ${JSON.stringify({ + actualConnString: actualConnString, + fakeConnString: fakeConnString, + })}` + ); } + // extract connection string parts and match call + const pairsMatched = getRealAndFakePairs(actualConnString, fakeConnString); + await addSanitizers(httpClient, url, recordingId, { + generalSanitizers: Object.entries(pairsMatched).map(([key, value]) => { + return { value, target: key }; + }), + }); +} - /** - * Internally, - * - connection strings are parsed and - * - each part of the connection string is mapped with its corresponding fake value - * - generalRegexSanitizer is applied for each of the parts with the real and fake values that are parsed - */ - async addConnectionStringSanitizer( - actualConnString: string | undefined, - fakeConnString: string - ): Promise { - if (!actualConnString) { - if (!isRecordMode()) return; - throw new RecorderError( - `Attempted to add an invalid sanitizer - ${JSON.stringify({ - actualConnString: actualConnString, - fakeConnString: fakeConnString, - })}` - ); - } - // extract connection string parts and match call - const pairsMatched = getRealAndFakePairs(actualConnString, fakeConnString); - await this.addSanitizers({ - generalRegexSanitizers: Object.entries(pairsMatched).map(([key, value]) => { - return { value, regex: key }; - }), +/** + * Adds a ContinuationSanitizer with the given options. + */ +async function addContinuationSanitizer( + httpClient: HttpClient, + url: string, + recordingId: string, + sanitizer: ContinuationSanitizer +) { + await addSanitizer(httpClient, url, recordingId, { + sanitizer: "ContinuationSanitizer", + body: { + ...sanitizer, + resetAfterFirst: sanitizer.resetAfterFirst.toString(), + }, + }); +} + +/** + * Adds a RemoveHeaderSanitizer with the given options. + */ +async function addRemoveHeaderSanitizer( + httpClient: HttpClient, + url: string, + recordingId: string, + sanitizer: RemoveHeaderSanitizer +) { + await addSanitizer(httpClient, url, recordingId, { + sanitizer: "RemoveHeaderSanitizer", + body: { + headersForRemoval: sanitizer.headersForRemoval.toString(), + }, + }); +} + +/** + * Adds a HeaderRegexSanitizer or HeaderStringSanitizer. + * + * HeaderSanitizer is a special case of FindReplaceSanitizer where a header name ('key') must be provided. + * Additionally, the 'target' option is not required. If target is unspecified, the header's value will always + * be replaced. + */ +async function addHeaderSanitizer( + httpClient: HttpClient, + url: string, + recordingId: string, + sanitizer: HeaderSanitizer +) { + if (sanitizer.regex || !sanitizer.target) { + await addSanitizer(httpClient, url, recordingId, { + sanitizer: "HeaderRegexSanitizer", + body: { + key: sanitizer.key, + value: sanitizer.value, + regex: sanitizer.target, + groupForReplace: sanitizer.groupForReplace, + }, + }); + } else { + await addSanitizer(httpClient, url, recordingId, { + sanitizer: "HeaderStringSanitizer", + body: { + key: sanitizer.key, + target: sanitizer.target, + value: sanitizer.value, + }, }); } +} - /** - * Atomic method to add a simple sanitizer. - */ - private async addSanitizer(options: { - sanitizer: ProxyToolSanitizers; - body: string | undefined; - }): Promise { - if (this.recordingId !== undefined) { - const uri = `${this.url}${paths.admin}${ - options.sanitizer !== "Reset" ? paths.addSanitizer : paths.reset - }`; - const req = this._createRecordingRequest(uri); - if (options.sanitizer !== "Reset") { - req.headers.set("x-abstraction-identifier", options.sanitizer); - } - req.headers.set("Content-Type", "application/json"); - req.body = options.body; - if (!this.httpClient) { - throw new RecorderError( - `Something went wrong, Recorder.httpClient should not have been undefined in ${getTestMode()} mode.` - ); - } - const rsp = await this.httpClient.sendRequest({ - ...req, - allowInsecureConnection: true, - }); - if (rsp.status !== 200) { - throw new RecorderError("addSanitizer request failed."); +const addSanitizersActions: { + [K in keyof SanitizerOptions]: AddSanitizer>; +} = { + generalSanitizers: pluralize( + makeAddFindReplaceSanitizer("GeneralRegexSanitizer", "GeneralStringSanitizer") + ), + bodySanitizers: pluralize( + makeAddFindReplaceSanitizer("BodyRegexSanitizer", "BodyStringSanitizer") + ), + headerSanitizers: pluralize(addHeaderSanitizer), + uriSanitizers: pluralize(makeAddFindReplaceSanitizer("UriRegexSanitizer", "UriStringSanitizer")), + connectionStringSanitizers: pluralize(addConnectionStringSanitizer), + bodyKeySanitizers: pluralize(makeAddSanitizer("BodyKeySanitizer")), + continuationSanitizers: pluralize(addContinuationSanitizer), + removeHeaderSanitizer: addRemoveHeaderSanitizer, + oAuthResponseSanitizer: makeAddBodilessSanitizer("OAuthResponseSanitizer"), + uriSubscriptionIdSanitizer: makeAddSanitizer("UriSubscriptionIdSanitizer"), + resetSanitizer: makeAddBodilessSanitizer("Reset"), +}; + +export async function addSanitizers( + httpClient: HttpClient, + url: string, + recordingId: string, + options: SanitizerOptions +): Promise { + await Promise.all( + Object.entries(options).map(([key, sanitizer]) => { + const action = addSanitizersActions[key as keyof SanitizerOptions]; + if (!action) { + throw new RecorderError(`Sanitizer ${key} not implemented`); } - } else { - throw new RecorderError("Bad state, recordingId is not defined when called addSanitizer()."); - } + + return action(httpClient, url, recordingId, sanitizer); + }) + ); +} + +/** + * Atomic method to add a simple sanitizer. + */ +async function addSanitizer( + httpClient: HttpClient, + url: string, + recordingId: string, + options: { + sanitizer: ProxyToolSanitizers; + body: Record | undefined; + } +): Promise { + const uri = `${url}${paths.admin}${ + options.sanitizer !== "Reset" ? paths.addSanitizer : paths.reset + }`; + const req = createRecordingRequest(uri, recordingId); + if (options.sanitizer !== "Reset") { + req.headers.set("x-abstraction-identifier", options.sanitizer); } + req.headers.set("Content-Type", "application/json"); + req.body = options.body !== undefined ? JSON.stringify(options.body) : undefined; - /** - * Adds the recording id headers to the requests that are sent to the proxy tool. - * These are required to appropriately save the recordings in the record mode and picking them up in playback. - */ - private _createRecordingRequest(url: string, method: HttpMethods = "POST") { - const req = createPipelineRequest({ url: url, method }); - if (this.recordingId !== undefined) { - req.headers.set("x-recording-id", this.recordingId); + const rsp = await httpClient.sendRequest({ + ...req, + allowInsecureConnection: true, + }); + if (rsp.status !== 200) { + throw new RecorderError("addSanitizer request failed."); + } +} + +/** + * Returns the html document of all the available transforms in the proxy-tool + */ +export async function transformsInfo( + httpClient: HttpClient, + url: string, + recordingId: string +): Promise { + if (recordingId) { + const infoUri = `${url}${paths.info}${paths.available}`; + const req = createRecordingRequest(infoUri, undefined, recordingId, "GET"); + if (!httpClient) { + throw new RecorderError( + `Something went wrong, Sanitizer.httpClient should not have been undefined in ${getTestMode()} mode.` + ); + } + const rsp = await httpClient.sendRequest({ + ...req, + allowInsecureConnection: true, + }); + if (rsp.status !== 200) { + throw new RecorderError("Info request failed."); } - return req; + return rsp.bodyAsText; + } else { + throw new RecorderError("Bad state, recordingId is not defined when called transformsInfo()."); } } diff --git a/sdk/test-utils/recorder/src/utils/createRecordingRequest.ts b/sdk/test-utils/recorder/src/utils/createRecordingRequest.ts new file mode 100644 index 000000000000..74ffb6ec59c7 --- /dev/null +++ b/sdk/test-utils/recorder/src/utils/createRecordingRequest.ts @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { createPipelineRequest, HttpMethods } from "@azure/core-rest-pipeline"; + +/** + * Adds the recording id headers to the requests that are sent to the proxy tool. + * These are required to appropriately save the recordings in the record mode and picking them up in playback. + */ +export function createRecordingRequest( + url: string, + sessionFile?: string, + recordingId?: string, + method: HttpMethods = "POST" +) { + const req = createPipelineRequest({ url: url, method }); + + if (sessionFile !== undefined) { + req.body = JSON.stringify({ "x-recording-file": sessionFile }); + } + + if (recordingId !== undefined) { + req.headers.set("x-recording-id", recordingId); + } + return req; +} diff --git a/sdk/test-utils/recorder/src/utils/envSetupForPlayback.ts b/sdk/test-utils/recorder/src/utils/envSetupForPlayback.ts index b15c08d9ad11..08c375daf054 100644 --- a/sdk/test-utils/recorder/src/utils/envSetupForPlayback.ts +++ b/sdk/test-utils/recorder/src/utils/envSetupForPlayback.ts @@ -1,9 +1,15 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -import { Sanitizer } from "../sanitizer"; +import { HttpClient } from "@azure/core-rest-pipeline"; +import { addSanitizers } from "../sanitizer"; import { env } from "./env"; -import { isPlaybackMode, isRecordMode, setEnvironmentVariables, RegexSanitizer } from "./utils"; +import { + isPlaybackMode, + isRecordMode, + setEnvironmentVariables, + FindReplaceSanitizer, +} from "./utils"; /** * Supposed to be used in record and playback modes. @@ -13,8 +19,10 @@ import { isPlaybackMode, isRecordMode, setEnvironmentVariables, RegexSanitizer } * 2. If the env variables are present in the recordings as plain strings, they will be replaced with the provided values in record mode */ export async function handleEnvSetup( - envSetupForPlayback: Record, - sanitizer: Sanitizer + httpClient: HttpClient, + url: string, + recordingId: string, + envSetupForPlayback: Record ): Promise { if (envSetupForPlayback) { if (isPlaybackMode()) { @@ -23,15 +31,15 @@ export async function handleEnvSetup( } else if (isRecordMode()) { // If the env variables are present in the recordings as plain strings, they will be replaced with the provided values in record mode - const generalRegexSanitizers: RegexSanitizer[] = []; + const generalSanitizers: FindReplaceSanitizer[] = []; for (const [key, value] of Object.entries(envSetupForPlayback)) { const envKey = env[key]; if (envKey) { - generalRegexSanitizers.push({ regex: envKey, value }); + generalSanitizers.push({ target: envKey, value }); } } - await sanitizer.addSanitizers({ - generalRegexSanitizers, + await addSanitizers(httpClient, url, recordingId, { + generalSanitizers, }); } } diff --git a/sdk/test-utils/recorder/src/utils/utils.ts b/sdk/test-utils/recorder/src/utils/utils.ts index 8e9d8dcc99b8..8e88c4ed1bf8 100644 --- a/sdk/test-utils/recorder/src/utils/utils.ts +++ b/sdk/test-utils/recorder/src/utils/utils.ts @@ -54,56 +54,70 @@ export class RecordingStateManager { */ export type ProxyToolSanitizers = | "GeneralRegexSanitizer" + | "GeneralStringSanitizer" | "RemoveHeaderSanitizer" | "BodyKeySanitizer" | "BodyRegexSanitizer" + | "BodyStringSanitizer" | "ContinuationSanitizer" | "HeaderRegexSanitizer" + | "HeaderStringSanitizer" | "OAuthResponseSanitizer" | "UriRegexSanitizer" + | "UriStringSanitizer" | "UriSubscriptionIdSanitizer" | "Reset"; -/** - * Maps the sanitizer options to the header value expected by the proxy-tool - * - * Keys = Keys of the SanitizerOptions(excluding `connectionStringSanitizers`) - * Values = Keywords that should be passed as part of the headers to the proxy-tool to be able to leverage the sanitizer. - */ -export const sanitizerKeywordMapping: Record< - Exclude, - ProxyToolSanitizers -> = { - bodyKeySanitizers: "BodyKeySanitizer", - bodyRegexSanitizers: "BodyRegexSanitizer", - continuationSanitizers: "ContinuationSanitizer", - generalRegexSanitizers: "GeneralRegexSanitizer", - headerRegexSanitizers: "HeaderRegexSanitizer", - oAuthResponseSanitizer: "OAuthResponseSanitizer", - removeHeaderSanitizer: "RemoveHeaderSanitizer", - resetSanitizer: "Reset", - uriRegexSanitizers: "UriRegexSanitizer", - uriSubscriptionIdSanitizer: "UriSubscriptionIdSanitizer", -}; - /** * This sanitizer offers a general regex replace across request/response Body, Headers, and URI. For the body, this means regex applying to the raw JSON. */ export interface RegexSanitizer { + /** + * Set to true to show that regex replacement is to be used. + */ + regex: true; + /** * The substitution value. */ value: string; + /** * A regex. Can be defined as a simple regex replace OR if groupForReplace is set, a substitution operation. */ - regex?: string; + target: string; /** * The capture group that needs to be operated upon. Do not set if you're invoking a simple replacement operation. */ groupForReplace?: string; } +/** + * A sanitizer that performs a simple find/replace based on a plain string. + */ +export interface StringSanitizer { + /** + * If regex is set to false or is not specified, plain-text matching will be performed. + */ + regex?: false; + + /** + * The string to be replaced. + */ + target: string; + + /** + * The value that the string should be replaced with. + */ + value: string; +} + +export type FindReplaceSanitizer = RegexSanitizer | StringSanitizer; + +export function isStringSanitizer(sanitizer: FindReplaceSanitizer): sanitizer is StringSanitizer { + return !sanitizer.regex; +} + /** * This sanitizer offers regex update of a specific JTokenPath. * @@ -115,12 +129,18 @@ export interface RegexSanitizer { * * If the body is NOT a JSON object, this sanitizer will NOT be applied. */ -interface BodyKeySanitizer extends RegexSanitizer { +type BodyKeySanitizer = { + regex?: string; + + value?: string; + + groupForReplace?: string; + /** * The SelectToken path (which could possibly match multiple entries) that will be used to select JTokens for value replacement. */ jsonPath: string; -} +}; /** * Can be used for multiple purposes: @@ -129,19 +149,22 @@ interface BodyKeySanitizer extends RegexSanitizer { * 2) To do a simple regex replace operation, define arguments "key", "value", and "regex" * 3) To do a targeted substitution of a specific group, define all arguments "key", "value", and "regex" */ -interface HeaderRegexSanitizer extends RegexSanitizer { - /** - * The name of the header we're operating against. - */ +export interface HeaderSanitizer { key: string; + + regex?: boolean; + target?: string; + value?: string; + groupForReplace?: string; } + /** * Internally, * - connection strings are parsed and * - each part of the connection string is mapped with its corresponding fake value * - `generalRegexSanitizer` is applied for each of the parts with the real and fake values that are parsed */ -interface ConnectionStringSanitizer { +export interface ConnectionStringSanitizer { /** * Real connection string with all the secrets */ @@ -152,6 +175,16 @@ interface ConnectionStringSanitizer { fakeConnString: string; } +export interface ContinuationSanitizer { + key: string; + method?: string; + resetAfterFirst: boolean; +} + +export interface RemoveHeaderSanitizer { + headersForRemoval: string[]; +} + /** * Test-proxy tool supports "extensions" or "customizations" to the recording experience. * This means that non-default sanitizations such as the generalized regex find/replace on different parts of the recordings in various ways are possible. @@ -160,14 +193,8 @@ export interface SanitizerOptions { /** * This sanitizer offers a general regex replace across request/response Body, Headers, and URI. For the body, this means regex applying to the raw JSON. */ - generalRegexSanitizers?: RegexSanitizer[]; - /** - * Internally, - * - connection strings are parsed and - * - each part of the connection string is mapped with its corresponding fake value - * - `generalRegexSanitizer` is applied for each of the parts with the real and fake values that are parsed - */ - connectionStringSanitizers?: ConnectionStringSanitizer[]; + generalSanitizers?: FindReplaceSanitizer[]; + /** * This sanitizer offers regex replace within a returned body. * @@ -176,7 +203,27 @@ export interface SanitizerOptions { * * Regardless, there are examples present in `recorder-new/test/testProxyTests.spec.ts`. */ - bodyRegexSanitizers?: RegexSanitizer[]; + bodySanitizers?: FindReplaceSanitizer[]; + /** + * Can be used for multiple purposes: + * + * 1) To replace a key with a specific value, do not set "regex" value. + * 2) To do a simple regex replace operation, define arguments "key", "value", and "regex" + * 3) To do a targeted substitution of a specific group, define all arguments "key", "value", and "regex" + */ + headerSanitizers?: HeaderSanitizer[]; + /** + * General use sanitizer for cleaning URIs via regex. Runs a regex replace on the member of your choice. + */ + uriSanitizers?: FindReplaceSanitizer[]; + /** + * Internally, + * - connection strings are parsed and + * - each part of the connection string is mapped with its corresponding fake value + * - `generalRegexSanitizer` is applied for each of the parts with the real and fake values that are parsed + */ + connectionStringSanitizers?: ConnectionStringSanitizer[]; + /** * This sanitizer offers regex update of a specific JTokenPath. * @@ -189,37 +236,24 @@ export interface SanitizerOptions { * If the body is NOT a JSON object, this sanitizer will NOT be applied. */ bodyKeySanitizers?: BodyKeySanitizer[]; + /** * TODO * Has a bug, not implemented fully. */ - continuationSanitizers?: Array<{ key: string; method?: string; resetAfterFirst: boolean }>; - /** - * Can be used for multiple purposes: - * - * 1) To replace a key with a specific value, do not set "regex" value. - * 2) To do a simple regex replace operation, define arguments "key", "value", and "regex" - * 3) To do a targeted substitution of a specific group, define all arguments "key", "value", and "regex" - */ - headerRegexSanitizers?: HeaderRegexSanitizer[]; - /** - * General use sanitizer for cleaning URIs via regex. Runs a regex replace on the member of your choice. - */ - uriRegexSanitizers?: RegexSanitizer[]; + continuationSanitizers?: ContinuationSanitizer[]; + /** * A simple sanitizer that should be used to clean out one or multiple headers by their key. * Removes headers from before saving a recording. */ - removeHeaderSanitizer?: { - /** - * Array of header names. - */ - headersForRemoval: string[]; - }; + removeHeaderSanitizer?: RemoveHeaderSanitizer; + /** * TODO: To be tested with scenarios, not to be used yet. */ oAuthResponseSanitizer?: boolean; + /** * This sanitizer relies on UriRegexSanitizer to replace real subscriptionIds within a URI w/ a default or configured fake value. * This sanitizer is targeted using the regex "/subscriptions/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})". This is not a setting that can be changed for this sanitizer. For full regex support, take a look at UriRegexSanitizer. You CAN modify the value that the subscriptionId is replaced WITH however. @@ -230,6 +264,7 @@ export interface SanitizerOptions { */ value: string; }; + /** * This clears the sanitizers that are added. */ diff --git a/sdk/test-utils/recorder/test/sanitizers.spec.ts b/sdk/test-utils/recorder/test/sanitizers.spec.ts index 937d79dbf4e2..17b4ff498423 100644 --- a/sdk/test-utils/recorder/test/sanitizers.spec.ts +++ b/sdk/test-utils/recorder/test/sanitizers.spec.ts @@ -2,9 +2,8 @@ // Licensed under the MIT license. import { ServiceClient } from "@azure/core-client"; -import { expect } from "chai"; -import { env, isPlaybackMode, Recorder } from "../src"; -import { isRecordMode, RecorderError, TestMode } from "../src/utils/utils"; +import { isPlaybackMode, Recorder } from "../src"; +import { TestMode } from "../src/utils/utils"; import { getTestServerUrl, makeRequestAndVerifyResponse, setTestMode } from "./utils/utils"; // These tests require the following to be running in parallel @@ -31,17 +30,46 @@ import { getTestServerUrl, makeRequestAndVerifyResponse, setTestMode } from "./u describe("Sanitizers - functionalities", () => { it("GeneralRegexSanitizer", async () => { - env.SECRET_INFO = "abcdef"; - const fakeSecretInfo = "fake_secret_info"; await recorder.start({ - envSetupForPlayback: { - SECRET_INFO: fakeSecretInfo, + envSetupForPlayback: {}, + sanitizerOptions: { + generalSanitizers: [ + { + regex: true, + target: "abc+def", + value: "fake_secret_info", + }, + ], + }, + }); + await makeRequestAndVerifyResponse( + client, + { + path: `/sample_response/abcdef`, + body: "abcdef", + method: "GET", + }, + { val: "I am the answer!" } + ); + }); + + it("GeneralStringSanitizer", async () => { + await recorder.start({ + envSetupForPlayback: {}, + sanitizerOptions: { + generalSanitizers: [ + { + target: "abcdef", + value: "fake_secret_info", + }, + ], }, - }); // Adds generalRegexSanitizers by default based on envSetupForPlayback + }); await makeRequestAndVerifyResponse( client, { - path: `/sample_response/${env.SECRET_INFO}`, + path: `/sample_response/abcdef`, + body: "abcdef", method: "GET", }, { val: "I am the answer!" } @@ -105,9 +133,10 @@ import { getTestServerUrl, makeRequestAndVerifyResponse, setTestMode } from "./u await recorder.start({ envSetupForPlayback: {}, sanitizerOptions: { - bodyRegexSanitizers: [ + bodySanitizers: [ { - regex: "(.*)&SECRET=(?[^&]*)&(.*)", + regex: true, + target: "(.*)&SECRET=(?[^&]*)&(.*)", value: fakeSecretValue, groupForReplace: "secret_content", }, @@ -129,15 +158,15 @@ import { getTestServerUrl, makeRequestAndVerifyResponse, setTestMode } from "./u ); }); - it("UriRegexSanitizer", async () => { + it("UriSanitizer", async () => { const secretEndpoint = "host.docker.internal"; const fakeEndpoint = "fake_endpoint"; await recorder.start({ envSetupForPlayback: {}, sanitizerOptions: { - uriRegexSanitizers: [ + uriSanitizers: [ { - regex: secretEndpoint, + target: secretEndpoint, value: fakeEndpoint, }, ], @@ -224,7 +253,7 @@ import { getTestServerUrl, makeRequestAndVerifyResponse, setTestMode } from "./u await recorder.start({ envSetupForPlayback: {}, sanitizerOptions: { - headerRegexSanitizers: [ + headerSanitizers: [ { key: "your_uuid", value: sanitizedValue, @@ -266,9 +295,10 @@ import { getTestServerUrl, makeRequestAndVerifyResponse, setTestMode } from "./u await recorder.start({ envSetupForPlayback: {}, sanitizerOptions: { - bodyRegexSanitizers: [ + bodySanitizers: [ { - regex: "(.*)&SECRET=(?[^&]*)&(.*)", + regex: true, + target: "(.*)&SECRET=(?[^&]*)&(.*)", value: fakeSecretValue, groupForReplace: "secret_content", }, @@ -307,64 +337,5 @@ import { getTestServerUrl, makeRequestAndVerifyResponse, setTestMode } from "./u ); }); }); - - describe("Sanitizers - handling undefined", () => { - beforeEach(async () => { - await recorder.start({ envSetupForPlayback: {} }); - }); - - const cases = [ - { - options: { - connectionStringSanitizers: [ - { actualConnString: undefined, fakeConnString: "a=b;c=d" }, - ], - generalRegexSanitizers: [{ regex: undefined, value: "fake-value" }], - }, - title: "all sanitizers are undefined", - type: "negative", - }, - { - options: { - connectionStringSanitizers: [ - { actualConnString: undefined, fakeConnString: "a=b;c=d" }, - { actualConnString: "1=2,3=4", fakeConnString: "a=b;c=d" }, - ], - generalRegexSanitizers: [{ regex: undefined, value: "fake-value" }], - }, - title: "partial sanitizers are undefined", - type: "negative", - }, - { - options: { - connectionStringSanitizers: [ - { actualConnString: "1=2,3=4", fakeConnString: "a=b;c=d" }, - ], - generalRegexSanitizers: [{ regex: "value", value: "fake-value" }], - }, - title: "all sanitizers are defined", - type: "positive", - }, - ]; - - cases.forEach((testCase) => { - it(`case - ${testCase.title}`, async () => { - try { - await recorder.addSanitizers(testCase.options); - throw new Error("error was not thrown from addSanitizers call"); - } catch (error) { - if (isRecordMode() && testCase.type === "negative") { - expect((error as RecorderError).message).includes( - `Attempted to add an invalid sanitizer` - ); - } else { - expect((error as RecorderError).message).includes( - `error was not thrown from addSanitizers call` - ); - } - } - }); - }); - }); }); }); diff --git a/sdk/test-utils/recorder/test/testProxyClient.spec.ts b/sdk/test-utils/recorder/test/testProxyClient.spec.ts index ca31b1e547ca..7d8294e099fa 100644 --- a/sdk/test-utils/recorder/test/testProxyClient.spec.ts +++ b/sdk/test-utils/recorder/test/testProxyClient.spec.ts @@ -9,6 +9,7 @@ import { } from "@azure/core-rest-pipeline"; import { expect } from "chai"; import { env, Recorder } from "../src"; +import { createRecordingRequest } from "../src/utils/createRecordingRequest"; import { getTestMode, isLiveMode, RecorderError, RecordingStateManager } from "../src/utils/utils"; const testRedirectedRequest = ( @@ -275,10 +276,13 @@ describe("TestProxyClient functions", () => { }); }); - describe("_createRecordingRequest", () => { - it("_createRecordingRequest adds the recording-file and recording-id headers", () => { - client.recordingId = "dummy-recording-id"; - const returnedRequest = client["_createRecordingRequest"](initialRequest.url); + describe("createRecordingRequest", () => { + it("createRecordingRequest adds the recording-file and recording-id headers", () => { + const returnedRequest = createRecordingRequest( + initialRequest.url, + client["sessionFile"], + client.recordingId + ); expect(returnedRequest.url).to.equal(initialRequest.url); expect(returnedRequest.method).to.equal("POST"); expect(returnedRequest.body).not.to.be.undefined; diff --git a/sdk/test-utils/recorder/test/testProxyTests.spec.ts b/sdk/test-utils/recorder/test/testProxyTests.spec.ts index 5a7c239621b8..a9cb6f3d6f05 100644 --- a/sdk/test-utils/recorder/test/testProxyTests.spec.ts +++ b/sdk/test-utils/recorder/test/testProxyTests.spec.ts @@ -2,8 +2,8 @@ // Licensed under the MIT license. import { ServiceClient } from "@azure/core-client"; -import { isLiveMode, isPlaybackMode, Recorder } from "../src"; -import { TestMode } from "../src/utils/utils"; +import { isPlaybackMode, Recorder } from "../src"; +import { isLiveMode, TestMode } from "../src/utils/utils"; import { getTestServerUrl, makeRequestAndVerifyResponse, setTestMode } from "./utils/utils"; // These tests require the following to be running in parallel @@ -111,12 +111,7 @@ import { getTestServerUrl, makeRequestAndVerifyResponse, setTestMode } from "./u it("transformsInfo()", async () => { if (!isLiveMode()) { await recorder.start({ envSetupForPlayback: {} }); - - if (!recorder["sanitizer"]) { - throw new Error("expected recorder.sanitizer to be defined at this point"); - } - - await recorder["sanitizer"].transformsInfo(); + await recorder.transformsInfo(); } }); }); diff --git a/sdk/test-utils/testing-recorder-new/test/core-v1-test.spec.ts b/sdk/test-utils/testing-recorder-new/test/core-v1-test.spec.ts index 32d9f0396cc7..8c7ec7ca5971 100644 --- a/sdk/test-utils/testing-recorder-new/test/core-v1-test.spec.ts +++ b/sdk/test-utils/testing-recorder-new/test/core-v1-test.spec.ts @@ -16,13 +16,13 @@ const recorderOptions: RecorderStartOptions = { const getSanitizerOptions = () => { return { - generalRegexSanitizers: [ + generalSanitizers: [ { - regex: assertEnvironmentVariable("STORAGE_SAS_URL").split("/")[2], + target: assertEnvironmentVariable("STORAGE_SAS_URL").split("/")[2], value: fakeSASUrl.split("/")[2], }, { - regex: assertEnvironmentVariable("STORAGE_SAS_URL").split("/")[3].split("?")[1], + target: assertEnvironmentVariable("STORAGE_SAS_URL").split("/")[3].split("?")[1], value: fakeSASUrl.split("/")[3].split("?")[1], }, ], diff --git a/sdk/test-utils/testing-recorder-new/test/core-v2-test.spec.ts b/sdk/test-utils/testing-recorder-new/test/core-v2-test.spec.ts index 99293a53e441..0a8eda0a4a42 100644 --- a/sdk/test-utils/testing-recorder-new/test/core-v2-test.spec.ts +++ b/sdk/test-utils/testing-recorder-new/test/core-v2-test.spec.ts @@ -15,7 +15,7 @@ const sanitizerOptions: SanitizerOptions = { }, ], removeHeaderSanitizer: { headersForRemoval: ["X-Content-Type-Options"] }, - generalRegexSanitizers: [{ regex: "abc", value: "fake_abc" }], + generalSanitizers: [{ target: "abc", value: "fake_abc" }], }; const recorderOptions: RecorderStartOptions = { diff --git a/sdk/test-utils/testing-recorder-new/test/noOpCredentialTest.spec.ts b/sdk/test-utils/testing-recorder-new/test/noOpCredentialTest.spec.ts index a34afb135df0..2238820bbd75 100644 --- a/sdk/test-utils/testing-recorder-new/test/noOpCredentialTest.spec.ts +++ b/sdk/test-utils/testing-recorder-new/test/noOpCredentialTest.spec.ts @@ -16,9 +16,9 @@ const getRecorderStartOptions = (): RecorderStartOptions => { AZURE_TENANT_ID: "azuretenantid", }, sanitizerOptions: { - bodyRegexSanitizers: [ + bodySanitizers: [ { - regex: env.TABLES_URL ? encodeURIComponent(env.TABLES_URL) : undefined, + target: encodeURIComponent(env.TABLES_URL ?? ""), value: encodeURIComponent(`https://fakeaccount.table.core.windows.net`), }, ],