Sync eng/common directory with azure-sdk-tools for PR 1463 (Azure#17376)

* Add 'replace' support. Improve handling of dynamic parameter types.

* Support display name import and overrides

* Support regex capture groups for replace. Force fully matching regex.

Co-authored-by: Ben Broderick Phillips <bebroder@microsoft.com>

eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml

@@ -10,6 +10,9 @@ parameters:
 - name: MatrixFilters
   type: object
   default: []
+- name: MatrixReplace
+  type: object
+  default: {}
 - name: JobTemplatePath
   type: string
 # Set this to false to do a full checkout for private repositories with the azure pipelines service connection
@@ -57,6 +60,7 @@ jobs:
               -Selection ${{ config.Selection }}
               -DisplayNameFilter "$(displayNameFilter)"
               -Filters "${{ join('","', parameters.MatrixFilters) }}","container=^$","SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}"
+              -Replace "${{ join('","', parameters.MatrixReplace) }}"
               -NonSparseParameters "${{ join('","', config.NonSparseParameters) }}"
           displayName: Generate VM Job Matrix ${{ config.Name }}
           name: generate_vm_job_matrix_${{ config.Name }}

eng/common/scripts/job-matrix/Create-JobMatrix.ps1

@@ -13,6 +13,7 @@ param (
     [Parameter(Mandatory=$True)][string] $Selection,
     [Parameter(Mandatory=$False)][string] $DisplayNameFilter,
     [Parameter(Mandatory=$False)][array] $Filters,
+    [Parameter(Mandatory=$False)][array] $Replace,
     [Parameter(Mandatory=$False)][array] $NonSparseParameters
 )
 
@@ -27,6 +28,7 @@ $Filters = $Filters | Where-Object { $_ }
     -selectFromMatrixType $Selection `
     -displayNameFilter $DisplayNameFilter `
     -filters $Filters `
+    -replace $Replace `
     -nonSparseParameters $NonSparseParameters
 
 $serialized = SerializePipelineMatrix $matrix

eng/common/scripts/job-matrix/README.md

@@ -14,6 +14,7 @@
     * [include/exclude](#includeexclude)
     * [displayNames](#displaynames-1)
     * [Filters](#filters)
+    * [Replace](#replace-values)
     * [NonSparseParameters](#nonsparseparameters)
     * [Under the hood](#under-the-hood)
 * [Testing](#testing)
@@ -54,6 +55,7 @@ jobs:
         Location: eastus2
         Cloud: Public
       MatrixFilters: []
+      MatrixReplace: []
 ```
 
 ## Matrix config file syntax
@@ -166,7 +168,8 @@ To import a matrix, add a parameter with the key `$IMPORT`:
 ```
 
 Importing can be useful, for example, in cases where there is a shared base matrix, but there is a need to run it
-once for each instance of a language version.
+once for each instance of a language version. Importing does not support overriding duplicate parameters. To achieve
+this, use the [Replace](#replace-values) argument instead.
 
 The processing order is as follows:
 
@@ -376,7 +379,7 @@ The logic for generating display names works like this:
 
 #### Filters
 
-Filters can be passed to the matrix as an array of strings, each matching the format of <key>=<regex>. When a matrix entry
+Filters can be passed to the matrix as an array of strings, each matching the format of `<key>=<regex>`. When a matrix entry
 does not contain the specified key, it will default to a value of empty string for regex parsing. This can be used to specify
 filters for keys that don't exist or keys that optionally exist and match a regex, as seen in the below example.
 
@@ -394,6 +397,71 @@ named "ExcludedKey", a framework variable containing either "461" or "5.0", and
   -Filters @("ExcludedKey=^$", "framework=(461|5\.0)", "SupportedClouds=^$|.*Public.*")
 ```
 
+#### Replace values
+
+Replacements for values can be passed to the matrix as an array of strings, each matching the format of `<keyRegex>=<valueRegex>/<replacementValue>`.
+The replace argument will find any permutations where the key fully matches the key regex and the value fully matches the value regex, and replace the value with
+the replacement specified.
+
+NOTE:
+- The replacement value supports regex capture groups, enabling substring transformations, e.g. `Foo=(.*)-replaceMe/$1-replaced`. See the below examples for usage.
+- For each key/value, the first replacement provided that matches will be the only one applied.
+- If `=` or `/` characters need to be part of the regex or replacement, escape them with `\`.
+
+For example, given a matrix config like below:
+
+```
+{
+  "matrix": {
+    "Agent": {
+      "ubuntu-1804": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" }
+    },
+    "JavaTestVersion": [ "1.8", "1.11" ]
+  }
+}
+
+```
+
+The normal matrix output (without replacements), looks like:
+
+```
+$ ./Create-JobMatrix.ps1 -ConfigPath <test> -Selection all
+{
+  "ubuntu1804_18": {
+    "OSVmImage": "MMSUbuntu18.04",
+    "Pool": "azsdk-pool-mms-ubuntu-1804-general",
+    "JavaTestVersion": "1.8"
+  },
+  "ubuntu1804_111": {
+    "OSVmImage": "MMSUbuntu18.04",
+    "Pool": "azsdk-pool-mms-ubuntu-1804-general",
+    "JavaTestVersion": "1.11"
+  }
+}
+```
+
+Passing in multiple replacements, the output will look like below. Note that replacing key/values that appear nested within a grouping
+will not affect that segment of the job name, since the job takes the grouping name (in this case "ubuntu1804").
+
+The below example includes samples of regex grouping references, and wildcard key/value regexes:
+
+```
+$ $replacements = @('.*Version=1.11/2.0', 'Pool=(.*ubuntu.*)-general/$1-custom')
+$ ../Create-JobMatrix.ps1 -ConfigPath ./test.Json -Selection all -Replace $replacements
+{
+  "ubuntu1804_18": {
+    "OSVmImage": "MMSUbuntu18.04",
+    "Pool": "azsdk-pool-mms-ubuntu-1804-custom",
+    "JavaTestVersion": "1.8"
+  },
+  "ubuntu1804_20": {
+    "OSVmImage": "MMSUbuntu18.04",
+    "Pool": "azsdk-pool-mms-ubuntu-1804-custom",
+    "JavaTestVersion": "2.0"
+  }
+}
+```
+
 #### NonSparseParameters
 
 Sometimes it may be necessary to generate a sparse matrix, but keep the full combination of a few parameters. The