Merge branch 'main' of https://github.com/Azure/azure-sdk-for-python …

…into new_metrics_advisor * 'main' of https://github.com/Azure/azure-sdk-for-python: (62 commits) pass polling interval 0 for playback (Azure#23514) [webpubsub] switch to customizations code (Azure#23504) [AutoRelease] t2-netapp-2022-02-11-58164(Do not merge) (Azure#23214) [Key Vault] Address architect feedback (Azure#23475) Sync eng/common directory with azure-sdk-tools for PR 2910 (Azure#23499) check content (Azure#23496) fix cspell issues (Azure#22948) API Key Not Being Passed On Paged Requests (Azure#23477) [AutoRelease] t2-labservices-2022-03-14-64385 (Azure#23492) [Release issue status] fix date bug and serve Python only (Azure#23491) [SchemaRegistry] migrate to test proxy (Azure#23438) [compute] update test (Azure#23456) update servicebus test (Azure#23401) Update search test (Azure#23341) Update resources test (Azure#23340) Update relay test (Azure#23339) Update notificationhubs test (Azure#23338) Update network-dns test (Azure#23337) update network test (Azure#23421) update logic (Azure#23463) ...
iscai-msft · Mar 15, 2022 · e051c27 · e051c27
2 parents 3865d22 + 3f21df3
commit e051c27
Show file tree

Hide file tree

Showing 460 changed files with 204,696 additions and 170,221 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -394,7 +394,7 @@
 #/<NotInRepo>/          @jfggdl
 
 # ServiceLabel: %Event Hubs %Service Attention
-#/<NotInRepo>/          @kasun04
+#/<NotInRepo>/          @kasun04 @saglodha
 
 # ServiceLabel: %Functions %Service Attention
 #/<NotInRepo>/          @ahmedelnably @fabiocav
@@ -580,7 +580,7 @@
 #/<NotInRepo>/          @chlahav
 
 # ServiceLabel: %Service Attention %Service Bus
-#/<NotInRepo>/          @jfggdl @EldertGrootenboer
+#/<NotInRepo>/          @saglodha @EldertGrootenboer
 
 # ServiceLabel: %Service Attention %Service Fabric
 #/<NotInRepo>/          @QingChenmsft @vaishnavk @juhacket

diff --git a/.github/fabricbot.json b/.github/fabricbot.json
@@ -2093,7 +2093,8 @@
             "Event Hubs"
           ],
           "mentionees": [
-            "kasun04"
+            "kasun04",
+            "saglodha"
           ]
         },
         {
@@ -2676,7 +2677,7 @@
             "Service Bus"
           ],
           "mentionees": [
-            "jfggdl",
+            "saglodha",
             "EldertGrootenboer"
           ]
         },
@@ -3936,4 +3937,4 @@
       ]
     }
   }
-]
+]
diff --git a/.vscode/cspell.json b/.vscode/cspell.json
@@ -20,6 +20,7 @@
     "*.tgz",
     "*.pfx",
     "sdk/**/azure-mgmt**/**",
+    "sdk/**/ci.yml",
     "sdk/**/azure-cognitiveservices**",
     "sdk/**/_generated/**",
     "**/*requirement*.txt",

diff --git a/eng/ci_tools.txt b/eng/ci_tools.txt
@@ -2,7 +2,7 @@
 cryptography<4
 setuptools==44.1.0; python_version == '2.7'
 setuptools==46.4.0; python_version >= '3.5'
-virtualenv==20.0.23
+virtualenv==20.13.2
 wheel==0.37.0
 Jinja2==2.11.2
 packaging==20.4
@@ -17,7 +17,6 @@ codecov==2.1.0
 beautifulsoup4==4.9.1
 pkginfo==1.5.0.1
 pip==20.3.3
-black==21.6b0; python_version >= '3.6'
 wrapt<=1.12.1; python_version == '2.7'
 markupsafe==2.0.1; python_version > '2.7'
 markupsafe==1.1.1; python_version == '2.7'

diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1
@@ -236,7 +236,8 @@ function BuildBicepFile([System.IO.FileSystemInfo] $file)
     return $templateFilePath
 }
 
-function BuildDeploymentOutputs([string]$serviceDirectoryPrefix, [object]$azContext, [object]$deployment) {
+function BuildDeploymentOutputs([string]$serviceName, [object]$azContext, [object]$deployment) {
+    $serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName
     # Add default values
     $deploymentOutputs = [Ordered]@{
         "${serviceDirectoryPrefix}CLIENT_ID" = $TestApplicationId;
@@ -249,6 +250,7 @@ function BuildDeploymentOutputs([string]$serviceDirectoryPrefix, [object]$azCont
         "${serviceDirectoryPrefix}AZURE_AUTHORITY_HOST" = $azContext.Environment.ActiveDirectoryAuthority;
         "${serviceDirectoryPrefix}RESOURCE_MANAGER_URL" = $azContext.Environment.ResourceManagerUrl;
         "${serviceDirectoryPrefix}SERVICE_MANAGEMENT_URL" = $azContext.Environment.ServiceManagementUrl;
+        "AZURE_SERVICE_DIRECTORY" = $serviceName.ToUpperInvariant();
     }
 
     MergeHashes $EnvironmentVariables $(Get-Variable deploymentOutputs)
@@ -268,8 +270,7 @@ function BuildDeploymentOutputs([string]$serviceDirectoryPrefix, [object]$azCont
 }
 
 function SetDeploymentOutputs([string]$serviceName, [object]$azContext, [object]$deployment, [object]$templateFile) {
-    $serviceDirectoryPrefix = $serviceName.ToUpperInvariant() + "_"
-    $deploymentOutputs = BuildDeploymentOutputs $serviceDirectoryPrefix $azContext $deployment
+    $deploymentOutputs = BuildDeploymentOutputs $serviceName $azContext $deployment
 
     if ($OutFile) {
         if (!$IsWindows) {
@@ -300,7 +301,7 @@ function SetDeploymentOutputs([string]$serviceName, [object]$azContext, [object]
             $EnvironmentVariables[$key] = $value
 
             if ($CI) {
-                if (ShouldMarkValueAsSecret $serviceDirectoryPrefix $key $value $notSecretValues) {
+                if (ShouldMarkValueAsSecret $serviceName $key $value $notSecretValues) {
                     # Treat all ARM template output variables as secrets since "SecureString" variables do not set values.
                     # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below.
                     LogVsoCommand "##vso[task.setvariable variable=_$key;issecret=true;]$value"
@@ -406,8 +407,7 @@ try {
         Write-Verbose "Location was not set. Using default location for environment: '$Location'"
     }
 
-    if (!$CI) {
-
+    if (!$CI -and $PSCmdlet.ParameterSetName -ne "Provisioner") {
         # Make sure the user is logged in to create a service principal.
         $context = Get-AzContext;
         if (!$context) {

diff --git a/eng/common/TestResources/SubConfig-Helpers.ps1 b/eng/common/TestResources/SubConfig-Helpers.ps1
@@ -1,4 +1,8 @@
-function ShouldMarkValueAsSecret([string]$serviceDirectoryPrefix, [string]$key, [string]$value, [array]$allowedValues = @())
+function BuildServiceDirectoryPrefix([string]$serviceName) {
+    return $serviceName.ToUpperInvariant() + "_"
+}
+
+function ShouldMarkValueAsSecret([string]$serviceName, [string]$key, [string]$value, [array]$allowedValues = @())
 {
     $logOutputNonSecret = @(
         # Environment Variables
@@ -14,6 +18,7 @@ function ShouldMarkValueAsSecret([string]$serviceDirectoryPrefix, [string]$key,
         "RESOURCE_MANAGER_URL",
         "SERVICE_MANAGEMENT_URL",
         "ENDPOINT_SUFFIX",
+        "SERVICE_DIRECTORY",
         # This is used in many places and is harder to extract from the base subscription config, so hardcode it for now.
         "STORAGE_ENDPOINT_SUFFIX",
         # Parameters
@@ -25,6 +30,8 @@ function ShouldMarkValueAsSecret([string]$serviceDirectoryPrefix, [string]$key,
         "ProvisionerApplicationId"
     )
 
+    $serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName
+
     $suffix1 = $key -replace $serviceDirectoryPrefix, ""
     $suffix2 = $key -replace "AZURE_", ""
     $variants = @($key, $suffix1, $suffix2)

diff --git a/eng/common/scripts/Update-DocsMsMetadata.ps1 b/eng/common/scripts/Update-DocsMsMetadata.ps1
@@ -81,13 +81,12 @@ function GetAdjustedReadmeContent($ReadmeContent, $PackageInfo, $PackageMetadata
   # The $PackageMetadata could be $null if there is no associated metadata entry
   # based on how the metadata CSV is filtered
   $service = $PackageInfo.ServiceDirectory.ToLower()
-  if ($PackageMetadata -and $PackageMetadata.ServiceName) {
-    # Normalize service name "Key Vault" -> "keyvault"
+  if ($PackageMetadata -and $PackageMetadata.MSDocService) {
+    # Use MSDocService in csv metadata to override the service directory
     # TODO: Use taxonomy for service name -- https://github.com/Azure/azure-sdk-tools/issues/1442
-    # probably from metadata
-    $service = $PackageMetadata.ServiceName.ToLower().Replace(" ", "")
+    $service = $PackageMetadata.MSDocService
   }
-
+  Write-Host "The service of package: $service"
   # Generate the release tag for use in link substitution
   $tag = "$($PackageInfo.Name)_$($PackageInfo.Version)"
   Write-Host "The tag of package: $tag"

diff --git a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1
@@ -18,7 +18,10 @@ param(
     [string]$Subscription,
 
     # Default to true in Azure Pipelines environments
-    [switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID)
+    [switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID),
+
+    # Optional namespace override, otherwise the shell user or chart annotation will be used
+    [string]$Namespace
 )
 
 . $PSScriptRoot/stress-test-deployment-lib.ps1

diff --git a/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 b/eng/common/scripts/stress-testing/find-all-stress-packages.ps1
@@ -11,7 +11,12 @@ class StressTestPackageInfo {
     [string]$DockerBuildDir
 }
 
-function FindStressPackages([string]$directory, [hashtable]$filters = @{}, [switch]$CI) {
+function FindStressPackages(
+    [string]$directory,
+    [hashtable]$filters = @{},
+    [switch]$CI,
+    [string]$namespaceOverride
+) {
     # Bare minimum filter for stress tests
     $filters['stressTest'] = 'true'
 
@@ -20,7 +25,11 @@ function FindStressPackages([string]$directory, [hashtable]$filters = @{}, [swit
     foreach ($chartFile in $chartFiles) {
         $chart = ParseChart $chartFile
         if (matchesAnnotations $chart $filters) {
-            $packages += NewStressTestPackageInfo -chart $chart -chartFile $chartFile -CI:$CI
+            $packages += NewStressTestPackageInfo `
+                            -chart $chart `
+                            -chartFile $chartFile `
+                            -CI:$CI `
+                            -namespaceOverride $namespaceOverride
         }
     }
 
@@ -41,8 +50,15 @@ function MatchesAnnotations([hashtable]$chart, [hashtable]$filters) {
     return $true
 }
 
-function NewStressTestPackageInfo([hashtable]$chart, [System.IO.FileInfo]$chartFile, [switch]$CI) {
-    $namespace = if ($CI) {
+function NewStressTestPackageInfo(
+    [hashtable]$chart,
+    [System.IO.FileInfo]$chartFile,
+    [switch]$CI,
+    [object]$namespaceOverride
+) {
+    $namespace = if ($namespaceOverride) {
+        $namespaceOverride
+    } elseif ($CI) {
         $chart.annotations.namespace
     } else {
         # Check GITHUB_USER for users in codespaces environments, since the default user is `codespaces` and

diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1
@@ -70,7 +70,8 @@ function DeployStressTests(
     [string]$deployId = 'local',
     [switch]$login,
     [string]$subscription = '',
-    [switch]$CI
+    [switch]$CI,
+    [string]$Namespace
 ) {
     if ($environment -eq 'test') {
         if ($clusterGroup -or $subscription) {
@@ -97,7 +98,7 @@ function DeployStressTests(
     Run helm repo update
     if ($LASTEXITCODE) { return $LASTEXITCODE }
 
-    $pkgs = FindStressPackages -directory $searchDirectory -filters $filters -CI:$CI
+    $pkgs = FindStressPackages -directory $searchDirectory -filters $filters -CI:$CI -namespaceOverride $Namespace
     Write-Host "" "Found $($pkgs.Length) stress test packages:"
     Write-Host $pkgs.Directory ""
     foreach ($pkg in $pkgs) {

diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml
@@ -32,7 +32,8 @@ steps:
       versionSpec: '${{ parameters.PythonVersion }}'
 
   - script: |
-      python -m pip install pip==20.1
+      python -m pip install pip==20.3.3
+      python -m pip install wheel==0.37.0
       pip install -r eng/ci_tools.txt
       pip --version
       pip freeze

diff --git a/eng/pipelines/templates/steps/run_black.yml b/eng/pipelines/templates/steps/run_black.yml
@@ -11,7 +11,7 @@ steps:
     condition: succeededOrFailed()
 
   - script: |
-      pip install -r eng/ci_tools.txt
+      pip install black==21.6b0
     displayName: 'Prep Environment'
     condition: succeededOrFailed()
 

diff --git a/eng/pipelines/templates/steps/seed-virtualenv-wheels.yml b/eng/pipelines/templates/steps/seed-virtualenv-wheels.yml
@@ -14,11 +14,9 @@ steps:
     displayName: Set VIRTUALENV_OVERRIDE_APP_DATA
 
   - pwsh: |
-      # ensure these can be pulled down from pypi.
-      $env:PIP_EXTRA_INDEX_URL="https://pypi.python.org/simple"
-      virtualenv --download --reset-app-data `
-        --pip="${{ parameters.PIP_VERSION }}" `
-        --setuptools="${{ parameters.SETUPTOOLS_VERSION }}" `
-        --wheel=="${{ parameters.WHEEL_VERSION }}" `
-        $(Agent.TempDirectory)/seed_env
+      $(Build.SourcesDirectory)/eng/scripts/seed-virtualenv-wheels.ps1 `
+        -Pip "${{ parameters.PIP_VERSION }}" `
+        -SetupTools "${{ parameters.SETUPTOOLS_VERSION }}" `
+        -Wheel "${{ parameters.WHEEL_VERSION }}" `
+        -SeedDirectory "$(Agent.TempDirectory)/seed_env"
     displayName: Populate the seed dir
diff --git a/eng/scripts/seed-virtualenv-wheels.ps1 b/eng/scripts/seed-virtualenv-wheels.ps1
@@ -0,0 +1,49 @@
+<#
+.SYNOPSIS
+Creates a virtual environment while seeding with specific versions of setuptools, wheel, and pip.
+
+.DESCRIPTION
+Creates a virtual environment with three specifically targeted wheel versions. When used in conjunction with the environment variable 
+VIRTUALENV_OVERRIDE_APP_DATA, further virtualenv creations will leverage these targeted wheels specifically.
+
+.PARAMETER Pip
+The targeted pip version.
+
+.PARAMETER SetupTools
+The targeted setuptools version.
+
+.PARAMETER Wheel
+The targeted wheel version.
+
+.PARAMETER SeedDirectory
+The location of the virtualenv that will be created.
+#>
+param (
+  $Pip,
+  $SetupTools,
+  $Wheel,
+  $SeedDirectory
+)
+
+$attempts = 0
+
+# ensure these can be pulled down from pypi.
+$env:PIP_EXTRA_INDEX_URL="https://pypi.python.org/simple"
+
+while ($attempts -lt 3) {
+  virtualenv --download --reset-app-data `
+    --pip="$Pip" `
+    --setuptools="$SetupTools" `
+    --wheel="$Wheel" `
+    "$SeedDirectory"
+  if ($LASTEXITCODE -eq 0) {
+    break
+  }
+
+  $attempts += 1
+}
+
+if ($LASTEXITCODE -ne 0) {
+  Write-Error "Unable to successfully populate an app data directory."
+  exit $LASTEXITCODE
+}
diff --git a/eng/tox/allowed_pylint_failures.py b/eng/tox/allowed_pylint_failures.py
@@ -55,7 +55,6 @@
     "azure-purview-scanning",
     "azure-purview-catalog",
     "azure-purview-account",
-    "azure-messaging-webpubsubservice",
     "azure-purview-administration",
     "azure-messaging-nspkg",
     "azure-agrifood-farming",

diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini
@@ -315,7 +315,7 @@ deps =
   {[base]deps}
 commands =
     # install API stub generator
-    {envbindir}/python -m pip install "git+https://github.com/azure/azure-sdk-tools.git#subdirectory=packages/python-packages/api-stub-generator&egg=api-stub-generator"
+    {envbindir}/python -m pip install "git+https://github.com/azure/azure-sdk-tools.git@api-stub-generator_v0.2.10#subdirectory=packages/python-packages/api-stub-generator&egg=api-stub-generator"
     {envbindir}/python -m pip freeze
     {envbindir}/python {toxinidir}/../../../eng/tox/run_apistubgen.py -t {toxinidir} -w {envtmpdir} {posargs}
 
@@ -364,4 +364,4 @@ deps =
   jsondiff==1.2.0
   -e {toxinidir}/../../scripts/breaking_changes_checker
 commands =
-    {envbindir}/python {toxinidir}/../../../scripts/breaking_changes_checker/detect_breaking_changes.py -t {toxinidir}
+    {envbindir}/python {toxinidir}/../../../scripts/breaking_changes_checker/detect_breaking_changes.py -t {toxinidir}
diff --git a/scripts/release_issue_status/main.py b/scripts/release_issue_status/main.py
@@ -190,7 +190,8 @@ def main():
     rest_repo = g.get_repo('Azure/azure-rest-api-specs')
     sdk_repo = g.get_repo('Azure/azure-sdk-for-python')
     label1 = request_repo.get_label('ManagementPlane')
-    open_issues = request_repo.get_issues(state='open', labels=[label1])
+    label2 = request_repo.get_label('Python')
+    open_issues = request_repo.get_issues(state='open', labels=[label1, label2])
     issue_status = []
     issue_status_python = []
     duplicated_issue = dict()
@@ -205,9 +206,13 @@ def main():
         issue.link = f'https://github.com/Azure/sdk-release-request/issues/{item.number}'
         issue.author = item.user.login
         issue.package = _extract(item.body.split('\n'), 'azure-.*')
-        issue.target_date = [x.split(':')[-1].strip() for x in item.body.split('\n') if 'Target release date' in x][0]
-        issue.days_from_target = int(
-            (time.mktime(time.strptime(issue.target_date, '%Y-%m-%d')) - time.time()) / 3600 / 24)
+        try:
+            issue.target_date = [x.split(':')[-1].strip() for x in item.body.split('\n') if 'Target release date' in x][0]
+            issue.days_from_target = int(
+                (time.mktime(time.strptime(issue.target_date, '%Y-%m-%d')) - time.time()) / 3600 / 24)
+        except Exception:
+            issue.target_date = 'fail to get.'
+            issue.days_from_target = 1000  # make a ridiculous data to remind failure when error happens
         issue.create_date = item.created_at.timestamp()
         issue.delay_from_create_date = int((time.time() - item.created_at.timestamp()) / 3600 / 24)
         issue.latest_update = item.updated_at.timestamp()

diff --git a/sdk/advisor/azure-mgmt-advisor/tests/conftest.py b/sdk/advisor/azure-mgmt-advisor/tests/conftest.py
@@ -31,6 +31,7 @@
 from dotenv import load_dotenv
 
 from devtools_testutils import test_proxy, add_general_regex_sanitizer
+from devtools_testutils import add_header_regex_sanitizer, add_body_key_sanitizer
 
 # Ignore async tests for Python < 3.5
 collect_ignore_glob = []
@@ -44,4 +45,7 @@ def add_sanitizers(test_proxy):
     subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID", "00000000-0000-0000-0000-000000000000")
     tenant_id = os.environ.get("AZURE_TENANT_ID", "00000000-0000-0000-0000-000000000000")
     add_general_regex_sanitizer(regex=subscription_id, value="00000000-0000-0000-0000-000000000000")
-    add_general_regex_sanitizer(regex=tenant_id, value="00000000-0000-0000-0000-000000000000")
+    add_general_regex_sanitizer(regex=tenant_id, value="00000000-0000-0000-0000-000000000000")
+    add_header_regex_sanitizer(key="Set-Cookie", value="[set-cookie;]")
+    add_header_regex_sanitizer(key="Cookie", value="cookie;")
+    add_body_key_sanitizer(json_path="$..access_token", value="access_token")