Merge branch 'master' of https://github.com/Azure/azure-sdk-for-python …

…into analyze_redesign * 'master' of https://github.com/Azure/azure-sdk-for-python: (35 commits) Sync eng/common directory with azure-sdk-tools for PR 1353 (Azure#16465) Normalize the package name for Doc.Ms readme (Azure#16401) fix changelog and version (Azure#16445) we should always run integration to publish from our artifacts. if the build pipeline crashed too early in the pipeline, this will fail regardless, due to inability to pull the artifacts down (Azure#15058) Fix DateTime bug (Azure#16456) Resolve Regression Failures (Azure#16455) [text analytics] Expose 'string_index_type' parameter in all service client methods where applicable (Azure#16412) adding devtools to the appconfig dev_reqs to solve python core issue (Azure#16381) Copy job matrix functionality (Azure#16450) Add APIView KV variable group to prepare pipelines bot (Azure#16451) [Datalake] Added support for PurePosixPath (Azure#16400) Regenerate baseline because last one break. (Azure#16415) adding step to test for crlf line endings (Azure#16398) [Datalake] Removed list_paths manual paging and deserialization (Azure#16309) Sync eng/common directory with azure-sdk-tools for PR 1351 (Azure#16448) Update auto_codegen.py (Azure#16443) First release purview (Azure#16440) 1ES pools update for release pipeline. (Azure#16419) Add Cloud environment for Teams user (Azure#16359) Sync eng/common directory with azure-sdk-tools for PR 1345 (Azure#16404) ...
iscai-msft · Feb 2, 2021 · 527dc58 · 527dc58
2 parents 3885590 + e1cce3e
commit 527dc58
Show file tree

Hide file tree

Showing 539 changed files with 202,184 additions and 26,419 deletions.
diff --git a/.gitignore b/.gitignore
@@ -33,6 +33,7 @@ build/
 
 # Test results
 TestResults/
+ENV_DIR/
 
 # tox generated artifacts
 test-junit-*.xml

diff --git a/build_package.py b/build_package.py
@@ -9,6 +9,7 @@
 import argparse
 import os
 import glob
+import sys
 from subprocess import check_call
 
 
@@ -19,8 +20,8 @@ def create_package(name, dest_folder=DEFAULT_DEST_FOLDER):
     absdirs = [os.path.dirname(package) for package in (glob.glob('{}/setup.py'.format(name)) + glob.glob('sdk/*/{}/setup.py'.format(name)))]
 
     absdirpath = os.path.abspath(absdirs[0])
-    check_call(['python', 'setup.py', 'bdist_wheel', '-d', dest_folder], cwd=absdirpath)
-    check_call(['python', 'setup.py', "sdist", "--format", "zip", '-d', dest_folder], cwd=absdirpath)
+    check_call([sys.executable, 'setup.py', 'bdist_wheel', '-d', dest_folder], cwd=absdirpath)
+    check_call([sys.executable, 'setup.py', "sdist", "--format", "zip", '-d', dest_folder], cwd=absdirpath)
 
 
 if __name__ == '__main__':

diff --git a/ci_template.yml b/ci_template.yml
@@ -0,0 +1,33 @@
+# DO NOT EDIT THIS FILE
+# This file is generated automatically and any changes will be lost.
+
+trigger:
+  branches:
+    include:
+    - master
+    - hotfix/*
+    - release/*
+    - restapi*
+  paths:
+    include:
+    - sdk/MyService/
+
+pr:
+  branches:
+    include:
+    - master
+    - feature/*
+    - hotfix/*
+    - release/*
+    - restapi*
+  paths:
+    include:
+    - sdk/MyService/
+
+extends:
+  template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml
+  parameters:
+    ServiceDirectory: MyService
+    Artifacts:
+    - name: azure_mgmt_MyService
+      safeName: azuremgmtMyService
diff --git a/eng/common/TestResources/README.md b/eng/common/TestResources/README.md
@@ -30,7 +30,7 @@ Connect-AzAccount -Subscription 'YOUR SUBSCRIPTION ID'
 eng\common\TestResources\New-TestResources.ps1 -ServiceDirectory 'search'
 ```
 
-The `OutFile` switch would be set if you are running this for a .NET project on Windows. This will save test environment settings
+The `OutFile` switch will be set by default if you are running this for a .NET project on Windows. This will save test environment settings
 into a test-resources.json.env file next to test-resources.json. The file is protected via DPAPI.
 The environment file would be scoped to the current repository directory and avoids the need to
 set environment variables or restart your IDE to recognize them.
@@ -89,6 +89,12 @@ Remove-AzADServicePrincipal -ApplicationId $sp.ApplicationId -Force
 
 If you persisted environment variables, you should also remove those as well.
 
+Some test-resources.json templates utilize the `AdditionalParameters` parameter to control additional resource configuration options. For example:
+
+```powershell
+New-TestResources.ps1 keyvault -AdditionalParameters @{enableHsm = $true}
+```
+
 ## In CI
 
 Test pipelines should include deploy-test-resources.yml and

diff --git a/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml b/eng/common/pipelines/templates/steps/daily-dev-build-variable.yml
@@ -1,7 +1,19 @@
 # This script fragment is used across our repos to set a variable "SetDevVersion" which
 # is used when this pipeline is going to be generating and publishing daily dev builds.
-
+parameters:
+  ServiceDirectory: ''
 steps:
+- ${{if ne(parameters.ServiceDirectory, '')}}:
+  - task: Powershell@2
+    inputs:
+      filePath: $(Build.SourcesDirectory)/eng/common/scripts/Save-Package-Properties.ps1
+      arguments: >
+        -ServiceDirectory ${{parameters.ServiceDirectory}}
+        -OutDirectory $(Build.ArtifactStagingDirectory)/PackageInfo
+      pwsh: true
+      workingDirectory: $(Pipeline.Workspace)
+    displayName: Dump Package properties
+    condition: succeeded()
 - pwsh: |
     $setDailyDevBuild = "false"
     if (('$(Build.Reason)' -eq 'Schedule') -and ('$(System.TeamProject)' -eq 'internal')) {

diff --git a/eng/common/scripts/Create-APIReview.ps1 b/eng/common/scripts/Create-APIReview.ps1
@@ -40,6 +40,7 @@ function Submit-APIReview($packagename, $filePath, $uri, $apiKey, $apiLabel)
     try
     {
         $Response = Invoke-WebRequest -Method 'POST' -Uri $uri -Body $multipartContent -Headers $headers
+        Write-Host "API Review: $($Response)"
         $StatusCode = $Response.StatusCode
     }
     catch
@@ -81,23 +82,47 @@ else
 }
 
 $FoundFailure = $False
+$pkgInfoPath = Join-Path -Path $ArtifactPath "PackageInfo"
 foreach ($pkgName in $responses.Keys)
 {    
     $respCode = $responses[$pkgName]
     if ($respCode -ne '200')
     {
-        $FoundFailure = $True
-        if ($respCode -eq '201')
+        $pkgPropPath = Join-Path -Path $pkgInfoPath ($PackageName + ".json")
+        if (-Not (Test-Path $pkgPropPath))
         {
-            Write-Host "API Review is pending for package $pkgName"
+            Write-Host " Package property file path $($pkgPropPath) is invalid."
+            $FoundFailure = $True
         }
         else
         {
-            Write-Host "Failed to create API Review for package $pkgName"
+            $pkgInfo = Get-Content $pkgPropPath | ConvertFrom-Json
+            $version = [AzureEngSemanticVersion]::ParseVersionString($pkgInfo.Version)
+            if ($version.IsPrerelease)
+            {
+                Write-Host "Package version is not GA. Ignoring API view approval status"
+            }
+            elseif ($pkgInfo.SdkType -eq "client" -and $pkgInfo.IsNewSdk)
+            {
+                $FoundFailure = $True
+                if ($respCode -eq '201')
+                {
+                    Write-Error "Automatic API Review approval is pending for package $($PackageName)"
+                }
+                else
+                {
+                    Write-Error "Failed to create API Review for package $($PackageName)"
+                }                
+            }
+            else
+            {
+                Write-Host "API review is not approved for package $($PackageName). Management and track1 package can be released without API review approval."
+            }      
         }
     }
 }
 if ($FoundFailure)
 {
-    Write-Host "Atleast one API review is not yet approved"
+    Write-Error "Automatic API review is not yet approved for package $($PackageName)"
+    exit 1
 }
diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1
@@ -10,6 +10,8 @@ class PackageProps
     [string]$ReadMePath
     [string]$ChangeLogPath
     [string]$Group
+    [string]$SdkType
+    [boolean]$IsNewSdk
 
     PackageProps([string]$name, [string]$version, [string]$directoryPath, [string]$serviceDirectory)
     {

diff --git a/eng/common/scripts/Prepare-Release.ps1 b/eng/common/scripts/Prepare-Release.ps1
@@ -55,7 +55,7 @@ if (!$ReleaseDate)
 }
 else
 {
-    $ParsedReleaseDate = ([datetime]$ReleaseDate, 'MM/dd/yyyy', [Globalization.CultureInfo]::InvariantCulture)
+    $ParsedReleaseDate = [datetime]$ReleaseDate
 }
 
 $releaseDateString = $ParsedReleaseDate.ToString("MM/dd/yyyy")

diff --git a/eng/common/scripts/Save-Package-Properties.ps1 b/eng/common/scripts/Save-Package-Properties.ps1
@@ -0,0 +1,34 @@
+[CmdletBinding()]
+Param (
+  [Parameter(Mandatory=$True)]
+  [string] $serviceDirectory,
+  [Parameter(Mandatory=$True)]
+  [string] $outDirectory
+)
+
+. (Join-Path $PSScriptRoot common.ps1)
+$allPackageProperties = Get-AllPkgProperties $serviceDirectory
+if ($allPackageProperties)
+{
+    New-Item -ItemType Directory -Force -Path $outDirectory
+    foreach($pkg in $allPackageProperties)
+    {
+        if ($pkg.IsNewSDK)
+        {
+            Write-Host "Package Name: $($pkg.Name)"
+            Write-Host "Package Version: $($pkg.Version)"
+            Write-Host "Package SDK Type: $($pkg.SdkType)"
+            $outputPath = Join-Path -Path $outDirectory ($pkg.Name + ".json")
+            $outputObject = $pkg | ConvertTo-Json
+            Set-Content -Path $outputPath -Value $outputObject
+        }        
+    }
+
+    Get-ChildItem -Path $outDirectory
+}
+else
+{
+    Write-Error "Package properties are not available for service directory $($serviceDirectory)"
+    exit 1
+}
+
diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1
@@ -123,14 +123,18 @@ function VerifyPackages($artifactLocation, $workingDirectory, $apiUrl, $releaseS
         Write-Host "Maybe a pkg version wasn't updated properly?"
         exit(1)
       }
-
+      $docsReadMeName = $parsedPackage.PackageId
+      if ($parsedPackage.DocsReadMeName) {
+        $docsReadMeName = $parsedPackage.DocsReadMeName
+      }
       $pkgList += New-Object PSObject -Property @{
         PackageId      = $parsedPackage.PackageId
         PackageVersion = $parsedPackage.PackageVersion
         GroupId        = $parsedPackage.GroupId
         Tag            = $parsedPackage.ReleaseTag
         ReleaseNotes   = $parsedPackage.ReleaseNotes
         ReadmeContent  = $parsedPackage.ReadmeContent
+        DocsReadMeName = $docsReadMeName
         IsPrerelease   = [AzureEngSemanticVersion]::ParseVersionString($parsedPackage.PackageVersion).IsPrerelease
       }
     }

diff --git a/eng/common/scripts/update-docs-metadata.ps1 b/eng/common/scripts/update-docs-metadata.ps1
@@ -103,7 +103,7 @@ foreach ($config in $targets) {
     Write-Host ($pkgsFiltered | % { $_.PackageId + " " + $_.PackageVersion })
 
     foreach ($packageInfo in $pkgsFiltered) {
-      $readmeName = "$($packageInfo.PackageId.Replace('azure-','').Replace('Azure.', '').Replace('@azure/', '').ToLower())-readme${suffix}.md"
+      $readmeName = "$($packageInfo.DocsReadMeName.ToLower())-readme${suffix}.md"
       $readmeFolder = Join-Path $DocRepoLocation $config.content_folder
       $readmeLocation = Join-Path $readmeFolder $readmeName
 

diff --git a/eng/pipelines/aggregate-reports.yml b/eng/pipelines/aggregate-reports.yml
@@ -17,25 +17,8 @@ jobs:
     vmImage: 'windows-2019'
 
   steps:
-  - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3
-    displayName: 'Run CredScan'
-    inputs:
-      suppressionsFile: 'eng\CredScanSuppression.json'
-  - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2
-    displayName: 'Post Analysis'
-    inputs:
-      GdnBreakAllTools: false
-      GdnBreakGdnToolCredScan: true
-      GdnBreakGdnToolCredScanSeverity: Error
-      GdnBreakBaselineFiles: $(Build.SourcesDirectory)\eng\python.gdnbaselines
-      GdnBreakBaselines: baseline     
-    continueOnError: true
-  - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3
-    displayName: 'Publish Security Analysis Logs'
-    continueOnError: true
-    condition: succeededOrFailed()
   - template: /eng/pipelines/templates/steps/analyze_dependency.yml
-
+  
   - task: AzureFileCopy@2
     displayName: 'Upload dependency report'
     condition: and(succeededOrFailed(), eq(variables['System.TeamProject'], 'internal'))
@@ -54,7 +37,29 @@ jobs:
       workingDirectory: $(Build.SourcesDirectory)
       filePath: eng/common/scripts/Verify-Resource-Ref.ps1
 
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3
+    displayName: 'Run CredScan'
+    condition: succeededOrFailed()
+    inputs:
+      suppressionsFile: 'eng\CredScanSuppression.json'
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2
+    displayName: 'Post Analysis'
+    condition: succeededOrFailed()
+    inputs:
+      GdnBreakAllTools: false
+      GdnBreakGdnToolCredScan: true
+      GdnBreakGdnToolCredScanSeverity: Error
+      GdnBreakBaselineFiles: $(Build.SourcesDirectory)\eng\python.gdnbaselines
+      GdnBreakBaselines: baseline   
+      # Used for generating baseline file.   
+      # GdnBreakOutputBaselineFile: python
+      # GdnBreakOutputBaseline: baseline
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3
+    displayName: 'Publish Security Analysis Logs'
+    continueOnError: true
+    condition: succeededOrFailed()
   - template: ../common/pipelines/templates/steps/verify-links.yml
     parameters:
       Directory: ""
       CheckLinkGuidance: $true
+      Condition: succeededOrFailed()
diff --git a/eng/pipelines/prepare-pipelines.yml b/eng/pipelines/prepare-pipelines.yml
@@ -6,5 +6,5 @@ extends:
     Repository: $(Build.Repository.Name)
     Prefix: python
     CIConventionOptions: ''
-    UPConventionOptions: '--variablegroup 58 --variablegroup 76 --variablegroup 56'
+    UPConventionOptions: '--variablegroup 58 --variablegroup 76 --variablegroup 56 --variablegroup 93'
     TestsConventionOptions: '--variablegroup 64'
diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml
@@ -23,7 +23,8 @@ stages:
             environment: github
 
             pool:
-              vmImage: ubuntu-18.04
+              name: azsdk-pool-mms-ubuntu-1804-general
+              vmImage: MMSUbuntu18.04
 
             strategy:
               runOnce:
@@ -70,7 +71,8 @@ stages:
               dependsOn: TagRepository
 
               pool:
-                vmImage: ubuntu-18.04
+                name: azsdk-pool-mms-ubuntu-1804-general
+                vmImage: MMSUbuntu18.04
 
               strategy:
                 runOnce:
@@ -128,7 +130,8 @@ stages:
               dependsOn: PublishPackage
 
               pool:
-                vmImage: windows-2019
+                name: azsdk-pool-mms-win-2019-general
+                vmImage: MMS2019
 
               strategy:
                 runOnce:
@@ -163,7 +166,8 @@ stages:
               dependsOn: PublishPackage
 
               pool:
-                vmImage: ubuntu-18.04
+                name: azsdk-pool-mms-ubuntu-1804-general
+                vmImage: MMSUbuntu18.04
 
               variables:
                 CIConfigs: "{'targets':[{'path_to_config':'ci-configs/packages-preview.json','Mode':'Preview','content_folder':'docs-ref-services/preview'},{'path_to_config':'ci-configs/packages-latest.json','mode':'Latest','content_folder':'docs-ref-services/latest'}]}"
@@ -213,7 +217,8 @@ stages:
               dependsOn: PublishPackage
 
               pool:
-                vmImage: ubuntu-18.04
+                name: azsdk-pool-mms-ubuntu-1804-general
+                vmImage: MMSUbuntu18.04
 
               strategy:
                 runOnce:
@@ -249,12 +254,14 @@ stages:
 
   - stage: Integration
     dependsOn: ${{parameters.DependsOn}}
+    condition: succeededOrFailed('${{parameters.DependsOn}}')
     jobs:
     - job: PublishPackages
       displayName: "Publish package to daily feed"
       condition: or(eq(variables['SetDevVersion'], 'true'), and(eq(variables['Build.Reason'],'Schedule'), eq(variables['System.TeamProject'], 'internal')))
       pool:
-        vmImage: ubuntu-18.04
+        name: azsdk-pool-mms-ubuntu-1804-general
+        vmImage: MMSUbuntu18.04
       steps:
       - checkout: none
       - download: current