From fbd6ec716bc7e87e465c7b5c08806c1440d70a0b Mon Sep 17 00:00:00 2001 From: Praveen Kuttappan Date: Thu, 7 Mar 2024 17:05:47 -0500 Subject: [PATCH] Create api review script changes to add a param to set release tag --- .../templates/steps/create-apireview.yml | 19 +- eng/common/scripts/Create-APIReview.ps1 | 307 ++++++++++++------ .../scripts/Helpers/ApiView-Helpers.ps1 | 87 ++++- 3 files changed, 300 insertions(+), 113 deletions(-) diff --git a/eng/common/pipelines/templates/steps/create-apireview.yml b/eng/common/pipelines/templates/steps/create-apireview.yml index e8500694377..c69d05d5ae3 100644 --- a/eng/common/pipelines/templates/steps/create-apireview.yml +++ b/eng/common/pipelines/templates/steps/create-apireview.yml @@ -2,28 +2,37 @@ parameters: ArtifactPath: $(Build.ArtifactStagingDirectory) Artifacts: [] ConfigFileDir: $(Build.ArtifactStagingDirectory)/PackageInfo + MarkPackageAsShipped: false + GenerateApiReviewForManualOnly: false + ArtifactName: 'packages' + PackageName: '' steps: # ideally this should be done as initial step of a job in caller template # We can remove this step later once it is added in caller - template: /eng/common/pipelines/templates/steps/set-default-branch.yml - - ${{ each artifact in parameters.Artifacts }}: + # Automatic API review is generated for a package when pipeline runs irrespective of how pipeline gets triggered. + # Below condition ensures that API review is generated only for manual pipeline runs when flag GenerateApiReviewForManualOnly is set to true. + - ${{ if or(ne(parameters.GenerateApiReviewForManualOnly, true), eq(variables['Build.Reason'], 'Manual')) }}: - task: Powershell@2 inputs: filePath: $(Build.SourcesDirectory)/eng/common/scripts/Create-APIReview.ps1 arguments: > + -ArtifactList ('${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json | Select-Object Name) -ArtifactPath ${{parameters.ArtifactPath}} - -APIViewUri $(azuresdk-apiview-uri) + -ArtifactName ${{ parameters.ArtifactName }} -APIKey $(azuresdk-apiview-apikey) - -APILabel "Auto Review - $(Build.SourceVersion)" - -PackageName ${{artifact.name}} + -PackageName '${{parameters.PackageName}}' -SourceBranch $(Build.SourceBranchName) -DefaultBranch $(DefaultBranch) -ConfigFileDir '${{parameters.ConfigFileDir}}' + -BuildId $(Build.BuildId) + -RepoName '$(Build.Repository.Name)' + -MarkPackageAsShipped $${{parameters.MarkPackageAsShipped}} pwsh: true workingDirectory: $(Pipeline.Workspace) - displayName: Create API Review for ${{ artifact.name}} + displayName: Create API Review condition: >- and( succeededOrFailed(), diff --git a/eng/common/scripts/Create-APIReview.ps1 b/eng/common/scripts/Create-APIReview.ps1 index 3d1f549458b..cdf46718026 100644 --- a/eng/common/scripts/Create-APIReview.ps1 +++ b/eng/common/scripts/Create-APIReview.ps1 @@ -1,27 +1,37 @@ [CmdletBinding()] Param ( [Parameter(Mandatory=$True)] - [string] $ArtifactPath, + [array] $ArtifactList, [Parameter(Mandatory=$True)] - [string] $APIViewUri, + [string] $ArtifactPath, [Parameter(Mandatory=$True)] - [string] $APIKey, - [Parameter(Mandatory=$True)] - [string] $APILabel, - [string] $PackageName, + [string] $APIKey, [string] $SourceBranch, [string] $DefaultBranch, - [string] $ConfigFileDir = "" + [string] $RepoName, + [string] $BuildId, + [string] $PackageName = "", + [string] $ConfigFileDir = "", + [string] $APIViewUri = "https://apiview.dev/AutoReview", + [string] $ArtifactName = "packages", + [bool] $MarkPackageAsShipped = $false ) +Set-StrictMode -Version 3 +. (Join-Path $PSScriptRoot common.ps1) +. (Join-Path $PSScriptRoot Helpers ApiView-Helpers.ps1) + # Submit API review request and return status whether current revision is approved or pending or failed to create review -function Submit-APIReview($packagename, $filePath, $uri, $apiKey, $apiLabel, $releaseStatus, $packageVersion) +function Upload-SourceArtifact($filePath, $apiLabel, $releaseStatus, $packageVersion) { + Write-Host "File path: $filePath" + $fileName = Split-Path -Leaf $filePath + Write-Host "File name: $fileName" $multipartContent = [System.Net.Http.MultipartFormDataContent]::new() $FileStream = [System.IO.FileStream]::new($filePath, [System.IO.FileMode]::Open) $fileHeader = [System.Net.Http.Headers.ContentDispositionHeaderValue]::new("form-data") $fileHeader.Name = "file" - $fileHeader.FileName = $packagename + $fileHeader.FileName = $fileName $fileContent = [System.Net.Http.StreamContent]::new($FileStream) $fileContent.Headers.ContentDisposition = $fileHeader $fileContent.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse("application/octet-stream") @@ -41,6 +51,13 @@ function Submit-APIReview($packagename, $filePath, $uri, $apiKey, $apiLabel, $re $versionContent.Headers.ContentDisposition = $versionParam $multipartContent.Add($versionContent) Write-Host "Request param, packageVersion: $packageVersion" + + $releaseTagParam = [System.Net.Http.Headers.ContentDispositionHeaderValue]::new("form-data") + $releaseTagParam.Name = "setReleaseTag" + $releaseTagParamContent = [System.Net.Http.StringContent]::new($MarkPackageAsShipped) + $releaseTagParamContent.Headers.ContentDisposition = $releaseTagParam + $multipartContent.Add($releaseTagParamContent) + Write-Host "Request param, setReleaseTag: $MarkPackageAsShipped" if ($releaseStatus -and ($releaseStatus -ne "Unreleased")) { @@ -52,6 +69,7 @@ function Submit-APIReview($packagename, $filePath, $uri, $apiKey, $apiLabel, $re Write-Host "Request param, compareAllRevisions: true" } + $uri = "${APIViewUri}/UploadAutoReview" $headers = @{ "ApiKey" = $apiKey; "content-type" = "multipart/form-data" @@ -60,7 +78,6 @@ function Submit-APIReview($packagename, $filePath, $uri, $apiKey, $apiLabel, $re try { $Response = Invoke-WebRequest -Method 'POST' -Uri $uri -Body $multipartContent -Headers $headers - Write-Host "API Review URL: $($Response.Content)" $StatusCode = $Response.StatusCode } catch @@ -72,119 +89,225 @@ function Submit-APIReview($packagename, $filePath, $uri, $apiKey, $apiLabel, $re return $StatusCode } +function Upload-ReviewTokenFile($packageName, $apiLabel, $releaseStatus, $reviewFileName, $packageVersion) +{ + $params = "buildId=${BuildId}&artifactName=${ArtifactName}&originalFilePath=${packageName}&reviewFilePath=${reviewFileName}" + $params += "&label=${apiLabel}&repoName=${RepoName}&packageName=${packageName}&project=internal&packageVersion=${packageVersion}" + if($MarkPackageAsShipped) { + $params += "&setReleaseTag=true" + } + $uri = "${APIViewUri}/CreateApiReview?${params}" + if ($releaseStatus -and ($releaseStatus -ne "Unreleased")) + { + $uri += "&compareAllRevisions=true" + } -. (Join-Path $PSScriptRoot common.ps1) + Write-Host "Request to APIView: $uri" + $headers = @{ + "ApiKey" = $APIKey; + } -Write-Host "Artifact path: $($ArtifactPath)" -Write-Host "Package Name: $($PackageName)" -Write-Host "Source branch: $($SourceBranch)" -Write-Host "Config File directory: $($ConfigFileDir)" + try + { + $Response = Invoke-WebRequest -Method 'GET' -Uri $uri -Headers $headers + $StatusCode = $Response.StatusCode + } + catch + { + Write-Host "Exception details: $($_.Exception)" + $StatusCode = $_.Exception.Response.StatusCode + } -$packages = @{} -if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn")) -{ - $packages = &$FindArtifactForApiReviewFn $ArtifactPath $PackageName + return $StatusCode } -else + +function Get-APITokenFileName($packageName) { - Write-Host "The function for 'FindArtifactForApiReviewFn' was not found.` - Make sure it is present in eng/scripts/Language-Settings.ps1 and referenced in eng/common/scripts/common.ps1.` - See https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/common_engsys.md#code-structure" - exit(1) + $reviewTokenFileName = "${packageName}_${LanguageShort}.json" + $tokenFilePath = Join-Path $ArtifactPath $packageName $reviewTokenFileName + if (Test-Path $tokenFilePath) { + Write-Host "Review token file is present at $tokenFilePath" + return $reviewTokenFileName + } + else { + Write-Host "Review token file is not present at $tokenFilePath" + return $null + } } -# Check if package config file is present. This file has package version, SDK type etc info. -if (-not $ConfigFileDir) +function Submit-APIReview($packageInfo, $packagePath) { - $ConfigFileDir = Join-Path -Path $ArtifactPath "PackageInfo" + $packageName = $packageInfo.Name + $apiLabel = "Source Branch:${SourceBranch}" + + # Get generated review token file if present + # APIView processes request using different API if token file is already generated + $reviewTokenFileName = Get-APITokenFileName $packageName + if ($reviewTokenFileName) { + Write-Host "Uploading review token file $reviewTokenFileName to APIView." + return Upload-ReviewTokenFile $packageName $apiLabel $packageInfo.ReleaseStatus $reviewTokenFileName $packageInfo.Version + } + else { + Write-Host "Uploading $packagePath to APIView." + return Upload-SourceArtifact $packagePath $apiLabel $packageInfo.ReleaseStatus $packageInfo.Version + } } -if ($packages) + +function ProcessPackage($packageName) { - foreach($pkgPath in $packages.Values) + $packages = @{} + if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn")) { - $pkg = Split-Path -Leaf $pkgPath - $pkgPropPath = Join-Path -Path $ConfigFileDir "$PackageName.json" - if (-Not (Test-Path $pkgPropPath)) - { - Write-Host " Package property file path $($pkgPropPath) is invalid." - continue - } - # Get package info from json file created before updating version to daily dev - $pkgInfo = Get-Content $pkgPropPath | ConvertFrom-Json - $version = [AzureEngSemanticVersion]::ParseVersionString($pkgInfo.Version) - $versionString = $pkgInfo.Version - if ($version -eq $null) - { - Write-Host "Version info is not available for package $PackageName, because version '$(pkgInfo.Version)' is invalid. Please check if the version follows Azure SDK package versioning guidelines." - exit 1 - } - - Write-Host "Version: $($version)" - Write-Host "SDK Type: $($pkgInfo.SdkType)" - Write-Host "Release Status: $($pkgInfo.ReleaseStatus)" + $packages = &$FindArtifactForApiReviewFn $ArtifactPath $packageName + } + else + { + Write-Host "The function for 'FindArtifactForApiReviewFn' was not found.` + Make sure it is present in eng/scripts/Language-Settings.ps1 and referenced in eng/common/scripts/common.ps1.` + See https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/common_engsys.md#code-structure" + return 1 + } - # Run create review step only if build is triggered from main branch or if version is GA. - # This is to avoid invalidating review status by a build triggered from feature branch - if ( ($SourceBranch -eq $DefaultBranch) -or (-not $version.IsPrerelease)) + if ($packages) + { + foreach($pkgPath in $packages.Values) { - Write-Host "Submitting API Review for package $($pkg)" - $respCode = Submit-APIReview -packagename $pkg -filePath $pkgPath -uri $APIViewUri -apiKey $APIKey -apiLabel $APILabel -releaseStatus $pkgInfo.ReleaseStatus -packageVersion $versionString - Write-Host "HTTP Response code: $($respCode)" - # HTTP status 200 means API is in approved status - if ($respCode -eq '200') - { - Write-Host "API review is in approved status." - } - elseif ($version.IsPrerelease) + $pkg = Split-Path -Leaf $pkgPath + $pkgPropPath = Join-Path -Path $ConfigFileDir "$packageName.json" + if (-Not (Test-Path $pkgPropPath)) { - # Check if package name is approved. Preview version cannot be released without package name approval - if ($respCode -eq '202' -and $pkgInfo.ReleaseStatus -and $pkgInfo.ReleaseStatus -ne "Unreleased") - { - Write-Host "Package name is not yet approved on APIView for $($PackageName). Package name must be approved by an API approver for a beta release if it was never released a stable version." - Write-Host "You can check http://aka.ms/azsdk/engsys/apireview/faq for more details on package name approval." - exit 1 - } - # Ignore API review status for prerelease version - Write-Host "Package version is not GA. Ignoring API view approval status" + Write-Host " Package property file path $($pkgPropPath) is invalid." + continue } - elseif (!$pkgInfo.ReleaseStatus -or $pkgInfo.ReleaseStatus -eq "Unreleased") + # Get package info from json file created before updating version to daily dev + $pkgInfo = Get-Content $pkgPropPath | ConvertFrom-Json + $version = [AzureEngSemanticVersion]::ParseVersionString($pkgInfo.Version) + if ($version -eq $null) { - Write-Host "Release date is not set for current version in change log file for package. Ignoring API review approval status since package is not yet ready for release." + Write-Host "Version info is not available for package $packageName, because version '$(pkgInfo.Version)' is invalid. Please check if the version follows Azure SDK package versioning guidelines." + return 1 } - else + + Write-Host "Version: $($version)" + Write-Host "SDK Type: $($pkgInfo.SdkType)" + Write-Host "Release Status: $($pkgInfo.ReleaseStatus)" + + # Run create review step only if build is triggered from main branch or if version is GA. + # This is to avoid invalidating review status by a build triggered from feature branch + if ( ($SourceBranch -eq $DefaultBranch) -or (-not $version.IsPrerelease) -or $MarkPackageAsShipped) { - # Return error code if status code is 201 for new data plane package - # Temporarily enable API review for spring SDK types. Ideally this should be done be using 'IsReviewRequired' method in language side - # to override default check of SDK type client - if (($pkgInfo.SdkType -eq "client" -or $pkgInfo.SdkType -eq "spring") -and $pkgInfo.IsNewSdk) + Write-Host "Submitting API Review request for package $($pkg), File path: $($pkgPath)" + $respCode = Submit-APIReview $pkgInfo $pkgPath + Write-Host "HTTP Response code: $($respCode)" + + # no need to check API review status when marking a package as shipped + if ($MarkPackageAsShipped) { - if ($respCode -eq '201') + if ($respCode -eq '500') { - Write-Host "Package version $($version) is GA and automatic API Review is not yet approved for package $($PackageName)." - Write-Host "Build and release is not allowed for GA package without API review approval." - Write-Host "You will need to queue another build to proceed further after API review is approved" - Write-Host "You can check http://aka.ms/azsdk/engsys/apireview/faq for more details on API Approval." + Write-Host "Failed to mark package ${packageName} as released. Please reach out to Azure SDK engineering systems on teams channel." + return 1 } - else + Write-Host "Package ${packageName} is marked as released." + return 0 + } + + $apiStatus = [PSCustomObject]@{ + IsApproved = $false + Details = "" + } + $pkgNameStatus = [PSCustomObject]@{ + IsApproved = $false + Details = "" + } + Process-ReviewStatusCode $respCode $packageName $apiStatus $pkgNameStatus + + if ($apiStatus.IsApproved) { + Write-Host "API status: $($apiStatus.Details)" + } + elseif (!$pkgInfo.ReleaseStatus -or $pkgInfo.ReleaseStatus -eq "Unreleased") { + Write-Host "Release date is not set for current version in change log file for package. Ignoring API review approval status since package is not yet ready for release." + } + elseif ($version.IsPrerelease) + { + # Check if package name is approved. Preview version cannot be released without package name approval + if (!$pkgNameStatus.IsApproved) { - Write-Host "Failed to create API Review for package $($PackageName). Please reach out to Azure SDK engineering systems on teams channel and share this build details." + Write-Error $($pkgNameStatus.Details) + return 1 } - exit 1 - } + # Ignore API review status for prerelease version + Write-Host "Package version is not GA. Ignoring API view approval status" + } else { - Write-Host "API review is not approved for package $($PackageName), however it is not required for this package type so it can still be released without API review approval." + # Return error code if status code is 201 for new data plane package + # Temporarily enable API review for spring SDK types. Ideally this should be done be using 'IsReviewRequired' method in language side + # to override default check of SDK type client + if (($pkgInfo.SdkType -eq "client" -or $pkgInfo.SdkType -eq "spring") -and $pkgInfo.IsNewSdk) + { + if (!$apiStatus.IsApproved) + { + Write-Host "Package version $($version) is GA and automatic API Review is not yet approved for package $($packageName)." + Write-Host "Build and release is not allowed for GA package without API review approval." + Write-Host "You will need to queue another build to proceed further after API review is approved" + Write-Host "You can check http://aka.ms/azsdk/engsys/apireview/faq for more details on API Approval." + } + return 1 + } + else { + Write-Host "API review is not approved for package $($packageName), however it is not required for this package type so it can still be released without API review approval." + } } } + else { + Write-Host "Build is triggered from $($SourceBranch) with prerelease version. Skipping API review status check." + } } - else - { - Write-Host "Build is triggered from $($SourceBranch) with prerelease version. Skipping API review status check." - } } + else { + Write-Host "No package is found in artifact path to submit review request" + } + return 0 +} + +$responses = @{} +# Check if package config file is present. This file has package version, SDK type etc info. +if (-not $ConfigFileDir) +{ + $ConfigFileDir = Join-Path -Path $ArtifactPath "PackageInfo" +} + +Write-Host "Artifact path: $($ArtifactPath)" +Write-Host "Source branch: $($SourceBranch)" +Write-Host "Config File directory: $($ConfigFileDir)" + +# if package name param is not empty then process only that package +if ($PackageName) +{ + Write-Host "Processing $($PackageName)" + $result = ProcessPackage -packageName $PackageName + $responses[$PackageName] = $result } else { - Write-Host "No package is found in artifact path to submit review request" + # process all packages in the artifact + foreach ($artifact in $ArtifactList) + { + Write-Host "Processing $($artifact.name)" + $result = ProcessPackage -packageName $artifact.name + $responses[$artifact.name] = $result + } +} + +$exitCode = 0 +foreach($pkg in $responses.keys) +{ + if ($responses[$pkg] -eq 1) + { + Write-Host "API changes are not approved for $($pkg)" + $exitCode = 1 + } } +exit $exitCode \ No newline at end of file diff --git a/eng/common/scripts/Helpers/ApiView-Helpers.ps1 b/eng/common/scripts/Helpers/ApiView-Helpers.ps1 index 73144204f4e..bf8b16a99e0 100644 --- a/eng/common/scripts/Helpers/ApiView-Helpers.ps1 +++ b/eng/common/scripts/Helpers/ApiView-Helpers.ps1 @@ -20,7 +20,7 @@ function MapLanguageName($language) return $lang } -function Check-ApiReviewStatus($packageName, $packageVersion, $language, $url, $apiKey) +function Check-ApiReviewStatus($packageName, $packageVersion, $language, $url, $apiKey, $apiApprovalStatus = $null, $packageNameStatus = $null) { # Get API view URL and API Key to check status Write-Host "Checking API review status" @@ -35,31 +35,86 @@ function Check-ApiReviewStatus($packageName, $packageVersion, $language, $url, $ packageVersion = $packageVersion } + if (!$apiApprovalStatus) { + $apiApprovalStatus = [PSCustomObject]@{ + IsApproved = $false + Details = "" + } + } + + if (!$packageNameStatus) { + $packageNameStatus = [PSCustomObject]@{ + IsApproved = $false + Details = "" + } + } + try { $response = Invoke-WebRequest $url -Method 'GET' -Headers $headers -Body $body - if ($response.StatusCode -eq '200') - { - Write-Host "API Review is approved for package $($packageName)" + Process-ReviewStatusCode -statusCode $response.StatusCode -packageName $packageName -apiApprovalStatus $apiApprovalStatus -packageNameStatus $packageNameStatus + if ($apiApprovalStatus.IsApproved) { + Write-Host $($apiApprovalStatus.Details) } - elseif ($response.StatusCode -eq '202') - { - Write-Host "Package name $($packageName) is not yet approved by an SDK API approver. Package name must be approved to release a beta version if $($packageName) was never released a stable version." - Write-Host "You can check http://aka.ms/azsdk/engsys/apireview/faq for more details on package name Approval." + else { + Write-warning $($apiApprovalStatus.Details) } - elseif ($response.StatusCode -eq '201') - { - Write-Warning "API Review is not approved for package $($packageName). Release pipeline will fail if API review is not approved for a stable version release." - Write-Host "You can check http://aka.ms/azsdk/engsys/apireview/faq for more details on API Approval." + if ($packageNameStatus.IsApproved) { + Write-Host $($packageNameStatus.Details) } - else - { - Write-Warning "API review status check returned unexpected response. $($response)" - Write-Host "You can check http://aka.ms/azsdk/engsys/apireview/faq for more details on API Approval." + else { + Write-warning $($packageNameStatus.Details) } } catch { Write-Warning "Failed to check API review status for package $($PackageName). You can check http://aka.ms/azsdk/engsys/apireview/faq for more details on API Approval." } +} + +function Process-ReviewStatusCode($statusCode, $packageName, $apiApprovalStatus, $packageNameStatus) +{ + $apiApproved = $false + $apiApprovalDetails = "API Review is not approved for package $($packageName). Release pipeline will fail if API review is not approved for a GA version release. You can check http://aka.ms/azsdk/engsys/apireview/faq for more details on API Approval." + + $packageNameApproved = $false + $packageNameApprovalDetails = "" + + # 200 API approved and Package name approved + # 201 API review is not approved, Package name is approved + # 202 API review is not approved, Package name is not approved + + switch ($statusCode) + { + 200 + { + $apiApprovalDetails = "API Review is approved for package $($packageName)" + $apiApproved = $true + + $packageNameApproved = $true + $packageNameApprovalDetails = "Package name is approved for package $($packageName)" + } + 201 + { + $packageNameApproved = $true + $packageNameApprovalDetails = "Package name is approved for package $($packageName)" + } + 202 + { + $packageNameApprovalDetails = "Package name $($packageName) is not yet approved by an SDK API approver. Package name must be approved to release a beta version if $($packageName) was never released as a stable version." + $packageNameApprovalDetails += " You can check http://aka.ms/azsdk/engsys/apireview/faq for more details on package name Approval." + } + default + { + $apiApprovalDetails = "Invalid status code from APIView. status code $($statusCode)" + $packageNameApprovalDetails = "Invalid status code from APIView. status code $($statusCode)" + Write-Error "Failed to process API Review status for for package $($PackageName). Please reach out to Azure SDK engineering systems on teams channel." + } + } + + $apiApprovalStatus.IsApproved = $apiApproved + $apiApprovalStatus.Details = $apiApprovalDetails + + $packageNameStatus.IsApproved = $packageNameApproved + $packageNameStatus.Details = $packageNameApprovalDetails } \ No newline at end of file