Skip to content

Commit

Permalink
Pipeline auto-discover stress test (Azure#21917)
Browse files Browse the repository at this point in the history 
Co-authored-by: Albert Cheng <albertcheng@microsoft.com>
  • Loading branch information
@azure-sdk @ckairen @hildurhodd
2 people authored and hildurhodd committed Jan 4, 2022
1 parent 2892999 commit 6a42aff
Show file tree
Hide file tree
Showing 2 changed files with 225 additions and 208 deletions.
211 changes: 3 additions & 208 deletions eng/common/scripts/stress-testing/deploy-stress-tests.ps1
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
[CmdletBinding(DefaultParameterSetName = 'Default')]
param(
[string]$SearchDirectory,
[hashtable]$Filters,
Expand All @@ -18,211 +17,7 @@ param(
[switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID)
)

$ErrorActionPreference = 'Stop'
. $PSScriptRoot/stress-test-deployment-lib.ps1

. $PSScriptRoot/find-all-stress-packages.ps1
$FailedCommands = New-Object Collections.Generic.List[hashtable]

. (Join-Path $PSScriptRoot "../Helpers" PSModule-Helpers.ps1)

# Powershell does not (at time of writing) treat exit codes from external binaries
# as cause for stopping execution, so do this via a wrapper function.
# See https://github.com/PowerShell/PowerShell-RFC/pull/277
function Run()
{
Write-Host "`n==> $args`n" -ForegroundColor Green
$command, $arguments = $args
& $command $arguments
if ($LASTEXITCODE) {
Write-Error "Command '$args' failed with code: $LASTEXITCODE" -ErrorAction 'Continue'
$FailedCommands.Add(@{ command = "$args"; code = $LASTEXITCODE })
}
}

function RunOrExitOnFailure()
{
run @args
if ($LASTEXITCODE) {
exit $LASTEXITCODE
}
}

function Login([string]$subscription, [string]$clusterGroup, [boolean]$pushImages)
{
Write-Host "Logging in to subscription, cluster and container registry"
az account show *> $null
if ($LASTEXITCODE) {
RunOrExitOnFailure az login --allow-no-subscriptions
}

# Discover cluster name, only one cluster per group is expected
Write-Host "Listing AKS cluster in $subscription/$clusterGroup"
$cluster = RunOrExitOnFailure az aks list -g $clusterGroup --subscription $subscription -o json
$clusterName = ($cluster | ConvertFrom-Json).name

RunOrExitOnFailure az aks get-credentials `
-n "$clusterName" `
-g "$clusterGroup" `
--subscription "$subscription" `
--overwrite-existing

if ($pushImages) {
$registry = RunOrExitOnFailure az acr list -g $clusterGroup --subscription $subscription -o json
$registryName = ($registry | ConvertFrom-Json).name
RunOrExitOnFailure az acr login -n $registryName
}
}

function DeployStressTests(
[string]$searchDirectory = '.',
[hashtable]$filters = @{},
[string]$environment = 'test',
[string]$repository = 'images',
[boolean]$pushImages = $false,
[string]$clusterGroup = 'rg-stress-cluster-test',
[string]$deployId = 'local',
[string]$subscription = 'Azure SDK Developer Playground'
) {
if ($PSCmdlet.ParameterSetName -eq 'DoLogin') {
Login $subscription $clusterGroup $pushImages
}

RunOrExitOnFailure helm repo add stress-test-charts https://stresstestcharts.blob.core.windows.net/helm/
Run helm repo update
if ($LASTEXITCODE) { return $LASTEXITCODE }

$pkgs = FindStressPackages $searchDirectory $filters $CI
Write-Host "" "Found $($pkgs.Length) stress test packages:"
Write-Host $pkgs.Directory ""
foreach ($pkg in $pkgs) {
Write-Host "Deploying stress test at '$($pkg.Directory)'"
DeployStressPackage $pkg $deployId $environment $repository $pushImages
}

Write-Host "Releases deployed by $deployId"
Run helm list --all-namespaces -l deployId=$deployId

if ($FailedCommands) {
Write-Warning "The following commands failed:"
foreach ($cmd in $FailedCommands) {
Write-Error "'$($cmd.command)' failed with code $($cmd.code)" -ErrorAction 'Continue'
}
exit 1
}

Write-Host "`nStress test telemetry links (dashboard, fileshare, etc.): https://aka.ms/azsdk/stress/dashboard"
}

function DeployStressPackage(
[object]$pkg,
[string]$deployId,
[string]$environment,
[string]$repository,
[boolean]$pushImages
) {
$registry = RunOrExitOnFailure az acr list -g $clusterGroup --subscription $subscription -o json
$registryName = ($registry | ConvertFrom-Json).name

Run helm dependency update $pkg.Directory
if ($LASTEXITCODE) { return }

if (Test-Path "$($pkg.Directory)/stress-test-resources.bicep") {
Run az bicep build -f "$($pkg.Directory)/stress-test-resources.bicep"
if ($LASTEXITCODE) { return }
}

if ($pushImages) {
$dockerFiles = Get-ChildItem "$($pkg.Directory)/Dockerfile*"
foreach ($dockerFile in $dockerFiles) {
# Infer docker image name from parent directory name, if file is named `Dockerfile`
# or from suffix, is file is named like `Dockerfile.myimage` (for multiple dockerfiles).
$prefix, $imageName = $dockerFile.Name.Split(".")
if (!$imageName) {
$imageName = $dockerFile.Directory.Name
}
$imageTag = "${registryName}.azurecr.io/$($repository.ToLower())/$($imageName):$deployId"
Write-Host "Building and pushing stress test docker image '$imageTag'"
Run docker build -t $imageTag -f $dockerFile.FullName $dockerFile.DirectoryName
if ($LASTEXITCODE) { return }
Run docker push $imageTag
if ($LASTEXITCODE) {
if ($PSCmdlet.ParameterSetName -ne 'DoLogin') {
Write-Warning "If docker push is failing due to authentication issues, try calling this script with '-Login'"
}
return
}
}
}

Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..."
kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f -
if ($LASTEXITCODE) {exit $LASTEXITCODE}

Write-Host "Installing or upgrading stress test $($pkg.ReleaseName) from $($pkg.Directory)"
Run helm upgrade $pkg.ReleaseName $pkg.Directory `
-n $pkg.Namespace `
--install `
--set repository=$registryName.azurecr.io/$repository `
--set tag=$deployId `
--set stress-test-addons.env=$environment
if ($LASTEXITCODE) {
# Issues like 'UPGRADE FAILED: another operation (install/upgrade/rollback) is in progress'
# can be the result of cancelled `upgrade` operations (e.g. ctrl-c).
# See https://github.com/helm/helm/issues/4558
Write-Warning "The issue may be fixable by first running 'helm rollback -n $($pkg.Namespace) $($pkg.ReleaseName)'"
return
}

# Helm 3 stores release information in kubernetes secrets. The only way to add extra labels around
# specific releases (thereby enabling filtering on `helm list`) is to label the underlying secret resources.
# There is not currently support for setting these labels via the helm cli.
$helmReleaseConfig = kubectl get secrets `
-n $pkg.Namespace `
-l status=deployed,name=$($pkg.ReleaseName) `
-o jsonpath='{.items[0].metadata.name}'

Run kubectl label secret -n $pkg.Namespace --overwrite $helmReleaseConfig deployId=$deployId
}

function CheckDependencies()
{
$deps = @(
@{
Command = "docker";
Help = "Docker must be installed: https://docs.docker.com/get-docker/";
}
@{
Command = "kubectl";
Help = "kubectl must be installed: https://kubernetes.io/docs/tasks/tools/#kubectl";
},
@{
Command = "helm";
Help = "helm must be installed: https://helm.sh/docs/intro/install/";
},
@{
Command = "az";
Help = "Azure CLI must be installed: https://docs.microsoft.com/en-us/cli/azure/install-azure-cli";
}
)

Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module

$shouldError = $false
foreach ($dep in $deps) {
if (!(Get-Command $dep.Command -ErrorAction SilentlyContinue)) {
$shouldError = $true
Write-Error $dep.Help
}
}

if ($shouldError) {
exit 1
}

}

# Don't call functions when the script is being dot sourced
if ($MyInvocation.InvocationName -ne ".") {
CheckDependencies
DeployStressTests @PSBoundParameters
}
CheckDependencies
DeployStressTests @PSBoundParameters
Loading

0 comments on commit 6a42aff

Please sign in to comment.