[CI Environment] Token Mechanism Uplift (support tokens as GitHub Secret + Migrate Settings.Json to Settings YAML)

.github/actions/templates/validateModuleDeployment/action.yml

@@ -14,7 +14,7 @@
 ##   |============================================================================================================================================================================|
 ##   | Parameter                 | Required | Default | Description                                           | Example                                                           |
 ##   |---------------------------|----------|---------|-------------------------------------------------------|-------------------------------------------------------------------|
-##   | templateFilePath          | true     | ''      | The path to the template file to use for deployment              | 'arm/Microsoft.ApiManagement/service/deploy.bicep'                |
+##   | templateFilePath          | true     | ''      | The path to the template file to use for deployment   | 'arm/Microsoft.ApiManagement/service/deploy.bicep'                |
 ##   | parameterFilePath         | true     | ''      | The path to the parameter file to use for deployment  | 'arm/Microsoft.ApiManagement/service/.parameters/parameters.json' |
 ##   | location                  | true     | ''      | The location to use for deployment                    | 'WestEurope'                                                      |
 ##   | resourceGroupName         | false    | ''      | The resource group to deploy to                       | 'validation-rg'                                                   |
@@ -129,18 +129,15 @@ runs:
           # Load used functions
           . (Join-Path $env:GITHUB_WORKSPACE 'utilities' 'pipelines' 'tokensReplacement' 'Convert-TokensInFile.ps1')
 
-          # Load Settings File
-          $Settings = Get-Content -Path "settings.json" | ConvertFrom-Json -AsHashTable
-
           # Construct Token Function Input
           $ConvertTokensInputs = @{
             Tokens       = @{}
             FilePath     = '${{ inputs.parameterFilePath }}'
-            TokenPrefix  = $Settings.parameterFileTokens.tokenPrefix
-            TokenSuffix  = $Settings.parameterFileTokens.tokenSuffix
+            TokenPrefix  = '${{ env.tokenPrefix }}'
+            TokenSuffix  = '${{ env.tokenSuffix }}'
           }
 
-          # Local tokens
+          # Add enforced tokens
           $ConvertTokensInputs.Tokens += @{
             resourceGroupName = '${{ inputs.resourceGroupName }}'
             subscriptionId    = '${{ inputs.subscriptionId }}'
@@ -149,14 +146,18 @@ runs:
             deploymentSpId    = '${{ env.DEPLOYMENT_SP_ID }}'
           }
 
-          # Add local tokens
-          if ($Settings.parameterFileTokens.localTokens) {
-            $tokenMap = @{}
-            foreach ($token in $Settings.parameterFileTokens.localTokens) {
-              $tokenMap += @{ $token.name = $token.value }
-            }
-            Write-Verbose ('Using local tokens [{0}]' -f ($tokenMap.Keys -join ', ')) -Verbose
-            $ConvertTokensInputs.Tokens += $tokenMap
+          # Add local (source control) tokens
+          $tokenMap = @{}
+          foreach ($token in (Get-ChildItem env: | Where-Object -Property Name -Like "localToken_*")) {
+            $tokenMap += @{ $token.Name.Replace('localToken_','') = $token.value }
+          }
+          Write-Verbose ('Using local tokens [{0}]' -f ($tokenMap.Keys -join ', ')) -Verbose
+          $ConvertTokensInputs.Tokens += $tokenMap
+
+          # Swap 'namePrefix' token if provided as a GitHub secret
+          if($ConvertTokensInputs.Tokens.namePrefix -and '${{ env.TOKEN_NAMEPREFIX }}'){
+            Write-Verbose 'Using [namePrefix] token from GitHub' -Verbose
+            $ConvertTokensInputs.Tokens.namePrefix = '${{ env.TOKEN_NAMEPREFIX }}'
           }
 
           # Add custom tokens (passed in via the pipeline)
@@ -200,10 +201,9 @@ runs:
             $functionInput['parameterFilePath'] = Join-Path $env:GITHUB_WORKSPACE '${{ inputs.parameterFilePath }}'
           }
 
-          $projectSettings = Get-Content -Path 'settings.json' | ConvertFrom-Json
-          if (-not [String]::IsNullOrEmpty($projectSettings.enableDefaultTelemetry) -and (Get-Content -Path $functionInput.templateFilePath -Raw) -like '*param enableDefaultTelemetry*') {
+          if (-not [System.Convert]::ToBoolean('${{  env.enableDefaultTelemetry }}') -and (Get-Content -Path $functionInput.templateFilePath -Raw) -like '*param enableDefaultTelemetry*') {
               $functionInput['additionalParameters'] += @{
-                  enableDefaultTelemetry = $projectSettings.enableDefaultTelemetry
+                  enableDefaultTelemetry = [System.Convert]::ToBoolean('${{  env.enableDefaultTelemetry }}')
               }
           }
 
@@ -242,10 +242,9 @@ runs:
             $functionInput['parameterFilePath'] = Join-Path $env:GITHUB_WORKSPACE '${{ inputs.parameterFilePath }}'
           }
 
-          $projectSettings = Get-Content -Path 'settings.json' | ConvertFrom-Json
-          if (-not [String]::IsNullOrEmpty($projectSettings.enableDefaultTelemetry) -and (Get-Content -Path $functionInput.templateFilePath -Raw) -like '*param enableDefaultTelemetry*') {
+          if (-not [System.Convert]::ToBoolean('${{  env.enableDefaultTelemetry }}') -and (Get-Content -Path $functionInput.templateFilePath -Raw) -like '*param enableDefaultTelemetry*') {
               $functionInput['additionalParameters'] += @{
-                  enableDefaultTelemetry = $projectSettings.enableDefaultTelemetry
+                  enableDefaultTelemetry = [System.Convert]::ToBoolean('${{  env.enableDefaultTelemetry }}')
               }
           }
 

.github/actions/templates/validateModulePester/action.yml

@@ -45,6 +45,7 @@ runs:
             # This is the latest tested Pester version. Uncomment the next line in case of a future breaking change in the default version installed on the runner.
             # @{ Name = 'Pester'; Version = '5.3.1' }
             @{ Name = 'Az.Resources' }
+            @{ Name = 'powershell-yaml'; Version = '0.4.2'}
         )
 
         # Set agent up
@@ -76,18 +77,32 @@ runs:
           Write-Verbose "- [($moduleFolderPath]" -Verbose
         }
 
-        $enforcedTokenList = @{}
+        $GlobalVariablesObject = Get-Content -Path 'global.variables.yml' | ConvertFrom-Yaml | Select-Object -ExpandProperty variables
+
+        # Construct Token Configuration Input
+        $tokenConfiguration = @{
+            Tokens      = @{}
+            TokenPrefix = $GlobalVariablesObject | Select-Object -ExpandProperty tokenPrefix
+            TokenSuffix = $GlobalVariablesObject | Select-Object -ExpandProperty tokenSuffix
+        }
+
+        ## Enforced Tokens
         if (-not [String]::IsNullOrEmpty('${{ env.ARM_SUBSCRIPTION_ID }}')) {
-            $enforcedTokenList['subscriptionId'] = '${{ env.ARM_SUBSCRIPTION_ID }}'
+            $tokenConfiguration.Tokens['subscriptionId'] = '${{ env.ARM_SUBSCRIPTION_ID }}'
         }
         if (-not [String]::IsNullOrEmpty('${{ env.ARM_MGMTGROUP_ID }}')) {
-            $enforcedTokenList['managementGroupId'] = '${{ env.ARM_MGMTGROUP_ID }}'
+            $tokenConfiguration.Tokens['managementGroupId'] = '${{ env.ARM_MGMTGROUP_ID }}'
         }
         if (-not [String]::IsNullOrEmpty('${{ env.DEPLOYMENT_SP_ID }}')) {
-            $enforcedTokenList['deploymentSpId'] = '${{ env.DEPLOYMENT_SP_ID }}'
+            $tokenConfiguration.Tokens['deploymentSpId'] = '${{ env.DEPLOYMENT_SP_ID }}'
         }
         if (-not [String]::IsNullOrEmpty('${{ env.ARM_TENANT_ID }}')) {
-            $enforcedTokenList['tenantId'] = '${{ env.ARM_TENANT_ID }}'
+            $tokenConfiguration.Tokens['tenantId'] = '${{ env.ARM_TENANT_ID }}'
+        }
+
+        ## Local Tokens from global.variables.yml
+        foreach ($localToken in $GlobalVariablesObject.Keys | ForEach-Object { if ($PSItem.contains('localToken_')) { $PSItem } }) {
+            $tokenConfiguration.Tokens[$localToken.Replace('localToken_', '')] = $GlobalVariablesObject.$localToken
         }
 
         # --------------------- #
@@ -96,8 +111,8 @@ runs:
         Invoke-Pester -Configuration @{
           Run        = @{
             Container = New-PesterContainer -Path 'arm/.global/global.module.tests.ps1' -Data @{
-              moduleFolderPaths = $moduleFolderPaths
-              enforcedTokenList = $enforcedTokenList
+              moduleFolderPaths  = $moduleFolderPaths
+              tokenConfiguration = $tokenConfiguration
             }
           }
           TestResult = @{

.github/workflows/ms.network.applicationsecuritygroups.yml

@@ -34,6 +34,7 @@ env:
   ARM_MGMTGROUP_ID: '${{ secrets.ARM_MGMTGROUP_ID }}'
   ARM_TENANT_ID: '${{ secrets.ARM_TENANT_ID }}'
   DEPLOYMENT_SP_ID: '${{ secrets.DEPLOYMENT_SP_ID }}'
+  TOKEN_NAMEPREFIX: '${{ secrets.TOKEN_NAMEPREFIX }}'
 
 jobs:
   ###########################

arm/.global/global.module.tests.ps1

@@ -6,19 +6,17 @@ param (
         (Get-ChildItem $_ -File -Depth 0 -Include @('deploy.json', 'deploy.bicep') -Force).Count -gt 0
         }),
 
-    # Tokens to test for (i.e. their value should not be used in the parameter files, but their placeholder)
+    # Dedicated Tokens configuration hashtable containing the tokens and token prefix and suffix.
     [Parameter(Mandatory = $false)]
-    [hashtable] $enforcedTokenList = @{}
+    [hashtable] $tokenConfiguration = @{}
 )
 
 $script:RepoRoot = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent
-$script:Settings = Get-Content -Path (Join-Path $PSScriptRoot '..\..\settings.json') | ConvertFrom-Json -AsHashtable
 $script:RGdeployment = 'https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#'
 $script:Subscriptiondeployment = 'https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#'
 $script:MGdeployment = 'https://schema.management.azure.com/schemas/2019-08-01/managementGroupDeploymentTemplate.json#'
 $script:Tenantdeployment = 'https://schema.management.azure.com/schemas/2019-08-01/tenantDeploymentTemplate.json#'
 $script:moduleFolderPaths = $moduleFolderPaths
-$script:enforcedTokenList = $enforcedTokenList
 
 # For runtime purposes, we cache the compiled template in a hashtable that uses a formatted relative module path as a key
 $script:convertedTemplates = @{}
@@ -147,7 +145,6 @@ Describe 'File/folder tests' -Tag Modules {
         }
     }
 }
-
 Describe 'Readme tests' -Tag Readme {
 
     Context 'Readme content tests' {
@@ -515,7 +512,7 @@ Describe 'Deployment template tests' -Tag Template {
                         parameterFile_AllParameterNames      = $parameterFile_AllParameterNames
                         templateFile_AllParameterNames       = $TemplateFile_AllParameterNames
                         templateFile_RequiredParametersNames = $TemplateFile_RequiredParametersNames
-                        tokenSettings                        = $Settings.parameterFileTokens
+                        tokenConfiguration                   = $tokenConfiguration
                     }
                 }
             }
@@ -920,35 +917,39 @@ Describe 'Deployment template tests' -Tag Template {
             if (Test-Path (Join-Path $moduleFolderPath '.parameters')) {
                 $ParameterFilePaths = (Get-ChildItem (Join-Path -Path $moduleFolderPath -ChildPath '.parameters' -AdditionalChildPath '*parameters.json') -Recurse -Force).FullName
                 foreach ($ParameterFilePath in $ParameterFilePaths) {
-                    foreach ($token in $enforcedTokenList.Keys) {
+                    foreach ($token in $tokenConfiguration.Tokens.Keys) {
                         $parameterFileTokenTestCases += @{
                             parameterFilePath = $ParameterFilePath
                             parameterFileName = Split-Path $ParameterFilePath -Leaf
-                            tokenSettings     = $Settings.parameterFileTokens
+                            tokenPrefix       = $tokenConfiguration.TokenPrefix
+                            tokenSuffix       = $tokenConfiguration.TokenSuffix
                             tokenName         = $token
-                            tokenValue        = $enforcedTokenList[$token]
+                            tokenValue        = $tokenConfiguration.Tokens[$token]
                             moduleFolderName  = $moduleFolderPath.Replace('\', '/').Split('/arm/')[1]
                         }
                     }
                 }
             }
         }
 
-        It '[<moduleFolderName>] [Tokens] Parameter file [<parameterFileName>] should not contain the plain value for token [<tokenName>] guid' -TestCases $parameterFileTokenTestCases {
-            param (
-                [string] $parameterFilePath,
-                [string] $parameterFileName,
-                [hashtable] $tokenSettings,
-                [string] $tokenName,
-                [string] $tokenValue,
-                [string] $moduleFolderName
-            )
-            $ParameterFileTokenName = -join ($tokenSettings.tokenPrefix, $tokenName, $tokenSettings.tokenSuffix)
-            $ParameterFileContent = Get-Content -Path $parameterFilePath
+        foreach ($parameterFileTokenTestCase in $parameterFileTokenTestCases) {
+            It '[<moduleFolderName>] [Tokens] Parameter file [<parameterFileName>] should not contain the plain value for token [<tokenName>]' -TestCases $parameterFileTokenTestCase {
+                param (
+                    [string] $parameterFilePath,
+                    [string] $parameterFileName,
+                    [string] $tokenPrefix,
+                    [string] $tokenSuffix,
+                    [string] $tokenName,
+                    [string] $tokenValue,
+                    [string] $moduleFolderName
+                )
+                $ParameterFileTokenName = -join ($tokenPrefix, $tokenName, $tokenSuffix)
+                $ParameterFileContent = Get-Content -Path $parameterFilePath
 
-            $incorrectReferencesFound = $ParameterFileContent | Select-String -Pattern $tokenValue -AllMatches
-            if ($incorrectReferencesFound.Matches) {
-                $incorrectReferencesFound.Matches.Count | Should -Be 0 -Because ('Parameter file should not contain the [{0}] value, instead should reference the token value [{1}]. Please check the {2} lines: [{3}]' -f $tokenName, $ParameterFileTokenName, $incorrectReferencesFound.Matches.Count, ($incorrectReferencesFound.Line.Trim() -join ",`n"))
+                $incorrectReferencesFound = $ParameterFileContent | Select-String -Pattern $tokenValue -AllMatches
+                if ($incorrectReferencesFound.Matches) {
+                    $incorrectReferencesFound.Matches.Count | Should -Be 0 -Because ('Parameter file should not contain the [{0}] value, instead should reference the token value [{1}]. Please check the {2} lines: [{3}]' -f $tokenName, $ParameterFileTokenName, $incorrectReferencesFound.Matches.Count, ($incorrectReferencesFound.Line.Trim() -join ",`n"))
+                }
             }
         }
     }

global.variables.yml

@@ -3,6 +3,26 @@ variables:
   ################################################## Common Variables #######################################################
   ###########################################################################################################################
 
+  # Enable or disable CARMLs CI telemetry tracking. Note: This tracks the number of deployments only.
+  # See: https://github.com/Azure/ResourceModules/wiki/The%20library%20-%20Module%20design#telemetry
+  enableDefaultTelemetry: false
+
+  ######################################
+  # Local tokens settings
+  ######################################
+
+  # the 'localToken_' prefix will be removed from the key name when the pipelines run.
+  # e.g. if you have a token in your parameter file as <<customKey>>, then the token defined in this file looks like "localToken_customKey": 'value'
+  localToken_namePrefix: 'zu3bx' # A 3-5 character length string, included in the resources names, is overridden if provided as GitHub Secret / ADO Variable
+
+  ######################################
+  # global tokens settings
+  ######################################
+
+  # this determines the starting prefix and ending suffix of the token in your file.
+  tokenPrefix: '<<'
+  tokenSuffix: '>>'
+
   ######################################
   # Validation deployment settings
   ######################################
@@ -24,7 +44,7 @@ variables:
   ######################################
 
   bicepRegistryDoPublish: true # Set to true, if you would like to publish module templates to a bicep registry
-  bicepRegistryName: adpsxxazacrx001 # The name of the bicep registry (ACR) to publish to. If it does not exist, it will be created.
+  bicepRegistryName: adpzu3bxazacrx001 # The name of the bicep registry (ACR) to publish to. If it does not exist, it will be created.
   bicepRegistryRGName: 'artifacts-rg' # The resource group that hosts the private bicep registry (ACR)
   bicepRegistryRgLocation: 'West Europe' # The location of the resource group to publish to
 
@@ -38,7 +58,7 @@ variables:
 
   vmImage: 'ubuntu-latest' # Use this for Microsoft-hosted agents
   poolName: '' # Use this for self-hosted agents
-  serviceConnection: 'CARML-CSU-Tenant-Connection'
+  serviceConnection: 'carml'
 
   ######################################
   # Source

utilities/pipelines/resourceRemoval/helper/Get-DependencyResourceNameList.ps1

@@ -33,24 +33,20 @@ function Get-DependencyResourceNameList {
     }
 
     # Replace tokens in dependency parameter files
-    $Settings = Get-Content -Path (Join-Path $repoRootPath 'settings.json') | ConvertFrom-Json -AsHashtable
+    $GlobalVariablesObject = Get-Content -Path (Join-Path $repoRootPath 'global.variables.yml') | ConvertFrom-Yaml -ErrorAction Stop | Select-Object -ExpandProperty variables
 
-    # Add local tokens
-    if ($Settings.parameterFileTokens.localTokens) {
-        $tokenMap = @{}
-        foreach ($token in $Settings.parameterFileTokens.localTokens) {
-            $tokenMap += @{ $token.name = $token.value }
-        }
-        Write-Verbose ('Using local tokens [{0}]' -f ($tokenMap.Keys -join ', '))
+    # Construct Token Configuration Input
+    $tokenConfiguration = @{
+        Tokens      = @{}
+        TokenPrefix = $GlobalVariablesObject | Select-Object -ExpandProperty tokenPrefix
+        TokenSuffix = $GlobalVariablesObject | Select-Object -ExpandProperty tokenSuffix
+    }
 
-        $ConvertTokensInputs = @{
-            Tokens      = $tokenMap
-            TokenPrefix = $Settings.parameterFileTokens.tokenPrefix
-            TokenSuffix = $Settings.parameterFileTokens.tokenSuffix
-            Verbose     = $false
-        }
-        $parameterFilePaths | ForEach-Object { $null = Convert-TokensInFile @ConvertTokensInputs -FilePath $_ }
+    ## Local Tokens from global.variables.yml
+    foreach ($localToken in $GlobalVariablesObject.Keys | ForEach-Object { if ($PSItem.contains('localToken_')) { $PSItem } }) {
+        $tokenConfiguration.Tokens[$localToken.Replace('localToken_', '')] = $GlobalVariablesObject.$localToken
     }
+    $parameterFilePaths | ForEach-Object { $null = Convert-TokensInFile @tokenConfiguration -FilePath $_ }
 
     $dependencyResourceNames = [System.Collections.ArrayList]@()
     foreach ($parameterFilePath in $parameterFilePaths) {
@@ -60,10 +56,8 @@ function Get-DependencyResourceNameList {
         }
     }
 
-    if ($Settings.parameterFileTokens.localTokens) {
-        Write-Verbose 'Restoring Tokens'
-        $parameterFilePaths | ForEach-Object { $null = Convert-TokensInFile @ConvertTokensInputs -FilePath $_ -SwapValueWithName $true }
-    }
+    Write-Verbose 'Restoring Tokens'
+    $parameterFilePaths | ForEach-Object { $null = Convert-TokensInFile @tokenConfiguration -FilePath $_ -SwapValueWithName $true }
 
     return $dependencyResourceNames
 }