Integration tests use DevOps MI for Cosmos (#4369)

* Base changes for Cosmos MI in Integration Tests * Get the ADServicePrincipal id * Pass through test-mi variables
microsoft · Sep 24, 2024 · b046235 · b046235
1 parent 5f6c02c
commit b046235
Show file tree

Hide file tree

Showing 12 changed files with 157 additions and 41 deletions.
diff --git a/build/jobs/add-resource-group-role-assignments.yml b/build/jobs/add-resource-group-role-assignments.yml
@@ -12,7 +12,24 @@ steps:
     azurePowerShellVersion: latestVersion
     ScriptType: inlineScript
     Inline: |
-      $clientId = (Get-AzContext).Account.Id
-      $servicePrincipalId = (Get-AzADServicePrincipal -Filter "appId eq '$clientId'").Id
+      $account = Get-AzContext
+      $principalId = (Get-AzADServicePrincipal -ApplicationId $account.Account.Id).Id
+      
       $resourceGroupResourceId = (Get-AzResourceGroup -Name ${{ parameters.resourceGroupName }}).ResourceId
-      New-AzRoleAssignment -ObjectId $servicePrincipalId -RoleDefinitionName "Storage Blob Data Contributor" -Scope $resourceGroupResourceId
+      
+      # Read and write PR storage accounts
+      New-AzRoleAssignment -ObjectId $principalId -RoleDefinitionName "Storage Blob Data Contributor" -Scope $resourceGroupResourceId
+
+      # Control Plane Actions on PR Cosmos accounts
+      $cosmosContributorRoleDefinitionId = "230815da-be43-4aae-9cb4-875f7bd000aa"
+      New-AzRoleAssignment `
+          -ObjectId $principalId `
+          -RoleDefinitionId $cosmosContributorRoleDefinitionId `
+          -Scope $resourceGroupResourceId
+
+      # Push on PR ACRs
+      $acrPushRoleDefinitionId = "8311e382-0749-4cb8-b61a-304f252e45ec"
+      New-AzRoleAssignment `
+          -ObjectId $principalId `
+          -RoleDefinitionId $acrPushRoleDefinitionId `
+          -Scope $resourceGroupResourceId
diff --git a/build/jobs/e2e-tests.yml b/build/jobs/e2e-tests.yml
@@ -53,7 +53,6 @@ steps:
         $exportStoreKey = Get-AzStorageAccountKey -ResourceGroupName $(ResourceGroupName) -Name "$exportStoreAccountName" | Where-Object {$_.KeyName -eq "key1"}
         
         Write-Host "##vso[task.setvariable variable=TestExportStoreUri]$($exportStoreUri)"
-        Write-Host "##vso[task.setvariable variable=TestExportStoreKey]$($exportStoreKey.Value)"
 
         $integrationStoreSettings = $appSettings | where {$_.Name -eq "FhirServer__Operations__IntegrationDataStore__StorageAccountUri"}
         $integrationStoreUri = $integrationStoreSettings[0].Value
@@ -100,9 +99,7 @@ steps:
       'Resource': $(Resource)
       'AllStorageAccounts': $(AllStorageAccounts)
       'TestExportStoreUri': $(TestExportStoreUri)
-      'TestExportStoreKey': $(TestExportStoreKey)
       'TestIntegrationStoreUri': $(TestIntegrationStoreUri)
-      'TestIntegrationStoreKey': $(TestIntegrationStoreKey)
       'tenant-admin-service-principal-name': $(tenant-admin-service-principal-name)
       'tenant-admin-service-principal-password': $(tenant-admin-service-principal-password)
       'tenant-admin-user-name': $(tenant-admin-user-name)

diff --git a/build/jobs/provision-deploy.yml b/build/jobs/provision-deploy.yml
@@ -128,6 +128,21 @@ jobs:
             }
           }
         }
+
+        if("${{ parameters.sql }}" -eq "false"){
+            Write-Host "Add DevOps MI permission to Cosmos database"
+
+            $account = Get-AzContext
+            $principalId = (Get-AzADServicePrincipal -ApplicationId $account.Account.Id).Id
+
+            New-AzCosmosDBSqlRoleAssignment `
+              -AccountName $webAppName `
+              -ResourceGroupName $resourceGroupName `
+              -Scope "/" `
+              -PrincipalId $principalId `
+              -RoleDefinitionId "00000000-0000-0000-0000-000000000002"
+        }
+
   - template: ./provision-healthcheck.yml
     parameters: 
       webAppName: ${{ parameters.webAppName }}
diff --git a/build/jobs/run-tests.yml b/build/jobs/run-tests.yml
@@ -20,7 +20,7 @@ jobs:
       downloadType: 'single'
       downloadPath: '$(System.ArtifactsDirectory)'
       artifactName: 'IntegrationTests'
-  
+
   - task: ExtractFiles@1
     displayName: 'Extract Integration Test Binaries'
     inputs:
@@ -37,6 +37,25 @@ jobs:
       azureSubscription: $(ConnectedServiceName)
       KeyVaultName: '${{ parameters.keyVaultName }}'
 
+  - task: AzurePowerShell@5
+    displayName: 'Set Workload Identity Variables'
+    inputs:
+      azureSubscription: $(ConnectedServiceName)
+      azurePowerShellVersion: latestVersion
+      ScriptType: inlineScript
+      Inline: |
+        Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_CLIENT_ID]$env:AZURESUBSCRIPTION_CLIENT_ID"
+        Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_TENANT_ID]$env:AZURESUBSCRIPTION_TENANT_ID"
+        Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_SERVICE_CONNECTION_ID]$env:AZURESUBSCRIPTION_SERVICE_CONNECTION_ID"
+
+        $appServiceName = '${{ parameters.appServiceName }}'
+        $appSettings = (Get-AzWebApp -ResourceGroupName $(ResourceGroupName) -Name $appServiceName).SiteConfig.AppSettings
+        $dataStoreResourceId = $appSettings | where {$_.Name -eq "FhirServer__ResourceManager__DataStoreResourceId"}
+        $dataStoreResourceId = $dataStoreResourceId[0].Value
+        Write-Host "$dataStoreResourceId"
+        Write-Host "##vso[task.setvariable variable=DataStoreResourceId]$($dataStoreResourceId)"
+
+
   - task: DotNetCoreCLI@2
     displayName: 'Run Cosmos Integration Tests'
     inputs:
@@ -45,8 +64,13 @@ jobs:
       workingDirectory: "$(System.ArtifactsDirectory)"
       testRunTitle: '${{ parameters.version }} Integration'
     env:
-      'CosmosDb:Host': $(CosmosDb--Host)
-      'CosmosDb:Key': $(CosmosDb--Key)
+      'CosmosDb__Host': $(CosmosDb--Host)
+      'FhirServer__ResourceManager__DataStoreResourceId': '$(DataStoreResourceId)'
+      'CosmosDb__UseManagedIdentity': true
+      'AZURESUBSCRIPTION_CLIENT_ID': '$(AZURESUBSCRIPTION_CLIENT_ID)'
+      'AZURESUBSCRIPTION_TENANT_ID': '$(AZURESUBSCRIPTION_TENANT_ID)'
+      'AZURESUBSCRIPTION_SERVICE_CONNECTION_ID': '$(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)'
+      'SYSTEM_ACCESSTOKEN': $(System.AccessToken)
 
 - job: "SqlIntegrationTests"
   pool:
@@ -59,7 +83,7 @@ jobs:
       downloadType: 'single'
       downloadPath: '$(System.ArtifactsDirectory)'
       artifactName: 'IntegrationTests'
-  
+
   - task: ExtractFiles@1
     displayName: 'Extract Integration Test Binaries'
     inputs:

diff --git a/samples/templates/default-azuredeploy-docker.json b/samples/templates/default-azuredeploy-docker.json
@@ -445,7 +445,7 @@
         },
         {
             "condition": "[equals(parameters('solutionType'),'FhirServerCosmosDB')]",
-            "apiVersion": "2019-12-12",
+            "apiVersion": "2021-06-15",
             "type": "Microsoft.DocumentDB/databaseAccounts",
             "tags": {
                 "FhirServerSolution": "[parameters('solutionType')]"
@@ -455,6 +455,7 @@
             "properties": {
                 "name": "[variables('serviceName')]",
                 "databaseAccountOfferType": "Standard",
+                "disableLocalAuth": true,
                 "consistencyPolicy": "[parameters('cosmosDbAccountConsistencyPolicy')]",
                 "keyVaultKeyUri": "[parameters('cosmosDbCmkUrl')]",
                 "locations": [
@@ -692,20 +693,6 @@
                 "[resourceId('Microsoft.DocumentDb/databaseAccounts', variables('serviceName'))]"
             ]
         },
-        {
-            "condition": "[equals(parameters('solutionType'),'FhirServerCosmosDB')]",
-            "type": "Microsoft.KeyVault/vaults/secrets",
-            "name": "[concat(variables('serviceName'), '/CosmosDb--Key')]",
-            "apiVersion": "2015-06-01",
-            "properties": {
-                "contentType": "text/plain",
-                "value": "[if(equals(parameters('solutionType'),'FhirServerCosmosDB'), listKeys(resourceId('Microsoft.DocumentDb/databaseAccounts', variables('serviceName')), '2015-04-08').primaryMasterKey, '')]"
-            },
-            "dependsOn": [
-                "[resourceId('Microsoft.KeyVault/vaults', variables('serviceName'))]",
-                "[resourceId('Microsoft.DocumentDb/databaseAccounts', variables('serviceName'))]"
-            ]
-        },
         {
             "condition": "[equals(parameters('solutionType'),'FhirServerSqlServer')]",
             "type": "Microsoft.KeyVault/vaults/secrets",

diff --git a/...ft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ResourceManagerCollectionSetup.cs b/...ft.Health.Fhir.CosmosDb.Initialization/Features/Storage/ResourceManagerCollectionSetup.cs
@@ -47,6 +47,7 @@ public ResourceManagerCollectionSetup(
         CosmosDataStoreConfiguration cosmosDataStoreConfiguration,
         IConfiguration genericConfiguration,
         IEnumerable<IStoredProcedureMetadata> storedProcedures,
+        TokenCredential tokenCredential,
         ILogger<ResourceManagerCollectionSetup> logger)
     {
         EnsureArg.IsNotNull(storedProcedures, nameof(storedProcedures));
@@ -55,13 +56,14 @@ public ResourceManagerCollectionSetup(
         _cosmosDataStoreConfiguration = EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration));
         _storeProceduresMetadata = EnsureArg.IsNotNull(storedProcedures, nameof(storedProcedures));
         _logger = EnsureArg.IsNotNull(logger, nameof(logger));
+        EnsureArg.IsNotNull(tokenCredential, nameof(tokenCredential));
 
         var dataStoreResourceId = EnsureArg.IsNotNullOrWhiteSpace(
                 genericConfiguration.GetValue(FhirServerResourceManagerDataStoreResourceId, string.Empty),
                 nameof(genericConfiguration),
                 fn => fn.WithMessage($"{FhirServerResourceManagerDataStoreResourceId} must be set."));
 
-        _armClient = new ArmClient(new DefaultAzureCredential());
+        _armClient = new ArmClient(tokenCredential);
         _resourceIdentifier = ResourceIdentifier.Parse(dataStoreResourceId);
         _account = _armClient.GetCosmosDBAccountResource(_resourceIdentifier);
     }

diff --git a/...osoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/FhirCosmosClientInitializerTests.cs b/...osoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/FhirCosmosClientInitializerTests.cs
@@ -5,6 +5,7 @@
 
 using System;
 using System.Collections.Generic;
+using Azure.Core;
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Health.Core.Features.Context;
 using Microsoft.Health.Fhir.Core.Features.Context;
@@ -39,6 +40,7 @@ public FhirCosmosClientInitializerTests()
                 clientTestProvider,
                 () => new[] { new TestRequestHandler() },
                 new RetryExceptionPolicyFactory(_cosmosDataStoreConfiguration, Substitute.For<RequestContextAccessor<IFhirRequestContext>>()),
+                new Lazy<TokenCredential>(() => Substitute.For<TokenCredential>()),
                 NullLogger<FhirCosmosClientInitializer>.Instance);
 
             _collectionInitializers = new List<ICollectionInitializer> { _collectionInitializer1, _collectionInitializer2 };

diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/FhirCosmosClientInitializer.cs
@@ -8,7 +8,7 @@
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
-using Azure.Identity;
+using Azure.Core;
 using EnsureThat;
 using Microsoft.Azure.Cosmos;
 using Microsoft.Azure.Cosmos.Fluent;
@@ -30,6 +30,7 @@ public class FhirCosmosClientInitializer : ICosmosClientInitializer
         private readonly ILogger<FhirCosmosClientInitializer> _logger;
         private readonly Func<IEnumerable<RequestHandler>> _requestHandlerFactory;
         private readonly RetryExceptionPolicyFactory _retryExceptionPolicyFactory;
+        private readonly Lazy<TokenCredential> _tokenCredential;
         private readonly object _lockObject;
 
         private CosmosClient _cosmosClient;
@@ -39,16 +40,19 @@ public FhirCosmosClientInitializer(
             ICosmosClientTestProvider testProvider,
             Func<IEnumerable<RequestHandler>> requestHandlerFactory,
             RetryExceptionPolicyFactory retryExceptionPolicyFactory,
+            Lazy<TokenCredential> tokenCredential,
             ILogger<FhirCosmosClientInitializer> logger)
         {
             EnsureArg.IsNotNull(testProvider, nameof(testProvider));
             EnsureArg.IsNotNull(requestHandlerFactory, nameof(requestHandlerFactory));
             EnsureArg.IsNotNull(retryExceptionPolicyFactory, nameof(retryExceptionPolicyFactory));
+            EnsureArg.IsNotNull(tokenCredential, nameof(tokenCredential));
             EnsureArg.IsNotNull(logger, nameof(logger));
 
             _testProvider = testProvider;
             _requestHandlerFactory = requestHandlerFactory;
             _retryExceptionPolicyFactory = retryExceptionPolicyFactory;
+            _tokenCredential = tokenCredential;
             _logger = logger;
             _lockObject = new object();
 
@@ -123,7 +127,7 @@ private CosmosClient CreateCosmosClientInternal(CosmosDataStoreConfiguration con
             CosmosClientBuilder builder;
 
             builder = configuration.UseManagedIdentity ?
-                new CosmosClientBuilder(host, new DefaultAzureCredential()) :
+                new CosmosClientBuilder(host, _tokenCredential.Value) :
                 new CosmosClientBuilder(host, key);
 
             builder

diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Microsoft.Health.Fhir.Shared.Api.projitems b/src/Microsoft.Health.Fhir.Shared.Api/Microsoft.Health.Fhir.Shared.Api.projitems
@@ -53,6 +53,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)Features\Routing\QueryStringParser.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Modules\FeatureFlags\Validate\ValidateFeatureModule.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Modules\FeatureFlags\Validate\ValidatePostConfigureOptions.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)Modules\IdentityModule.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Modules\MediationModule.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Modules\MvcModule.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Features\Exceptions\BaseExceptionMiddleware.cs" />

diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Modules/IdentityModule.cs b/src/Microsoft.Health.Fhir.Shared.Api/Modules/IdentityModule.cs
@@ -0,0 +1,21 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using Azure.Core;
+using Azure.Identity;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
+using Microsoft.Health.Extensions.DependencyInjection;
+
+namespace Microsoft.Health.Fhir.Api.Modules;
+
+public class IdentityModule : IStartupModule
+{
+    public void Load(IServiceCollection services)
+    {
+        // Provide support for Azure Managed Identity
+        services.TryAddSingleton<TokenCredential, DefaultAzureCredential>();
+    }
+}
diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Modules/SearchModule.cs b/src/Microsoft.Health.Fhir.Shared.Api/Modules/SearchModule.cs
@@ -11,7 +11,6 @@
 using Microsoft.Health.Extensions.DependencyInjection;
 using Microsoft.Health.Fhir.Api.Configs;
 using Microsoft.Health.Fhir.Api.Features.Filters;
-using Microsoft.Health.Fhir.Api.Features.Resources.Bundle;
 using Microsoft.Health.Fhir.Api.Features.Routing;
 using Microsoft.Health.Fhir.Core.Extensions;
 using Microsoft.Health.Fhir.Core.Features.Compartment;