Merge pull request #82 from ndlib/MEL-462

Mel 462
ndlib · Jul 29, 2019 · 7e8adae · 7e8adae
2 parents ceea137 + 5c693ab
commit 7e8adae
Show file tree

Hide file tree

Showing 2 changed files with 234 additions and 2 deletions.
diff --git a/deploy/cloudformation/manifest-pipeline-pipeline.yml b/deploy/cloudformation/manifest-pipeline-pipeline.yml
@@ -109,6 +109,9 @@ Outputs:
       Name: !Join [ ':', [ !Ref 'AWS::StackName', 'ApprovalTopic']]
 
 Resources:
+  ImageProcessorContainerRepository:
+    Type: AWS::ECR::Repository
+
   CodeBuildTrustRole:
     Type: 'AWS::IAM::Role'
     Properties:
@@ -139,6 +142,34 @@ Resources:
             Action:
               - 's3:GetObject'
               - 's3:PutObject'
+          # The codebuild pipeline is going to create ECR records. Need to allow certain permissions for the authorization
+          - Effect: Allow
+            Action:
+              - 'ecr:InitiateLayerUpload'
+              - 'ecr:UploadLayerPart'
+              - 'ecr:BatchCheckLayerAvailability'
+              - 'ecr:GetDownloadUrlForLayer'
+              - 'ecr:GetRepositoryPolicy'
+              - 'ecr:DescribeRepositories'
+              - 'ecr:ListImages'
+              - 'ecr:DescribeImages'
+              - 'ecr:BatchGetImage'
+              - 'ecr:CompleteLayerUpload'
+              - 'ecr:PutImage'
+            Resource: 
+              - !Sub 'arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${ImageProcessorContainerRepository}'
+          - Effect: Allow
+            Action:
+              - 'ecr:GetAuthorizationToken'
+            Resource: '*'
+          - Effect: Allow
+            Action:
+              - ssm:GetParameters
+            Resource:
+              - !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/esu/github/ndlib-git/oauth"
+              - !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/all/stacks/${ImageServiceProdStackName}/hostname"
+              - !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/all/stacks/${ImageServiceTestStackName}/hostname"
+
       Roles:
         - !Ref CodeBuildTrustRole
   CloudFormationTrustRole:
@@ -294,6 +325,18 @@ Resources:
               - 'route53:GetChange'
             Resource: '*'
             Effect: Allow
+          # The manifest pipeline is going to need to make changes to the TaskDefinition for ImageTaskDefinition
+          - Action:
+              - 'ecs:RegisterTaskDefinition'
+              - 'ecs:DeregisterTaskDefinition'
+            Resource: '*'
+            Effect: Allow
+          # The manifest pipeline is going to need PassRole for ImageTaskDefinition
+          - Action:
+              - 'iam:PassRole'
+            Resource:
+              - !Sub 'arn:aws:iam::${AWS::AccountId}:role/${InfrastructureStackName}-*'
+            Effect: Allow
       Roles:
         - !Ref CloudFormationTrustRole
   CodePipelineTrustRole:
@@ -436,6 +479,151 @@ Resources:
               - test-stack-configuration.json
               - prod-stack-configuration.json
 
+  DockerImageProcessorBuilder:
+    Type: 'AWS::CodeBuild::Project'
+    Properties:
+      Name: !Sub '${AWS::StackName}-docker-image-processor'
+      Description: 'Build Docker Image from GitHub for Image Processor'
+      ServiceRole: !GetAtt
+        - CodeBuildTrustRole
+        - Arn
+      TimeoutInMinutes: 10
+      Source:
+        Type: CODEPIPELINE
+        GitCloneDepth: 1
+        BuildSpec: |
+          version: 0.2
+          phases:
+            pre_build:
+              commands:
+                - echo Logging in to Amazon ECR...
+                - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION)
+                - REPOSITORY_URI=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME
+                - COMMIT_HASH=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7)
+                - echo $COMMIT_HASH
+                - IMAGE_TAG=${COMMIT_HASH:=latest}
+            build:
+              commands:
+                - echo Build started on `date`
+                - echo Building the Docker image...
+                - docker build -t $REPOSITORY_URI:latest -t $REPOSITORY_URI:$COMMIT_HASH ./pyramid-generator
+                - docker tag $REPOSITORY_URI:latest $REPOSITORY_URI:$COMMIT_HASH
+                - docker tag $REPOSITORY_URI:$COMMIT_HASH $REPOSITORY_URI:$COMMIT_HASH
+            post_build:
+              commands:
+                - echo Build completed on `date`
+                - echo Pushing the Docker images...
+                - docker push $REPOSITORY_URI:latest
+                - docker push $REPOSITORY_URI:$COMMIT_HASH
+                - printf '[{"name":"%s","imageUri":"%s"}]' $TEST_CONTAINER_NAME $REPOSITORY_URI:$IMAGE_TAG > imagedefinitionstest.json
+                - printf '[{"name":"%s","imageUri":"%s"}]' $PROD_CONTAINER_NAME $REPOSITORY_URI:$IMAGE_TAG > imagedefinitionsprod.json
+                - printf $CODEBUILD_RESOLVED_SOURCE_VERSION > sha.txt
+                - printf $REPOSITORY_URI:$IMAGE_TAG > deployed_tag.txt
+          artifacts:
+            files:
+              - imagedefinitionstest.json
+              - imagedefinitionsprod.json
+              - sha.txt
+              - deployed_tag.txt
+      Artifacts:
+        Type: CODEPIPELINE
+      Environment:
+        Type: LINUX_CONTAINER
+        ComputeType: BUILD_GENERAL1_SMALL
+        Image: aws/codebuild/docker:17.09.0
+        EnvironmentVariables:
+          - Name: AWS_DEFAULT_REGION
+            Value: !Ref AWS::Region
+          - Name: AWS_ACCOUNT_ID
+            Value: !Ref AWS::AccountId
+          - Name: IMAGE_REPO_NAME
+            Value: !Ref ImageProcessorContainerRepository
+
+  PostDeployDocker:
+    Type: AWS::CodeBuild::Project
+    Properties:
+      Name: !Sub '${AWS::StackName}-docker-add-tag'
+      Description: 'CodeBuild to update the current_release tag in ECR'
+      ServiceRole: !GetAtt
+        - CodeBuildTrustRole
+        - Arn
+      TimeoutInMinutes: 10
+      Source:
+        Type: CODEPIPELINE
+        GitCloneDepth: 1
+        BuildSpec: |
+          version: 0.2
+          phases:
+            pre_build:
+              commands:
+                - echo Pre-build started on `date`
+                - echo Logging in to Amazon ECR...
+                - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION)
+                - REPOSITORY_URI=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME
+            build:
+              commands:
+                - DEPLOYED_VERSION=$(cat deployed_tag.txt)
+                - docker pull $DEPLOYED_VERSION
+                - docker tag $DEPLOYED_VERSION $REPOSITORY_URI:current_release
+            post_build:
+              commands:
+                - echo Beginning post build on `date`
+                - docker push $REPOSITORY_URI:current_release
+      Artifacts:
+        Type: CODEPIPELINE
+      Environment:
+        Type: LINUX_CONTAINER
+        ComputeType: BUILD_GENERAL1_SMALL
+        Image: aws/codebuild/docker:17.09.0
+        EnvironmentVariables:
+          - Name: IMAGE_REPO_NAME
+            Value: !Ref ImageProcessorContainerRepository
+          - Name: AWS_DEFAULT_REGION
+            Value: !Ref AWS::Region
+          - Name: AWS_ACCOUNT_ID
+            Value: !Ref AWS::AccountId
+
+  PostDeployGitHub:
+    Type: AWS::CodeBuild::Project
+    Properties:
+      Name: !Sub '${AWS::StackName}-add-github-status'
+      Description: 'CodeBuild to tag the latest commit in repo post-deploy'
+      ServiceRole: !GetAtt
+        - CodeBuildTrustRole
+        - Arn
+      TimeoutInMinutes: 10
+      Source:
+        Type: CODEPIPELINE
+        GitCloneDepth: 1
+        BuildSpec: |
+          version: 0.2
+          phases:
+            pre_build:
+              commands:
+                - echo Pre-build started on `date`
+            build:
+              commands:
+                - SOURCE_SHA=$(cat sha.txt)
+                - DATE=$(date)
+            post_build:
+              commands:
+                - curl -i -X POST -H "Authorization:token ${OAUTH_TOKEN}" "${API_URL}/${SOURCE_SHA}" -d "{\"state\":\"success\",\"description\":\"Deployed to production on ${DATE}\",\"context\":\"${CODEBUILD_INITIATOR}\",\"target_url\":\"https://${TARGET_HOST}\"}"
+      Artifacts:
+        Type: CODEPIPELINE
+      Environment:
+        Type: LINUX_CONTAINER
+        ComputeType: BUILD_GENERAL1_SMALL
+        Image: aws/codebuild/ubuntu-base:14.04
+        EnvironmentVariables:
+          - Name: OAUTH_TOKEN
+            Type: PARAMETER_STORE
+            Value: /esu/github/ndlib-git/oauth
+          - Name: API_URL
+            Value: !Sub "https://api.github.com/repos/${GitHubUser}/${ManifestPipelineRepoName}/statuses"
+          - Name: TARGET_HOST
+            Type: PARAMETER_STORE
+            Value: !Sub "/all/stacks/${ImageServiceProdStackName}/hostname"
+
   S3Bucket:
     Type: AWS::S3::Bucket
     DeletionPolicy: Retain
@@ -516,6 +704,20 @@ Resources:
             ProjectName: !Ref CodeBuildProject
             PrimarySource: AppCode
           RunOrder: 1
+        - Name: BuildDockerImage
+          InputArtifacts:
+          - Name: AppCode
+          ActionTypeId:
+            Category: Build
+            Owner: AWS
+            Version: "1"
+            Provider: CodeBuild
+          OutputArtifacts:
+          - Name: BuiltCodeDockerImage
+          Configuration:
+            ProjectName: !Ref DockerImageProcessorBuilder
+            PrimarySource: AppCode
+          RunOrder: 2
 
       - Name: Test
         Actions:
@@ -535,6 +737,7 @@ Resources:
               ChangeSetName: TestChangeSetName
               TemplateConfiguration: BuiltCode::test-stack-configuration.json
               TemplatePath: BuiltCode::output.yml
+              ParameterOverrides: !Sub '{"ContainerImageUrl": "${ImageProcessorContainerRepository}:latest"}'
             RunOrder: 1
           - Name: Deploy
             ActionTypeId:
@@ -578,6 +781,7 @@ Resources:
               ChangeSetName: ProdChangeSetName
               TemplateConfiguration: BuiltCode::prod-stack-configuration.json
               TemplatePath: BuiltCode::output.yml
+              ParameterOverrides: !Sub '{"ContainerImageUrl": "${ImageProcessorContainerRepository}:latest"}'
             RunOrder: 1
           - Name: Deploy
             ActionTypeId:
@@ -592,6 +796,32 @@ Resources:
               StackName: !Ref ProdStackName
             RunOrder: 2
 
+      - Name: PostProduction
+        Actions:
+          - Name: "AddDockerTag"
+            InputArtifacts:
+              - Name: BuiltCodeDockerImage
+            ActionTypeId:
+              Owner: AWS
+              Category: Build
+              Provider: CodeBuild
+              Version: "1"
+            Configuration:
+              ProjectName: !Ref PostDeployDocker
+            RunOrder: 1
+
+          - Name: "AddGitHubStatus"
+            InputArtifacts:
+              - Name: BuiltCodeDockerImage
+            ActionTypeId:
+              Owner: AWS
+              Category: Build
+              Provider: CodeBuild
+              Version: "1"
+            Configuration:
+              ProjectName: !Ref PostDeployGitHub
+            RunOrder: 1
+
       ArtifactStore:
         Type: S3
         Location: !Ref S3Bucket

diff --git a/deploy/cloudformation/manifest-pipeline.yml b/deploy/cloudformation/manifest-pipeline.yml
@@ -65,7 +65,9 @@ Parameters:
     Type: Number
     Default: 2048
     Description: How much memory in megabytes to give the container
-
+  ContainerImageUrl:
+    Type: String
+    Description: The url of a docker image that will handle image processing
 Conditions:
 
   CreateDNS: !Equals [ !Ref CreateDNSRecord, 'True' ]
@@ -526,7 +528,7 @@ Resources:
           Cpu: !Ref 'ContainerCpu'
           Memory: !Ref 'ContainerMemory'
           EntryPoint: ['pyramid.py']
-          Image: hesburgh/marble-image-processing
+          Image: !Ref ContainerImageUrl
           LogConfiguration:
             LogDriver: awslogs
             Options: