Merge pull request #98 from ndlib/test-cf-stepfunctions

Add Mets Step functions and schema processing step functions

deploy/cloudformation/manifest-pipeline.yml

@@ -33,6 +33,10 @@ Parameters:
   AppConfigPath:
     Type: String
     Description: The path to look for the application config from
+  MarbleProcessingKeyPath:
+    Type: String
+    Description: The ssm path to look for the marble data processing config from
+    Default: '/all/marble-data-processing/prod'
   NoReplyEmailAddr:
     Type: String
     Description: Email address notification emails are sent from
@@ -73,8 +77,11 @@ Conditions:
   CreateDNS: !Equals [ !Ref CreateDNSRecord, 'True' ]
 
 Outputs:
-  StateMachine:
-    Value: !Ref StateMachine
+  SchemaStateMachine:
+    Value: !Ref SchemaStateMachine
+
+  StateMachineMets:
+    Value: !Ref StateMachineMets
 
   ProcessBucket:
     Value: !Ref ProcessBucket
@@ -266,7 +273,7 @@ Resources:
           LambdaFunctionAssociations:
             -
               EventType: origin-request
-              LambdaFunctionARN: !Sub ${SPARedirectionLambda.Arn}:${SPARedirectionLambdaV1.Version}
+              LambdaFunctionARN: !Sub ${SPARedirectionLambda.Arn}:${SPARedirectionLambdaV2.Version}
         Origins:
           - Id: Bucket
             DomainName: !GetAtt ManifestBucket.DomainName
@@ -322,7 +329,7 @@ Resources:
           exports.handler = (event, context, callback) => {
               var request = event.Records[0].cf.request;
 
-              if (!request.uri.endsWith('/index.json')) {
+              if (!request.uri.endsWith('/index.json') && !request.uri.endsWith('.xml')) {
                 if (request.uri.endsWith('/')) {
                   request.uri = request.uri + 'index.json'
                 } else {
@@ -336,7 +343,7 @@ Resources:
       Role: !GetAtt LambdaEdgeBasicExecutionRole.Arn
       Runtime: nodejs8.10
 
-  SPARedirectionLambdaV1:
+  SPARedirectionLambdaV2:
     Type: AWS::Lambda::Version
     Properties:
       FunctionName: !Ref SPARedirectionLambda
@@ -353,6 +360,16 @@ Resources:
               Service: lambda.amazonaws.com
             Action: "sts:AssumeRole"
       Policies:
+        - PolicyName: "StartStateMachine"
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              -
+                Effect: Allow
+                Action:
+                  - 'states:ListStateMachines'
+                  - 'states:StartExecution'
+                Resource: !Sub "arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${AWS::StackName}-SchemaStateMachine"
         - PolicyName: ManifestPipelineSsmPermissions
           PolicyDocument:
             Version: 2012-10-17
@@ -361,7 +378,9 @@ Resources:
                 Action:
                   - "ssm:GetParametersByPath"
                 Effect: "Allow"
-                Resource: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${AppConfigPath}/*"
+                Resource:
+                  - !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${AppConfigPath}/*"
+                  - !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${MarbleProcessingKeyPath}/*"
         - PolicyName: ImageDerivativeCacheBucketPermissions
           PolicyDocument:
             Version: 2012-10-17
@@ -427,14 +446,6 @@ Resources:
       CodeUri: autorun/
       Runtime: python3.7
       Timeout: 300
-      Policies:
-        - Version: '2012-10-17'
-          Statement:
-            - Effect: Allow
-              Action:
-                - 'states:ListStateMachines'
-                - 'states:StartExecution'
-              Resource: !Sub "arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:*"
       Events:
         S3FileListener:
           Type: S3
@@ -448,7 +459,7 @@ Resources:
                     Value: 'main.csv'
       Environment:
         Variables:
-          STATE_MACHINE: !Sub "arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${AWS::StackName}-StateMachine"
+          STATE_MACHINE: !Sub "arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${AWS::StackName}-SchemaStateMachine"
 
   InitManifestLambdaFunction:
     Type: AWS::Serverless::Function
@@ -491,15 +502,6 @@ Resources:
       Timeout: 90
       CodeUri: create_manifest/
 
-  SchemaLambdaFunction:
-    Type: AWS::Serverless::Function
-    Properties:
-      Handler: handler.run
-      Role: !GetAtt [ LambdaExecutionRole, Arn ]
-      Runtime: python3.7
-      Timeout: 90
-      CodeUri: manifestpy/
-
   FinalizeLambdaFunction:
     Type: AWS::Serverless::Function
     Properties:
@@ -628,10 +630,10 @@ Resources:
                 Resource:
                   - !Sub "arn:${AWS::Partition}:events:${AWS::Region}:${AWS::AccountId}:rule/StepFunctionsGetEventsForECSTaskRule"
 
-  StateMachine:
+  SchemaStateMachine:
     Type: "AWS::StepFunctions::StateMachine"
     Properties:
-      StateMachineName: !Sub "${AWS::StackName}-StateMachine"
+      StateMachineName: !Sub "${AWS::StackName}-SchemaStateMachine"
       DefinitionString:
         !Sub
           - |-
@@ -711,7 +713,7 @@ Resources:
                       "Next": "Finalize"
                     }
                   ],
-                  "Next": "ProcessSchema"
+                  "Next": "ProcessManifest"
                 },
                 "ProcessMetsInput": {
                   "Type": "Task",
@@ -723,18 +725,6 @@ Resources:
                       "Next": "Finalize"
                     }
                   ],
-                  "Next": "ProcessSchema"
-                },
-                "ProcessSchema": {
-                  "Type": "Task",
-                  "Resource": "${schemaLambdaArn}",
-                  "Catch": [
-                    {
-                      "ErrorEquals": [ "States.ALL" ],
-                      "ResultPath": "$.unexpected",
-                      "Next": "Finalize"
-                    }
-                  ],
                   "Next": "ProcessManifest"
                 },
                 "ProcessManifest": {
@@ -764,12 +754,116 @@ Resources:
               initManifestLambdaArn: !GetAtt [InitManifestLambdaFunction, Arn],
               processCsvInputLambdaArn: !GetAtt [ ProcessCsvInputLambdaFunction, Arn ],
               createManifestLambdaArn: !GetAtt [ CreateManifestLambdaFunction, Arn ],
-              schemaLambdaArn: !GetAtt [ SchemaLambdaFunction, Arn ],
               processMetsInputLambdaArn: !GetAtt [ProcessMetsInputLambdaFunction, Arn],
               finalizeLambdaArn: !GetAtt [ FinalizeLambdaFunction, Arn ]
             }
       RoleArn: !GetAtt [ StatesExecutionRole, Arn ]
 
+  FindObjectsLambdaFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      Handler: handler.run
+      Role: !GetAtt [ LambdaExecutionRole, Arn ]
+      Runtime: python3.7
+      Timeout: 90
+      CodeUri: find_objects/
+      Environment:
+        Variables:
+          SSM_KEY_BASE: !Ref AppConfigPath
+          SSM_MARBLE_DATA_PROCESSING_KEY_BASE: !Ref MarbleProcessingKeyPath
+
+  FindImagesLambdaFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      Handler: handler.run
+      Role: !GetAtt [ LambdaExecutionRole, Arn ]
+      Runtime: python3.7
+      Timeout: 90
+      CodeUri: find_images_for_objects/
+      Environment:
+        Variables:
+          SSM_KEY_BASE: !Ref AppConfigPath
+          SSM_MARBLE_DATA_PROCESSING_KEY_BASE: !Ref MarbleProcessingKeyPath
+
+  PopulatePipelineLambdaFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      Handler: handler.run
+      Role: !GetAtt [ LambdaExecutionRole, Arn ]
+      Runtime: python3.7
+      Timeout: 900
+      CodeUri: send_objects_to_pipeline/
+      Policies:
+        - Version: '2012-10-17'
+          Statement:
+            - Effect: Allow
+              Action:
+                - 'states:ListStateMachines'
+                - 'states:StartExecution'
+              Resource: !Ref StateMachineMets
+      Environment:
+        Variables:
+          SSM_KEY_BASE: !Ref AppConfigPath
+          SSM_MARBLE_DATA_PROCESSING_KEY_BASE: !Ref MarbleProcessingKeyPath
+          STATE_MACHINE: !Sub "arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${AWS::StackName}-SchemaStateMachine"
+
+
+  StateMachineMets:
+    Type: "AWS::StepFunctions::StateMachine"
+    Properties:
+      StateMachineName: !Sub "${AWS::StackName}-MetsStateMachine"
+      DefinitionString:
+        !Sub
+          - |-
+            {
+              "Comment": "Populate Manifest Pipeline with Changed metadata and images",
+              "StartAt": "FindObjectsTask",
+              "States": {
+                "FindObjectsTask": {
+                  "Type": "Task",
+                  "Resource": "${findObjectsLambdaArn}",
+                  "Next": "FindImagesTask"
+                },
+                "FindImagesTask": {
+                  "Type": "Task",
+                  "Resource": "${findImagesLambdaArn}",
+                  "Next": "PopulatePipelineTask"
+                },
+                "PopulatePipelineTask": {
+                  "Type": "Task",
+                  "Resource": "${populatePipelineLambdaArn}",
+                  "Next": "ChoiceSourceType"
+                },
+                "ChoiceSourceType": {
+                  "Type": "Choice",
+                  "Choices": [
+                    {
+                      "Variable": "$.populatePipelineCompleted",
+                      "BooleanEquals": false,
+                      "Next": "PopulatePipelineTask"
+                    },
+                    {
+                      "Variable": "$.populatePipelineCompleted",
+                      "BooleanEquals": true,
+                      "Next": "SuccessState"
+                    }
+                  ]
+                },
+                "SuccessState": {
+                  "Type": "Succeed"
+                }
+              }
+            }
+          -
+            {
+              findObjectsLambdaArn: !GetAtt [FindObjectsLambdaFunction, Arn],
+              findImagesLambdaArn: !GetAtt [ FindImagesLambdaFunction, Arn ],
+              populatePipelineLambdaArn: !GetAtt [ PopulatePipelineLambdaFunction, Arn ],
+            }
+      RoleArn: !GetAtt [ StatesExecutionRole, Arn ]
+
+
+
   DebugPermissions:
     Type: AWS::IAM::ManagedPolicy
     Properties:
@@ -799,8 +893,10 @@ Resources:
               - 'states:GetExecutionHistory'
               - 'states:DescribeStateMachineForExecution'
             Resource:
-              - !Sub 'arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${StateMachine.Name}'
-              - !Sub 'arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${StateMachine.Name}:*'
+              - !Sub 'arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${SchemaStateMachine.Name}'
+              - !Sub 'arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${SchemaStateMachine.Name}:*'
+              - !Sub 'arn:aws:states:${AWS::Region}:${AWS::AccountId}:stateMachine:${StateMachineMets.Name}'
+              - !Sub 'arn:aws:states:${AWS::Region}:${AWS::AccountId}:execution:${StateMachineMets.Name}:*'
 
   DataAdminPermissions:
     Type: AWS::IAM::ManagedPolicy