Skip to content

Commit

Permalink
chore(stepfunctions-tasks): add a doc and an integ test for spot inst…
Browse files Browse the repository at this point in the history
…ance fleets in EMR (#27809)

This PR adds the following points for spot instance fleet in EMR.

- a doc that blockDurationMinutes is no longer available to new customers from July 1, 2021.
  - https://docs.aws.amazon.com/emr/latest/APIReference/API_SpotProvisioningSpecification.html
- an integ test for spot instance fleet in EMR.
  - There was no integ test, so I added.

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
  • Loading branch information
go-to-k authored Nov 6, 2023
1 parent 7385ffb commit 5d7c1aa
Show file tree
Hide file tree
Showing 10 changed files with 972 additions and 0 deletions.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -0,0 +1,253 @@
{
"Resources": {
"EmrCreateClusterServiceRole5251910D": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": "sts:AssumeRole",
"Condition": {
"StringEquals": {
"aws:RequestTag/for-use-with-amazon-emr-managed-policies": "true"
}
},
"Effect": "Allow",
"Principal": {
"Service": "elasticmapreduce.amazonaws.com"
}
}
],
"Version": "2012-10-17"
},
"ManagedPolicyArns": [
{
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":iam::aws:policy/service-role/AmazonEMRServicePolicy_v2"
]
]
}
]
}
},
"EmrCreateClusterInstanceRoleC80466F5": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "ec2.amazonaws.com"
}
}
],
"Version": "2012-10-17"
}
}
},
"EmrCreateClusterInstanceProfileC1729180": {
"Type": "AWS::IAM::InstanceProfile",
"Properties": {
"InstanceProfileName": {
"Ref": "EmrCreateClusterInstanceRoleC80466F5"
},
"Roles": [
{
"Ref": "EmrCreateClusterInstanceRoleC80466F5"
}
]
}
},
"SMRole49C19C48": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "states.amazonaws.com"
}
}
],
"Version": "2012-10-17"
}
}
},
"SMRoleDefaultPolicy34CA15C7": {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyDocument": {
"Statement": [
{
"Action": [
"elasticmapreduce:DescribeCluster",
"elasticmapreduce:RunJobFlow",
"elasticmapreduce:TerminateJobFlows"
],
"Effect": "Allow",
"Resource": "*"
},
{
"Action": "iam:PassRole",
"Effect": "Allow",
"Resource": [
{
"Fn::GetAtt": [
"EmrCreateClusterInstanceRoleC80466F5",
"Arn"
]
},
{
"Fn::GetAtt": [
"EmrCreateClusterServiceRole5251910D",
"Arn"
]
}
]
},
{
"Action": "elasticmapreduce:AddTags",
"Effect": "Allow",
"Resource": {
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":elasticmapreduce:",
{
"Ref": "AWS::Region"
},
":",
{
"Ref": "AWS::AccountId"
},
":cluster/*"
]
]
}
},
{
"Action": [
"events:DescribeRule",
"events:PutRule",
"events:PutTargets"
],
"Effect": "Allow",
"Resource": {
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":events:",
{
"Ref": "AWS::Region"
},
":",
{
"Ref": "AWS::AccountId"
},
":rule/StepFunctionsGetEventForEMRRunJobFlowRule"
]
]
}
}
],
"Version": "2012-10-17"
},
"PolicyName": "SMRoleDefaultPolicy34CA15C7",
"Roles": [
{
"Ref": "SMRole49C19C48"
}
]
}
},
"SM934E715A": {
"Type": "AWS::StepFunctions::StateMachine",
"Properties": {
"DefinitionString": {
"Fn::Join": [
"",
[
"{\"StartAt\":\"EmrCreateCluster\",\"States\":{\"EmrCreateCluster\":{\"End\":true,\"Type\":\"Task\",\"Resource\":\"arn:",
{
"Ref": "AWS::Partition"
},
":states:::elasticmapreduce:createCluster.sync\",\"Parameters\":{\"Instances\":{\"InstanceFleets\":[{\"InstanceFleetType\":\"MASTER\",\"InstanceTypeConfigs\":[{\"InstanceType\":\"m5.xlarge\",\"WeightedCapacity\":1}],\"LaunchSpecifications\":{\"SpotSpecification\":{\"AllocationStrategy\":\"capacity-optimized\",\"TimeoutAction\":\"TERMINATE_CLUSTER\",\"TimeoutDurationMinutes\":60}},\"Name\":\"Master\",\"TargetSpotCapacity\":1}],\"KeepJobFlowAliveWhenNoSteps\":true},\"JobFlowRole\":\"",
{
"Ref": "EmrCreateClusterInstanceRoleC80466F5"
},
"\",\"Name\":\"Cluster\",\"ServiceRole\":\"",
{
"Ref": "EmrCreateClusterServiceRole5251910D"
},
"\",\"ReleaseLabel\":\"emr-5.36.1\",\"Tags\":[{\"Key\":\"Key\",\"Value\":\"Value\"}],\"VisibleToAllUsers\":true}}}}"
]
]
},
"RoleArn": {
"Fn::GetAtt": [
"SMRole49C19C48",
"Arn"
]
}
},
"DependsOn": [
"SMRoleDefaultPolicy34CA15C7",
"SMRole49C19C48"
],
"UpdateReplacePolicy": "Delete",
"DeletionPolicy": "Delete"
}
},
"Parameters": {
"BootstrapVersion": {
"Type": "AWS::SSM::Parameter::Value<String>",
"Default": "/cdk-bootstrap/hnb659fds/version",
"Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]"
}
},
"Rules": {
"CheckBootstrapVersion": {
"Assertions": [
{
"Assert": {
"Fn::Not": [
{
"Fn::Contains": [
[
"1",
"2",
"3",
"4",
"5"
],
{
"Ref": "BootstrapVersion"
}
]
}
]
},
"AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI."
}
]
}
}
}

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 5d7c1aa

Please sign in to comment.