From 0755561b79d6be0744b0b21504fe54ffcf2b618a Mon Sep 17 00:00:00 2001 From: Rico Hermans Date: Wed, 2 Oct 2024 14:16:19 +0200 Subject: [PATCH] feat(cli): `cdk rollback` (#31407) Add a CLI feature to roll a stuck change back. This is mostly useful for deployments performed using `--no-rollback`: if a failure occurs, the stack gets stuck in an `UPDATE_FAILED` state from which there are 2 options: - Try again using a new template - Roll back to the last stable state There used to be no way to perform the second operation using the CDK CLI, but there now is. `cdk rollback` works in 2 situations: - A paused fail state; it will initiating a fresh rollback (on `CREATE_FAILED`, `UPDATE_FAILED`). - A paused rollback state; it will retry the rollback, optionally skipping some resources (on `UPDATE_ROLLBACK_FAILED` -- it seems there is no way to continue a rollback in `ROLLBACK_FAILED` state). `cdk rollback --orphan ` can be used to skip resource rollbacks that are causing problems. `cdk rollback --force` will look up all failed resources and continue skipping them until the rollback has finished. This change requires new bootstrap permissions, so the bootstrap stack is updated to add the following IAM permissions to the `deploy-action` role: ``` - cloudformation:RollbackStack - cloudformation:ContinueUpdateRollback ``` These are necessary to call the 2 CloudFormation APIs that start and continue a rollback. Relates to (but does not close yet) #30546. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk-testing/cli-integ/README.md | 2 +- .../cli-integ/lib/with-cdk-app.ts | 19 +- .../cdk-apps/rollback-test-app/app.js | 100 ++++++ .../cdk-apps/rollback-test-app/cdk.json | 7 + .../tests/cli-integ-tests/cli.integtest.ts | 79 +++++ packages/aws-cdk/README.md | 63 ++-- .../lib/api/bootstrap/bootstrap-template.yaml | 4 +- packages/aws-cdk/lib/api/cxapp/exec.ts | 3 + packages/aws-cdk/lib/api/deployments.ts | 205 ++++++++++- .../cloudformation/stack-activity-monitor.ts | 90 ++--- .../util/cloudformation/stack-event-poller.ts | 172 +++++++++ .../api/util/cloudformation/stack-status.ts | 36 ++ packages/aws-cdk/lib/cdk-toolkit.ts | 86 +++++ packages/aws-cdk/lib/cli.ts | 31 ++ .../api/cloudformation-deployments.test.ts | 334 ++++++++++++------ .../test/api/fake-cloudformation-stack.ts | 22 +- .../test/api/stack-activity-monitor.test.ts | 12 + packages/aws-cdk/test/cdk-toolkit.test.ts | 33 +- packages/aws-cdk/test/util/mock-sdk.ts | 14 +- .../aws-cdk/test/util/stack-monitor.test.ts | 27 +- 20 files changed, 1121 insertions(+), 218 deletions(-) create mode 100644 packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/app.js create mode 100644 packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/cdk.json create mode 100644 packages/aws-cdk/lib/api/util/cloudformation/stack-event-poller.ts diff --git a/packages/@aws-cdk-testing/cli-integ/README.md b/packages/@aws-cdk-testing/cli-integ/README.md index 2dc2e9c70d8cc..d1dd485660151 100644 --- a/packages/@aws-cdk-testing/cli-integ/README.md +++ b/packages/@aws-cdk-testing/cli-integ/README.md @@ -37,7 +37,7 @@ Test suites are written as a collection of Jest tests, and they are run using Je ### Setup -Building the @aws-cdk-testing package is not very different from building the rest of the CDK. However, If you are having issues with the tests, you can ensure your enviornment is built properly by following the steps below: +Building the @aws-cdk-testing package is not very different from building the rest of the CDK. However, If you are having issues with the tests, you can ensure your environment is built properly by following the steps below: ```shell yarn install # Install dependencies diff --git a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts index b5778a0d1af0d..c28f5eccb4d4b 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts @@ -24,7 +24,8 @@ export const EXTENDED_TEST_TIMEOUT_S = 30 * 60; * For backwards compatibility with existing tests (so we don't have to change * too much) the inner block is expected to take a `TestFixture` object. */ -export function withCdkApp( +export function withSpecificCdkApp( + appName: string, block: (context: TestFixture) => Promise, ): (context: TestContext & AwsContext & DisableBootstrapContext) => Promise { return async (context: TestContext & AwsContext & DisableBootstrapContext) => { @@ -36,7 +37,7 @@ export function withCdkApp( context.output.write(` Test directory: ${integTestDir}\n`); context.output.write(` Region: ${context.aws.region}\n`); - await cloneDirectory(path.join(RESOURCES_DIR, 'cdk-apps', 'app'), integTestDir, context.output); + await cloneDirectory(path.join(RESOURCES_DIR, 'cdk-apps', appName), integTestDir, context.output); const fixture = new TestFixture( integTestDir, stackNamePrefix, @@ -87,6 +88,16 @@ export function withCdkApp( }; } +/** + * Like `withSpecificCdkApp`, but uses the default integration testing app with a million stacks in it + */ +export function withCdkApp( + block: (context: TestFixture) => Promise, +): (context: TestContext & AwsContext & DisableBootstrapContext) => Promise { + // 'app' is the name of the default integration app in the `cdk-apps` directory + return withSpecificCdkApp('app', block); +} + export function withCdkMigrateApp(language: string, block: (context: TestFixture) => Promise) { return async (context: A) => { const stackName = `cdk-migrate-${language}-integ-${context.randomString}`; @@ -188,6 +199,10 @@ export function withDefaultFixture(block: (context: TestFixture) => Promise Promise) { + return withAws(withTimeout(DEFAULT_TEST_TIMEOUT_S, withSpecificCdkApp(appName, block))); +} + export function withExtendedTimeoutFixture(block: (context: TestFixture) => Promise) { return withAws(withTimeout(EXTENDED_TEST_TIMEOUT_S, withCdkApp(block))); } diff --git a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/app.js b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/app.js new file mode 100644 index 0000000000000..419e30898c9bf --- /dev/null +++ b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/app.js @@ -0,0 +1,100 @@ +const cdk = require('aws-cdk-lib'); +const lambda = require('aws-cdk-lib/aws-lambda'); +const cr = require('aws-cdk-lib/custom-resources'); + +/** + * This stack will be deployed in multiple phases, to achieve a very specific effect + * + * It contains resources r1 and r2, where r1 gets deployed first. + * + * - PHASE = 1: both resources deploy regularly. + * - PHASE = 2a: r1 gets updated, r2 will fail to update + * - PHASE = 2b: r1 gets updated, r2 will fail to update, and r1 will fail its rollback. + * + * To exercise this app: + * + * ``` + * env PHASE=1 npx cdk deploy + * env PHASE=2b npx cdk deploy --no-rollback + * # This will leave the stack in UPDATE_FAILED + * + * env PHASE=2b npx cdk rollback + * # This will start a rollback that will fail because r1 fails its rollabck + * + * env PHASE=2b npx cdk rollback --force + * # This will retry the rollabck and skip r1 + * ``` + */ +class RollbacktestStack extends cdk.Stack { + constructor(scope, id, props) { + super(scope, id, props); + + let r1props = {}; + let r2props = {}; + + const phase = process.env.PHASE; + switch (phase) { + case '1': + // Normal deployment + break; + case '2a': + // r1 updates normally, r2 fails updating + r2props.FailUpdate = true; + break; + case '2b': + // r1 updates normally, r2 fails updating, r1 fails rollback + r1props.FailRollback = true; + r2props.FailUpdate = true; + break; + } + + const fn = new lambda.Function(this, 'Fun', { + runtime: lambda.Runtime.NODEJS_LATEST, + code: lambda.Code.fromInline(`exports.handler = async function(event, ctx) { + const key = \`Fail\${event.RequestType}\`; + if (event.ResourceProperties[key]) { + throw new Error(\`\${event.RequestType} fails!\`); + } + if (event.OldResourceProperties?.FailRollback) { + throw new Error('Failing rollback!'); + } + return {}; + }`), + handler: 'index.handler', + timeout: cdk.Duration.minutes(1), + }); + const provider = new cr.Provider(this, "MyProvider", { + onEventHandler: fn, + }); + + const r1 = new cdk.CustomResource(this, 'r1', { + serviceToken: provider.serviceToken, + properties: r1props, + }); + const r2 = new cdk.CustomResource(this, 'r2', { + serviceToken: provider.serviceToken, + properties: r2props, + }); + r2.node.addDependency(r1); + } +} + +const app = new cdk.App({ + context: { + '@aws-cdk/core:assetHashSalt': process.env.CODEBUILD_BUILD_ID, // Force all assets to be unique, but consistent in one build + }, +}); + +const defaultEnv = { + account: process.env.CDK_DEFAULT_ACCOUNT, + region: process.env.CDK_DEFAULT_REGION +}; + +const stackPrefix = process.env.STACK_NAME_PREFIX; +if (!stackPrefix) { + throw new Error(`the STACK_NAME_PREFIX environment variable is required`); +} + +// Sometimes we don't want to synthesize all stacks because it will impact the results +new RollbacktestStack(app, `${stackPrefix}-test-rollback`, { env: defaultEnv }); +app.synth(); \ No newline at end of file diff --git a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/cdk.json b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/cdk.json new file mode 100644 index 0000000000000..44809158dbdac --- /dev/null +++ b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/rollback-test-app/cdk.json @@ -0,0 +1,7 @@ +{ + "app": "node app.js", + "versionReporting": false, + "context": { + "aws-cdk:enableDiffNoFail": "true" + } +} diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts index f714f99b9bb0c..4bf11212617d2 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts @@ -32,6 +32,7 @@ import { withCDKMigrateFixture, withExtendedTimeoutFixture, randomString, + withSpecificFixture, withoutBootstrap, } from '../../lib'; @@ -2260,6 +2261,84 @@ integTest( }), ); +integTest( + 'test cdk rollback', + withSpecificFixture('rollback-test-app', async (fixture) => { + let phase = '1'; + + // Should succeed + await fixture.cdkDeploy('test-rollback', { + options: ['--no-rollback'], + modEnv: { PHASE: phase }, + verbose: false, + }); + try { + phase = '2a'; + + // Should fail + const deployOutput = await fixture.cdkDeploy('test-rollback', { + options: ['--no-rollback'], + modEnv: { PHASE: phase }, + verbose: false, + allowErrExit: true, + }); + expect(deployOutput).toContain('UPDATE_FAILED'); + + // Rollback + await fixture.cdk(['rollback'], { + modEnv: { PHASE: phase }, + verbose: false, + }); + } finally { + await fixture.cdkDestroy('test-rollback'); + } + }), +); + +integTest( + 'test cdk rollback --force', + withSpecificFixture('rollback-test-app', async (fixture) => { + let phase = '1'; + + // Should succeed + await fixture.cdkDeploy('test-rollback', { + options: ['--no-rollback'], + modEnv: { PHASE: phase }, + verbose: false, + }); + try { + phase = '2b'; // Fail update and also fail rollback + + // Should fail + const deployOutput = await fixture.cdkDeploy('test-rollback', { + options: ['--no-rollback'], + modEnv: { PHASE: phase }, + verbose: false, + allowErrExit: true, + }); + + expect(deployOutput).toContain('UPDATE_FAILED'); + + // Should still fail + const rollbackOutput = await fixture.cdk(['rollback'], { + modEnv: { PHASE: phase }, + verbose: false, + allowErrExit: true, + }); + + expect(rollbackOutput).toContain('Failing rollback'); + + // Rollback and force cleanup + await fixture.cdk(['rollback', '--force'], { + modEnv: { PHASE: phase }, + verbose: false, + }); + } finally { + await fixture.cdkDestroy('test-rollback'); + } + }), +); + integTest('cdk notices are displayed correctly', withDefaultFixture(async (fixture) => { const cache = { diff --git a/packages/aws-cdk/README.md b/packages/aws-cdk/README.md index ca79343503706..227a040cd5fb2 100644 --- a/packages/aws-cdk/README.md +++ b/packages/aws-cdk/README.md @@ -19,6 +19,7 @@ The AWS CDK Toolkit provides the `cdk` command-line interface that can be used t | [`cdk synth`](#cdk-synthesize) | Synthesize a CDK app to CloudFormation template(s) | | [`cdk diff`](#cdk-diff) | Diff stacks against current state | | [`cdk deploy`](#cdk-deploy) | Deploy a stack into an AWS account | +| [`cdk rollback`](#cdk-rollback) | Roll back a failed deployment | | [`cdk import`](#cdk-import) | Import existing AWS resources into a CDK stack | | [`cdk migrate`](#cdk-migrate) | Migrate AWS resources, CloudFormation stacks, and CloudFormation templates to CDK | | [`cdk watch`](#cdk-watch) | Watches a CDK app for deployable and hotswappable changes | @@ -204,6 +205,10 @@ $ cdk deploy --no-rollback $ cdk deploy -R ``` +If a deployment fails you can update your code and immediately retry the +deployment from the point of failure. If you would like to explicitly roll back a failed, paused deployment, +use `cdk rollback`. + NOTE: you cannot use `--no-rollback` for any updates that would cause a resource replacement, only for updates and creations of new resources. @@ -397,7 +402,7 @@ development, your prod app may not have any resources or the resources are comme out. In this scenario, you will receive an error message stating that the app has no stacks. -To bypass this error messages, you can pass the `--ignore-no-stacks` flag to the +To bypass this error messages, you can pass the `--ignore-no-stacks` flag to the `deploy` command: ```console @@ -468,6 +473,24 @@ and might have breaking changes in the future. > *: `Fn::GetAtt` is only partially supported. Refer to [this implementation](https://github.com/aws/aws-cdk/blob/main/packages/aws-cdk/lib/api/evaluate-cloudformation-template.ts#L477-L492) for supported resources and attributes. +### `cdk rollback` + +If a deployment performed using `cdk deploy --no-rollback` fails, your +deployment will be left in a failed, paused state. From this state you can +update your code and try the deployment again, or roll the deployment back to +the last stable state. + +To roll the deployment back, use `cdk rollback`. This will initiate a rollback +to the last stable state of your stack. + +Some resources may fail to roll back. If they do, you can try again by calling +`cdk rollback --orphan ` (can be specified multiple times). Or, run +`cdk rollback --force` to have the CDK CLI automatically orphan all failing +resources. + +(`cdk rollback` requires version 23 of the bootstrap stack, since it depends on +new permissions necessary to call the appropriate CloudFormation APIs) + ### `cdk watch` The `watch` command is similar to `deploy`, @@ -598,9 +621,9 @@ This feature currently has the following limitations: ### `cdk migrate` -⚠️**CAUTION**⚠️: CDK Migrate is currently experimental and may have breaking changes in the future. +⚠️**CAUTION**⚠️: CDK Migrate is currently experimental and may have breaking changes in the future. -CDK Migrate generates a CDK app from deployed AWS resources using `--from-scan`, deployed AWS CloudFormation stacks using `--from-stack`, and local AWS CloudFormation templates using `--from-path`. +CDK Migrate generates a CDK app from deployed AWS resources using `--from-scan`, deployed AWS CloudFormation stacks using `--from-stack`, and local AWS CloudFormation templates using `--from-path`. To learn more about the CDK Migrate feature, see [Migrate to AWS CDK](https://docs.aws.amazon.com/cdk/v2/guide/migrate.html). For more information on `cdk migrate` command options, see [cdk migrate command reference](https://docs.aws.amazon.com/cdk/v2/guide/ref-cli-cdk-migrate.html). @@ -632,7 +655,7 @@ Account and Region information are retrieved from default CDK CLI sources. Use ` $ cdk migrate --language typescript --from-scan --stack-name "myCloudFormationStack" ``` -Since CDK Migrate relies on the IaC generator service, any limitations of IaC generator will apply to CDK Migrate. For general limitations, see [Considerations](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/generate-IaC.html#generate-template-considerations). +Since CDK Migrate relies on the IaC generator service, any limitations of IaC generator will apply to CDK Migrate. For general limitations, see [Considerations](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/generate-IaC.html#generate-template-considerations). IaC generator limitations with discovering resource and property values will also apply here. As a result, CDK Migrate will only migrate resources supported by IaC generator. Some of your resources may not be supported and some property values may not be accessible. For more information, see [Iac generator and write-only properties](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/generate-IaC-write-only-properties.html) and [Supported resource types](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/generate-IaC-supported-resources.html). @@ -649,8 +672,8 @@ $ # template.json is a valid cloudformation template in the local directory $ cdk migrate --stack-name MyAwesomeApplication --language typescript --from-path MyTemplate.json ``` -This command generates a new directory named `MyAwesomeApplication` within your current working directory, and -then initializes a new CDK application within that directory. The CDK app contains a `MyAwesomeApplication` stack with resources configured to match those in your local CloudFormation template. +This command generates a new directory named `MyAwesomeApplication` within your current working directory, and +then initializes a new CDK application within that directory. The CDK app contains a `MyAwesomeApplication` stack with resources configured to match those in your local CloudFormation template. This results in a CDK application with the following structure, where the lib directory contains a stack definition with the same resource configuration as the provided template.json. @@ -680,13 +703,13 @@ This will generate a Python CDK app which will synthesize the same configuration ##### Generate a TypeScript CDK app from deployed AWS resources that are not associated with a stack -If you have resources in your account that were provisioned outside AWS IaC tools and would like to manage them with the CDK, you can use the `--from-scan` option to generate the application. +If you have resources in your account that were provisioned outside AWS IaC tools and would like to manage them with the CDK, you can use the `--from-scan` option to generate the application. In this example, we use the `--filter` option to specify which resources to migrate. You can filter resources to limit the number of resources migrated to only those specified by the `--filter` option, including any resources they depend on, or resources that depend on them (for example A filter which specifies a single Lambda Function, will find that specific table and any alarms that may monitor it). The `--filter` argument offers both AND as well as OR filtering. OR filtering can be specified by passing multiple `--filter` options, and AND filtering can be specified by passing a single `--filter` option with multiple comma separated key/value pairs as seen below (see below for examples). It is recommended to use the `--filter` option to limit the number of resources returned as some resource types provide sample resources by default in all accounts which can add to the resource limits. -`--from-scan` takes 3 potential arguments: `--new`, `most-recent`, and undefined. If `--new` is passed, CDK Migrate will initiate a new scan of the account and use that new scan to discover resources. If `--most-recent` is passed, CDK Migrate will use the most recent scan of the account to discover resources. If neither `--new` nor `--most-recent` are passed, CDK Migrate will take the most recent scan of the account to discover resources, unless there is no recent scan, in which case it will initiate a new scan. +`--from-scan` takes 3 potential arguments: `--new`, `most-recent`, and undefined. If `--new` is passed, CDK Migrate will initiate a new scan of the account and use that new scan to discover resources. If `--most-recent` is passed, CDK Migrate will use the most recent scan of the account to discover resources. If neither `--new` nor `--most-recent` are passed, CDK Migrate will take the most recent scan of the account to discover resources, unless there is no recent scan, in which case it will initiate a new scan. ```console # Filtering options @@ -719,14 +742,14 @@ $ cdk migrate --stack-name MyAwesomeApplication --language typescript --from-sca - CDK Migrate will only generate L1 constructs and does not currently support any higher level abstractions. - CDK Migrate successfully generating an application does *not* guarantee the application is immediately deployable. -It simply generates a CDK application which will synthesize a template that has identical resource configurations -to the provided template. +It simply generates a CDK application which will synthesize a template that has identical resource configurations +to the provided template. - - CDK Migrate does not interact with the CloudFormation service to verify the template -provided can deploy on its own. Although by default any CDK app generated using the `--from-scan` option exclude -CloudFormation managed resources, CDK Migrate will not verify prior to deployment that any resources scanned, or in the provided + - CDK Migrate does not interact with the CloudFormation service to verify the template +provided can deploy on its own. Although by default any CDK app generated using the `--from-scan` option exclude +CloudFormation managed resources, CDK Migrate will not verify prior to deployment that any resources scanned, or in the provided template are already managed in other CloudFormation templates, nor will it verify that the resources in the provided -template are available in the desired regions, which may impact ADC or Opt-In regions. +template are available in the desired regions, which may impact ADC or Opt-In regions. - If the provided template has parameters without default values, those will need to be provided before deploying the generated application. @@ -743,13 +766,13 @@ In practice this is how CDK Migrate generated applications will operate in the f ##### **The provided template is already deployed to CloudFormation in the account/region** -If the provided template came directly from a deployed CloudFormation stack, and that stack has not experienced any drift, +If the provided template came directly from a deployed CloudFormation stack, and that stack has not experienced any drift, then the generated application will be immediately deployable, and will not cause any changes to the deployed resources. Drift might occur if a resource in your template was modified outside of CloudFormation, namely via the AWS Console or AWS CLI. ##### **The provided template is not deployed to CloudFormation in the account/region, and there *is not* overlap with existing resources in the account/region** -If the provided template represents a set of resources that have no overlap with resources already deployed in the account/region, +If the provided template represents a set of resources that have no overlap with resources already deployed in the account/region, then the generated application will be immediately deployable. This could be because the stack has never been deployed, or the application was generated from a stack deployed in another account/region. @@ -766,16 +789,16 @@ In practice this means for any resource in the provided template, for example, } ``` -There must not exist a resource of that type with the same identifier in the desired region. In this example that identfier +There must not exist a resource of that type with the same identifier in the desired region. In this example that identfier would be "MyBucket" ##### **The provided template is not deployed to CloudFormation in the account/region, and there *is* overlap with existing resources in the account/region** -If the provided template represents a set of resources that overlap with resources already deployed in the account/region, -then the generated application will not be immediately deployable. If those overlapped resources are already managed by +If the provided template represents a set of resources that overlap with resources already deployed in the account/region, +then the generated application will not be immediately deployable. If those overlapped resources are already managed by another CloudFormation stack in that account/region, then those resources will need to be manually removed from the provided template. Otherwise, if the overlapped resources are not managed by another CloudFormation stack, then first remove those -resources from your CDK Application Stack, deploy the cdk application successfully, then re-add them and run `cdk import` +resources from your CDK Application Stack, deploy the cdk application successfully, then re-add them and run `cdk import` to import them into your deployed stack. ### `cdk destroy` diff --git a/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml b/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml index 8ed4bb8595446..ad71c39535426 100644 --- a/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml +++ b/packages/aws-cdk/lib/api/bootstrap/bootstrap-template.yaml @@ -485,6 +485,8 @@ Resources: - cloudformation:ExecuteChangeSet - cloudformation:CreateStack - cloudformation:UpdateStack + - cloudformation:RollbackStack + - cloudformation:ContinueUpdateRollback Resource: "*" - Sid: PipelineCrossAccountArtifactsBucket # Read/write buckets in different accounts. Permissions to buckets in @@ -651,7 +653,7 @@ Resources: Type: String Name: Fn::Sub: '/cdk-bootstrap/${Qualifier}/version' - Value: '22' + Value: '23' Outputs: BucketName: Description: The name of the S3 bucket owned by the CDK toolkit stack diff --git a/packages/aws-cdk/lib/api/cxapp/exec.ts b/packages/aws-cdk/lib/api/cxapp/exec.ts index 31f2fca029dd9..6b62d7ae2527f 100644 --- a/packages/aws-cdk/lib/api/cxapp/exec.ts +++ b/packages/aws-cdk/lib/api/cxapp/exec.ts @@ -49,6 +49,9 @@ export async function execProgram(aws: SdkProvider, config: Configuration): Prom if (!outdir) { throw new Error('unexpected: --output is required'); } + if (typeof outdir !== 'string') { + throw new Error(`--output takes a string, got ${JSON.stringify(outdir)}`); + } try { await fs.mkdirp(outdir); } catch (error: any) { diff --git a/packages/aws-cdk/lib/api/deployments.ts b/packages/aws-cdk/lib/api/deployments.ts index 5a1422b4ea616..f3aae0bec571a 100644 --- a/packages/aws-cdk/lib/api/deployments.ts +++ b/packages/aws-cdk/lib/api/deployments.ts @@ -1,3 +1,4 @@ +import { randomUUID } from 'crypto'; import * as cxapi from '@aws-cdk/cx-api'; import * as cdk_assets from 'cdk-assets'; import { AssetManifest, IManifestEntry } from 'cdk-assets'; @@ -11,12 +12,16 @@ import { deployStack, DeployStackResult, destroyStack, DeploymentMethod } from ' import { EnvironmentResources, EnvironmentResourcesRegistry } from './environment-resources'; import { HotswapMode } from './hotswap/common'; import { loadCurrentTemplateWithNestedStacks, loadCurrentTemplate, RootTemplateWithNestedStacks } from './nested-stack-helpers'; -import { CloudFormationStack, Template, ResourcesToImport, ResourceIdentifierSummaries } from './util/cloudformation'; -import { StackActivityProgress } from './util/cloudformation/stack-activity-monitor'; +import { CloudFormationStack, Template, ResourcesToImport, ResourceIdentifierSummaries, stabilizeStack } from './util/cloudformation'; +import { StackActivityMonitor, StackActivityProgress } from './util/cloudformation/stack-activity-monitor'; +import { StackEventPoller } from './util/cloudformation/stack-event-poller'; +import { RollbackChoice } from './util/cloudformation/stack-status'; import { replaceEnvPlaceholders } from './util/placeholders'; import { makeBodyParameterAndUpload } from './util/template-body-parameter'; import { buildAssets, publishAssets, BuildAssetsOptions, PublishAssetsOptions, PublishingAws, EVENT_TO_LOGGER } from '../util/asset-publishing'; +const BOOTSTRAP_STACK_VERSION_FOR_ROLLBACK = 23; + /** * SDK obtained by assuming the lookup role * for a given environment @@ -209,6 +214,77 @@ export interface DeployStackOptions { ignoreNoStacks?: boolean; } +export interface RollbackStackOptions { + /** + * Stack to roll back + */ + readonly stack: cxapi.CloudFormationStackArtifact; + + /** + * Execution role for the deployment (pass through to CloudFormation) + * + * @default - Current role + */ + readonly roleArn?: string; + + /** + * Don't show stack deployment events, just wait + * + * @default false + */ + readonly quiet?: boolean; + + /** + * Whether we are on a CI system + * + * @default false + */ + readonly ci?: boolean; + + /** + * Name of the toolkit stack, if not the default name + * + * @default 'CDKToolkit' + */ + readonly toolkitStackName?: string; + + /** + * Whether to force a rollback or not + * + * Forcing a rollback will orphan all undeletable resources. + * + * @default false + */ + readonly force?: boolean; + + /** + * Orphan the resources with the given logical IDs + * + * @default - No orphaning + */ + readonly orphanLogicalIds?: string[]; + + /** + * Display mode for stack deployment progress. + * + * @default - StackActivityProgress.Bar - stack events will be displayed for + * the resource currently being deployed. + */ + readonly progress?: StackActivityProgress; + + /** + * Whether to validate the version of the bootstrap stack permissions + * + * @default true + */ + readonly validateBootstrapStackVersion?: boolean; +} + +export interface RollbackStackResult { + readonly notInRollbackableState?: boolean; + readonly success?: boolean; +} + interface AssetOptions { /** * Stack with assets to build. @@ -418,6 +494,125 @@ export class Deployments { }); } + public async rollbackStack(options: RollbackStackOptions): Promise { + let resourcesToSkip: string[] = options.orphanLogicalIds ?? []; + if (options.force && resourcesToSkip.length > 0) { + throw new Error('Cannot combine --force with --orphan'); + } + + const { + stackSdk, + resolvedEnvironment: _, + cloudFormationRoleArn, + envResources, + } = await this.prepareSdkFor(options.stack, options.roleArn, Mode.ForWriting); + + if (options.validateBootstrapStackVersion ?? true) { + // Do a verification of the bootstrap stack version + await this.validateBootstrapStackVersion( + options.stack.stackName, + BOOTSTRAP_STACK_VERSION_FOR_ROLLBACK, + options.stack.bootstrapStackVersionSsmParameter, + envResources); + } + + const cfn = stackSdk.cloudFormation(); + const deployName = options.stack.stackName; + + // We loop in case of `--force` and the stack ends up in `CONTINUE_UPDATE_ROLLBACK`. + let maxLoops = 10; + while (maxLoops--) { + let cloudFormationStack = await CloudFormationStack.lookup(cfn, deployName); + + switch (cloudFormationStack.stackStatus.rollbackChoice) { + case RollbackChoice.NONE: + warning(`Stack ${deployName} does not need a rollback: ${cloudFormationStack.stackStatus}`); + return { notInRollbackableState: true }; + + case RollbackChoice.START_ROLLBACK: + debug(`Initiating rollback of stack ${deployName}`); + await cfn.rollbackStack({ + StackName: deployName, + RoleARN: cloudFormationRoleArn, + ClientRequestToken: randomUUID(), + // Enabling this is just the better overall default, the only reason it isn't the upstream default is backwards compatibility + RetainExceptOnCreate: true, + }).promise(); + break; + + case RollbackChoice.CONTINUE_UPDATE_ROLLBACK: + if (options.force) { + // Find the failed resources from the deployment and automatically skip them + // (Using deployment log because we definitely have `DescribeStackEvents` permissions, and we might not have + // `DescribeStackResources` permissions). + const poller = new StackEventPoller(cfn, { + stackName: deployName, + stackStatuses: ['ROLLBACK_IN_PROGRESS', 'UPDATE_ROLLBACK_IN_PROGRESS'], + }); + await poller.poll(); + resourcesToSkip = poller.resourceErrors + .filter(r => !r.isStackEvent && r.parentStackLogicalIds.length === 0) + .map(r => r.event.LogicalResourceId ?? ''); + } + + const skipDescription = resourcesToSkip.length > 0 + ? ` (orphaning: ${resourcesToSkip.join(', ')})` + : ''; + warning(`Continuing rollback of stack ${deployName}${skipDescription}`); + await cfn.continueUpdateRollback({ + StackName: deployName, + ClientRequestToken: randomUUID(), + RoleARN: cloudFormationRoleArn, + ResourcesToSkip: resourcesToSkip, + }).promise(); + break; + + case RollbackChoice.ROLLBACK_FAILED: + warning(`Stack ${deployName} failed creation and rollback. This state cannot be rolled back. You can recreate this stack by running 'cdk deploy'.`); + return { notInRollbackableState: true }; + + default: + throw new Error(`Unexpected rollback choice: ${cloudFormationStack.stackStatus.rollbackChoice}`); + } + + const monitor = options.quiet ? undefined : StackActivityMonitor.withDefaultPrinter(cfn, deployName, options.stack, { + ci: options.ci, + }).start(); + + let stackErrorMessage: string | undefined = undefined; + let finalStackState = cloudFormationStack; + try { + const successStack = await stabilizeStack(cfn, deployName); + + // This shouldn't really happen, but catch it anyway. You never know. + if (!successStack) { throw new Error('Stack deploy failed (the stack disappeared while we were rolling it back)'); } + finalStackState = successStack; + + const errors = monitor?.errors?.join(', '); + if (errors) { + stackErrorMessage = errors; + } + } catch (e: any) { + stackErrorMessage = suffixWithErrors(e.message, monitor?.errors); + } finally { + await monitor?.stop(); + } + + if (finalStackState.stackStatus.isRollbackSuccess || !stackErrorMessage) { + return { success: true }; + } + + // Either we need to ignore some resources to continue the rollback, or something went wrong + if (finalStackState.stackStatus.rollbackChoice === RollbackChoice.CONTINUE_UPDATE_ROLLBACK && options.force) { + // Do another loop-de-loop + continue; + } + + throw new Error(`${stackErrorMessage} (fix problem and retry, or orphan these resources using --orphan or --force)`);; + } + throw new Error('Rollback did not finish after a large number of iterations; stopping because it looks like we\'re not making progress anymore. You can retry if rollback was progressing as expected.'); + } + public async destroyStack(options: DestroyStackOptions): Promise { const { stackSdk, cloudFormationRoleArn: roleArn } = await this.prepareSdkFor(options.stack, options.roleArn, Mode.ForWriting); @@ -729,3 +924,9 @@ class ParallelSafeAssetProgress implements cdk_assets.IPublishProgressListener { */ export class CloudFormationDeployments extends Deployments { } + +function suffixWithErrors(msg: string, errors?: string[]) { + return errors && errors.length > 0 + ? `${msg}: ${errors.join(', ')}` + : msg; +} \ No newline at end of file diff --git a/packages/aws-cdk/lib/api/util/cloudformation/stack-activity-monitor.ts b/packages/aws-cdk/lib/api/util/cloudformation/stack-activity-monitor.ts index 1b2422a219168..6db3b7f67941c 100644 --- a/packages/aws-cdk/lib/api/util/cloudformation/stack-activity-monitor.ts +++ b/packages/aws-cdk/lib/api/util/cloudformation/stack-activity-monitor.ts @@ -3,11 +3,11 @@ import * as cxschema from '@aws-cdk/cloud-assembly-schema'; import * as cxapi from '@aws-cdk/cx-api'; import * as aws from 'aws-sdk'; import * as chalk from 'chalk'; +import { ResourceEvent, StackEventPoller } from './stack-event-poller'; import { error, logLevel, LogLevel, setLogLevel } from '../../../logging'; import { RewritableBlock } from '../display'; -export interface StackActivity { - readonly event: aws.CloudFormation.StackEvent; +export interface StackActivity extends ResourceEvent { readonly metadata?: ResourceMetadata; } @@ -116,17 +116,13 @@ export class StackActivityMonitor { } /** - * Resource errors found while monitoring the deployment + * The poller used to read stack events */ - public readonly errors = new Array(); + public readonly poller: StackEventPoller; - private active = false; - private activity: { [eventId: string]: StackActivity } = { }; + public readonly errors: string[] = []; - /** - * Determines which events not to display - */ - private readonly startTime: number; + private active = false; /** * Current tick timer @@ -139,13 +135,16 @@ export class StackActivityMonitor { private readPromise?: Promise; constructor( - private readonly cfn: aws.CloudFormation, + cfn: aws.CloudFormation, private readonly stackName: string, private readonly printer: IActivityPrinter, private readonly stack?: cxapi.CloudFormationStackArtifact, changeSetCreationTime?: Date, ) { - this.startTime = changeSetCreationTime?.getTime() ?? Date.now(); + this.poller = new StackEventPoller(cfn, { + stackName, + startTime: changeSetCreationTime?.getTime() ?? Date.now(), + }); } public start() { @@ -221,61 +220,17 @@ export class StackActivityMonitor { * see a next page and the last event in the page is new to us (and within the time window). * haven't seen the final event */ - private async readNewEvents(stackName?: string): Promise { - const stackToPollForEvents = stackName ?? this.stackName; - const events: StackActivity[] = []; - const CFN_SUCCESS_STATUS = ['UPDATE_COMPLETE', 'CREATE_COMPLETE', 'DELETE_COMPLETE', 'DELETE_SKIPPED']; - try { - let nextToken: string | undefined; - let finished = false; - while (!finished) { - const response = await this.cfn.describeStackEvents({ StackName: stackToPollForEvents, NextToken: nextToken }).promise(); - const eventPage = response?.StackEvents ?? []; - - for (const event of eventPage) { - // Event from before we were interested in 'em - if (event.Timestamp.valueOf() < this.startTime) { - finished = true; - break; - } - - // Already seen this one - if (event.EventId in this.activity) { - finished = true; - break; - } - - // Fresh event - events.push(this.activity[event.EventId] = { - event: event, - metadata: this.findMetadataFor(event.LogicalResourceId), - }); - - if (event.ResourceType === 'AWS::CloudFormation::Stack' && !CFN_SUCCESS_STATUS.includes(event.ResourceStatus ?? '')) { - // If the event is not for `this` stack and has a physical resource Id, recursively call for events in the nested stack - if (event.PhysicalResourceId && event.PhysicalResourceId !== stackToPollForEvents) { - await this.readNewEvents(event.PhysicalResourceId); - } - } - } + private async readNewEvents(): Promise { + const pollEvents = await this.poller.poll(); - // We're also done if there's nothing left to read - nextToken = response?.NextToken; - if (nextToken === undefined) { - finished = true; - } - } - } catch (e: any) { - if (e.code === 'ValidationError' && e.message === `Stack [${stackToPollForEvents}] does not exist`) { - return; - } - throw e; - } + const activities: StackActivity[] = pollEvents.map(event => ({ + ...event, + metadata: this.findMetadataFor(event.event.LogicalResourceId), + })); - events.reverse(); - for (const event of events) { - this.checkForErrors(event); - this.printer.addActivity(event); + for (const activity of activities) { + this.checkForErrors(activity); + this.printer.addActivity(activity ); } } @@ -298,6 +253,7 @@ export class StackActivityMonitor { } private checkForErrors(activity: StackActivity) { + if (hasErrorMessage(activity.event.ResourceStatus ?? '')) { const isCancelled = (activity.event.ResourceStatusReason ?? '').indexOf('cancelled') > -1; @@ -550,7 +506,7 @@ export class HistoryActivityPrinter extends ActivityPrinterBase { this.stream.write('\nFailed resources:\n'); for (const failure of this.failures) { // Root stack failures are not interesting - if (failure.event.StackName === failure.event.LogicalResourceId) { + if (failure.isStackEvent) { continue; } @@ -707,7 +663,7 @@ export class CurrentActivityPrinter extends ActivityPrinterBase { const lines = new Array(); for (const failure of this.failures) { // Root stack failures are not interesting - if (failure.event.StackName === failure.event.LogicalResourceId) { + if (failure.isStackEvent) { continue; } diff --git a/packages/aws-cdk/lib/api/util/cloudformation/stack-event-poller.ts b/packages/aws-cdk/lib/api/util/cloudformation/stack-event-poller.ts new file mode 100644 index 0000000000000..8bc218a568ac3 --- /dev/null +++ b/packages/aws-cdk/lib/api/util/cloudformation/stack-event-poller.ts @@ -0,0 +1,172 @@ +import * as aws from 'aws-sdk'; + +export interface StackEventPollerProps { + /** + * The stack to poll + */ + readonly stackName: string; + + /** + * IDs of parent stacks of this resource, in case of resources in nested stacks + */ + readonly parentStackLogicalIds?: string[]; + + /** + * Timestamp for the oldest event we're interested in + * + * @default - Read all events + */ + readonly startTime?: number; + + /** + * Stop reading when we see the stack entering this status + * + * Should be something like `CREATE_IN_PROGRESS`, `UPDATE_IN_PROGRESS`, + * `DELETE_IN_PROGRESS, `ROLLBACK_IN_PROGRESS`. + * + * @default - Read all events + */ + readonly stackStatuses?: string[]; +} + +export interface ResourceEvent { + readonly event: aws.CloudFormation.StackEvent; + readonly parentStackLogicalIds: string[]; + + /** + * Whether this event regards the root stack + * + * @default false + */ + readonly isStackEvent?: boolean; +} + +export class StackEventPoller { + public readonly events: ResourceEvent[] = []; + public complete: boolean = false; + + private readonly eventIds = new Set(); + private readonly nestedStackPollers: Record = {}; + + constructor(private readonly cfn: aws.CloudFormation, private readonly props: StackEventPollerProps) { + } + + /** + * From all accumulated events, return only the errors + */ + public get resourceErrors(): ResourceEvent[] { + return this.events.filter(e => e.event.ResourceStatus?.endsWith('_FAILED') && !e.isStackEvent); + } + + /** + * Poll for new stack events + * + * Will not return events older than events indicated by the constructor filters. + * + * Recurses into nested stacks, and returns events old-to-new. + */ + public async poll(): Promise { + const events: ResourceEvent[] = []; + try { + let nextToken: string | undefined; + let finished = false; + while (!finished) { + const response = await this.cfn.describeStackEvents({ StackName: this.props.stackName, NextToken: nextToken }).promise(); + const eventPage = response?.StackEvents ?? []; + + for (const event of eventPage) { + // Event from before we were interested in 'em + if (this.props.startTime !== undefined && event.Timestamp.valueOf() < this.props.startTime) { + finished = true; + break; + } + + // Already seen this one + if (this.eventIds.has(event.EventId)) { + finished = true; + break; + } + this.eventIds.add(event.EventId); + + // The events for the stack itself are also included next to events about resources; we can test for them in this way. + const isParentStackEvent = event.PhysicalResourceId === event.StackId; + + if (isParentStackEvent && this.props.stackStatuses?.includes(event.ResourceStatus ?? '')) { + finished = true; + break; + } + + // Fresh event + const resEvent: ResourceEvent = { + event: event, + parentStackLogicalIds: this.props.parentStackLogicalIds ?? [], + isStackEvent: isParentStackEvent, + }; + events.push(resEvent); + + if (!isParentStackEvent && event.ResourceType === 'AWS::CloudFormation::Stack' && isStackBeginOperationState(event.ResourceStatus)) { + // If the event is not for `this` stack and has a physical resource Id, recursively call for events in the nested stack + this.trackNestedStack(event, [...this.props.parentStackLogicalIds ?? [], event.LogicalResourceId ?? '']); + } + + if (isParentStackEvent && isStackTerminalState(event.ResourceStatus)) { + this.complete = true; + } + } + + // We're also done if there's nothing left to read + nextToken = response?.NextToken; + if (nextToken === undefined) { + finished = true; + } + } + } catch (e: any) { + if (e.code === 'ValidationError' && e.message === `Stack [${this.props.stackName}] does not exist`) { + // Ignore + } else { + throw e; + } + } + + // Also poll all nested stacks we're currently tracking + for (const [logicalId, poller] of Object.entries(this.nestedStackPollers)) { + events.push(...await poller.poll()); + if (poller.complete) { + delete this.nestedStackPollers[logicalId]; + } + } + + // Return what we have so far + events.sort((a, b) => a.event.Timestamp.valueOf() - b.event.Timestamp.valueOf()); + this.events.push(...events); + return events; + } + + /** + * On the CREATE_IN_PROGRESS, UPDATE_IN_PROGRESS, DELETE_IN_PROGRESS event of a nested stack, poll the nested stack updates + */ + private trackNestedStack(event: aws.CloudFormation.StackEvent, parentStackLogicalIds: string[]) { + const logicalId = event.LogicalResourceId ?? ''; + if (!this.nestedStackPollers[logicalId]) { + this.nestedStackPollers[logicalId] = new StackEventPoller(this.cfn, { + stackName: event.PhysicalResourceId ?? '', + parentStackLogicalIds: parentStackLogicalIds, + startTime: event.Timestamp.valueOf(), + }); + } + } +} + +function isStackBeginOperationState(state: string | undefined) { + return [ + 'CREATE_IN_PROGRESS', + 'UPDATE_IN_PROGRESS', + 'DELETE_IN_PROGRESS', + 'UPDATE_ROLLBACK_IN_PROGRESS', + 'ROLLBACK_IN_PROGRESS', + ].includes(state ?? ''); +} + +function isStackTerminalState(state: string | undefined) { + return !(state ?? '').endsWith('_IN_PROGRESS'); +} \ No newline at end of file diff --git a/packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts b/packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts index 473858b4bac18..4dd113aaa30db 100644 --- a/packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts +++ b/packages/aws-cdk/lib/api/util/cloudformation/stack-status.ts @@ -46,7 +46,43 @@ export class StackStatus { || this.name === 'UPDATE_ROLLBACK_COMPLETE'; } + /** + * Whether the stack is in a paused state due to `--no-rollback`. + * + * The possible actions here are retrying a new `--no-rollback` deployment, or initiating a rollback. + */ + get rollbackChoice(): RollbackChoice { + switch (this.name) { + case 'CREATE_FAILED': + case 'UPDATE_FAILED': + return RollbackChoice.START_ROLLBACK; + case 'UPDATE_ROLLBACK_FAILED': + return RollbackChoice.CONTINUE_UPDATE_ROLLBACK; + case 'ROLLBACK_FAILED': + // Unfortunately there is no option to continue a failed rollback without + // a stable target state. + return RollbackChoice.ROLLBACK_FAILED; + default: + return RollbackChoice.NONE; + } + } + public toString(): string { return this.name + (this.reason ? ` (${this.reason})` : ''); } } + +/** + * Describe the current rollback options for this state + */ +export enum RollbackChoice { + START_ROLLBACK, + CONTINUE_UPDATE_ROLLBACK, + /** + * A sign that stack creation AND its rollback have failed. + * + * There is no way to recover from this, other than recreating the stack. + */ + ROLLBACK_FAILED, + NONE, +} \ No newline at end of file diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index 51c0b47a35b0f..e8f59b2fbda5b 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -430,6 +430,50 @@ export class CdkToolkit { }); } + /** + * Roll back the given stack or stacks. + */ + public async rollback(options: RollbackOptions) { + const startSynthTime = new Date().getTime(); + const stackCollection = await this.selectStacksForDeploy(options.selector, true); + const elapsedSynthTime = new Date().getTime() - startSynthTime; + print('\n✨ Synthesis time: %ss\n', formatTime(elapsedSynthTime)); + + if (stackCollection.stackCount === 0) { + // eslint-disable-next-line no-console + console.error('No stacks selected'); + return; + } + + let anyRollbackable = false; + + for (const stack of stackCollection.stackArtifacts) { + print('Rolling back %s', chalk.bold(stack.displayName)); + const startRollbackTime = new Date().getTime(); + try { + const result = await this.props.deployments.rollbackStack({ + stack, + roleArn: options.roleArn, + toolkitStackName: options.toolkitStackName, + force: options.force, + validateBootstrapStackVersion: options.validateBootstrapStackVersion, + orphanLogicalIds: options.orphanLogicalIds, + }); + if (!result.notInRollbackableState) { + anyRollbackable = true; + } + const elapsedRollbackTime = new Date().getTime() - startRollbackTime; + print('\n✨ Rollback time: %ss\n', formatTime(elapsedRollbackTime)); + } catch (e: any) { + error('\n ❌ %s failed: %s', chalk.bold(stack.displayName), e.message); + throw new Error('Rollback failed (use --force to orphan failing resources)'); + } + } + if (!anyRollbackable) { + throw new Error('No stacks were in a state that could be rolled back'); + } + } + public async watch(options: WatchOptions) { const rootDir = path.dirname(path.resolve(PROJECT_CONFIG)); debug("root directory used for 'watch' is: %s", rootDir); @@ -1345,6 +1389,48 @@ export interface DeployOptions extends CfnDeployOptions, WatchOptions { readonly ignoreNoStacks?: boolean; } +export interface RollbackOptions { + /** + * Criteria for selecting stacks to deploy + */ + readonly selector: StackSelector; + + /** + * Name of the toolkit stack to use/deploy + * + * @default CDKToolkit + */ + readonly toolkitStackName?: string; + + /** + * Role to pass to CloudFormation for deployment + * + * @default - Default stack role + */ + readonly roleArn?: string; + + /** + * Whether to force the rollback or not + * + * @default false + */ + readonly force?: boolean; + + /** + * Logical IDs of resources to orphan + * + * @default - No orphaning + */ + readonly orphanLogicalIds?: string[]; + + /** + * Whether to validate the version of the bootstrap stack permissions + * + * @default true + */ + readonly validateBootstrapStackVersion?: boolean; +} + export interface ImportOptions extends CfnDeployOptions { /** * Build a physical resource mapping and write it to the given file, without performing the actual import operation diff --git a/packages/aws-cdk/lib/cli.ts b/packages/aws-cdk/lib/cli.ts index 9d61a448c9a91..9a91e6257db76 100644 --- a/packages/aws-cdk/lib/cli.ts +++ b/packages/aws-cdk/lib/cli.ts @@ -176,6 +176,27 @@ async function parseCommandLineArguments(args: string[]) { .option('asset-prebuild', { type: 'boolean', desc: 'Whether to build all assets before deploying the first stack (useful for failing Docker builds)', default: true }) .option('ignore-no-stacks', { type: 'boolean', desc: 'Whether to deploy if the app contains no stacks', default: false }), ) + .command('rollback [STACKS..]', 'Rolls back the stack(s) named STACKS to their last stable state', (yargs: Argv) => yargs + .option('all', { type: 'boolean', default: false, desc: 'Roll back all available stacks' }) + .option('toolkit-stack-name', { type: 'string', desc: 'The name of the CDK toolkit stack the environment is bootstrapped with', requiresArg: true }) + .option('force', { + alias: 'f', + type: 'boolean', + desc: 'Orphan all resources for which the rollback operation fails.', + }) + .option('validate-bootstrap-version', { + type: 'boolean', + desc: 'Whether to validate the bootstrap stack version. Defaults to \'true\', disable with --no-validate-bootstrap-version.', + }) + .option('orphan', { + // alias: 'o' conflicts with --output + type: 'array', + nargs: 1, + requiresArg: true, + desc: 'Orphan the given resources, identified by their logical ID (can be specified multiple times)', + default: [], + }), + ) .command('import [STACK]', 'Import existing resource(s) into the given STACK', (yargs: Argv) => yargs .option('execute', { type: 'boolean', desc: 'Whether to execute ChangeSet (--no-execute will NOT execute the ChangeSet)', default: true }) .option('change-set-name', { type: 'string', desc: 'Name of the CloudFormation change set to create' }) @@ -596,6 +617,16 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise = jest.fn(); +let mockContinueUpdateRollback: MockedHandlerType = jest.fn(); +let mockDescribeStackEvents: MockedHandlerType = jest.fn(); beforeEach(() => { jest.resetAllMocks(); sdkProvider = new MockSdkProvider(); @@ -35,6 +38,9 @@ beforeEach(() => { StackResourceSummaries: stackResources, }; }, + rollbackStack: mockRollbackStack, + continueUpdateRollback: mockContinueUpdateRollback, + describeStackEvents: mockDescribeStackEvents, }); ToolkitInfo.lookup = mockToolkitInfoLookup = jest.fn().mockResolvedValue(ToolkitInfo.bootstrapStackNotFoundInfo('TestBootstrapStack')); @@ -331,90 +337,76 @@ test('readCurrentTemplateWithNestedStacks() can handle non-Resources in the temp }); test('readCurrentTemplateWithNestedStacks() with a 3-level nested + sibling structure works', async () => { - const cfnStack = new FakeCloudformationStack({ - stackName: 'MultiLevelRoot', - stackId: 'StackId', - }); - CloudFormationStack.lookup = (async (_, stackName: string) => { - switch (stackName) { - case 'MultiLevelRoot': - cfnStack.template = async () => ({ - Resources: { - NestedStack: { - Type: 'AWS::CloudFormation::Stack', - Properties: { - TemplateURL: 'https://www.magic-url.com', - }, - Metadata: { - 'aws:asset:path': 'one-resource-two-stacks-stack.nested.template.json', - }, + givenStacks({ + MultiLevelRoot: { + template: { + Resources: { + NestedStack: { + Type: 'AWS::CloudFormation::Stack', + Properties: { + TemplateURL: 'https://www.magic-url.com', + }, + Metadata: { + 'aws:asset:path': 'one-resource-two-stacks-stack.nested.template.json', }, }, - }); - break; - - case 'NestedStack': - cfnStack.template = async () => ({ - Resources: { - SomeResource: { - Type: 'AWS::Something', - Properties: { - Property: 'old-value', - }, + }, + }, + }, + NestedStack: { + template: { + Resources: { + SomeResource: { + Type: 'AWS::Something', + Properties: { + Property: 'old-value', }, - GrandChildStackA: { - Type: 'AWS::CloudFormation::Stack', - Properties: { - TemplateURL: 'https://www.magic-url.com', - }, - Metadata: { - 'aws:asset:path': 'one-resource-stack.nested.template.json', - }, + }, + GrandChildStackA: { + Type: 'AWS::CloudFormation::Stack', + Properties: { + TemplateURL: 'https://www.magic-url.com', }, - GrandChildStackB: { - Type: 'AWS::CloudFormation::Stack', - Properties: { - TemplateURL: 'https://www.magic-url.com', - }, - Metadata: { - 'aws:asset:path': 'one-resource-stack.nested.template.json', - }, + Metadata: { + 'aws:asset:path': 'one-resource-stack.nested.template.json', }, }, - }); - break; - - case 'GrandChildStackA': - cfnStack.template = async () => ({ - Resources: { - SomeResource: { - Type: 'AWS::Something', - Properties: { - Property: 'old-value', - }, + GrandChildStackB: { + Type: 'AWS::CloudFormation::Stack', + Properties: { + TemplateURL: 'https://www.magic-url.com', + }, + Metadata: { + 'aws:asset:path': 'one-resource-stack.nested.template.json', }, }, - }); - break; - - case 'GrandChildStackB': - cfnStack.template = async () => ({ - Resources: { - SomeResource: { - Type: 'AWS::Something', - Properties: { - Property: 'old-value', - }, + }, + }, + }, + GrandChildStackA: { + template: { + Resources: { + SomeResource: { + Type: 'AWS::Something', + Properties: { + Property: 'old-value', }, }, - }); - break; - - default: - throw new Error('unknown stack name ' + stackName + ' found in deployments.test.ts'); - } - - return cfnStack; + }, + }, + }, + GrandChildStackB: { + template: { + Resources: { + SomeResource: { + Type: 'AWS::Something', + Properties: { + Property: 'old-value', + }, + }, + }, + }, + }, }); const rootStack = testStack({ @@ -682,36 +674,31 @@ test('readCurrentTemplateWithNestedStacks() on an undeployed parent stack with a test('readCurrentTemplateWithNestedStacks() caches calls to listStackResources()', async () => { // GIVEN - const cfnStack = new FakeCloudformationStack({ - stackName: 'CachingRoot', - stackId: 'StackId', - }); - CloudFormationStack.lookup = (async (_cfn, _stackName: string) => { - cfnStack.template = async () => ({ - Resources: - { - NestedStackA: { - Type: 'AWS::CloudFormation::Stack', - Properties: { - TemplateURL: 'https://www.magic-url.com', - }, - Metadata: { - 'aws:asset:path': 'one-resource-stack.nested.template.json', - }, - }, - NestedStackB: { - Type: 'AWS::CloudFormation::Stack', - Properties: { - TemplateURL: 'https://www.magic-url.com', + givenStacks({ + '*': { + template: { + Resources: { + NestedStackA: { + Type: 'AWS::CloudFormation::Stack', + Properties: { + TemplateURL: 'https://www.magic-url.com', + }, + Metadata: { + 'aws:asset:path': 'one-resource-stack.nested.template.json', + }, }, - Metadata: { - 'aws:asset:path': 'one-resource-stack.nested.template.json', + NestedStackB: { + Type: 'AWS::CloudFormation::Stack', + Properties: { + TemplateURL: 'https://www.magic-url.com', + }, + Metadata: { + 'aws:asset:path': 'one-resource-stack.nested.template.json', + }, }, }, }, - }); - - return cfnStack; + }, }); const rootStack = testStack({ @@ -756,15 +743,112 @@ test('readCurrentTemplateWithNestedStacks() caches calls to listStackResources() expect(numberOfTimesListStackResourcesWasCalled).toEqual(1); }); -test('readCurrentTemplateWithNestedStacks() succesfully ignores stacks without metadata', async () => { +test('rollback stack assumes role if necessary', async() => { + const mockForEnvironment = jest.fn().mockImplementation(() => { return { sdk: sdkProvider.sdk }; }); + sdkProvider.forEnvironment = mockForEnvironment; + givenStacks({ + '*': { template: {} }, + }); + + await deployments.rollbackStack({ + stack: testStack({ + stackName: 'boop', + properties: { + assumeRoleArn: 'bloop:${AWS::Region}:${AWS::AccountId}', + }, + }), + validateBootstrapStackVersion: false, + }); + + expect(mockForEnvironment).toHaveBeenCalledWith(expect.anything(), expect.anything(), expect.objectContaining({ + assumeRoleArn: 'bloop:here:123456789012', + })); +}); + +test('rollback stack allows rolling back from UPDATE_FAILED', async() => { // GIVEN - const cfnStack = new FakeCloudformationStack({ - stackName: 'MetadataRoot', - stackId: 'StackId', + givenStacks({ + '*': { template: {}, stackStatus: 'UPDATE_FAILED' }, }); - CloudFormationStack.lookup = (async (_, stackName: string) => { - if (stackName === 'MetadataRoot') { - cfnStack.template = async () => ({ + + // WHEN + await deployments.rollbackStack({ + stack: testStack({ stackName: 'boop' }), + validateBootstrapStackVersion: false, + }); + + // THEN + expect(mockRollbackStack).toHaveBeenCalled(); +}); + +test('rollback stack allows continue rollback from UPDATE_ROLLBACK_FAILED', async() => { + // GIVEN + givenStacks({ + '*': { template: {}, stackStatus: 'UPDATE_ROLLBACK_FAILED' }, + }); + + // WHEN + await deployments.rollbackStack({ + stack: testStack({ stackName: 'boop' }), + validateBootstrapStackVersion: false, + }); + + // THEN + expect(mockContinueUpdateRollback).toHaveBeenCalled(); +}); + +test('rollback stack fails in UPDATE_COMPLETE state', async() => { + // GIVEN + givenStacks({ + '*': { template: {}, stackStatus: 'UPDATE_COMPLETE' }, + }); + + // WHEN + const response = await deployments.rollbackStack({ + stack: testStack({ stackName: 'boop' }), + validateBootstrapStackVersion: false, + }); + + // THEN + expect(response.notInRollbackableState).toBe(true); +}); + +test('continue rollback stack with force ignores any failed resources', async() => { + // GIVEN + givenStacks({ + '*': { template: {}, stackStatus: 'UPDATE_ROLLBACK_FAILED' }, + }); + mockDescribeStackEvents.mockReturnValue({ + StackEvents: [ + { + EventId: 'asdf', + StackId: 'stack/MyStack', + StackName: 'MyStack', + Timestamp: new Date(), + LogicalResourceId: 'Xyz', + ResourceStatus: 'UPDATE_FAILED', + }, + ], + }); + + // WHEN + await deployments.rollbackStack({ + stack: testStack({ stackName: 'boop' }), + validateBootstrapStackVersion: false, + force: true, + }); + + // THEN + expect(mockContinueUpdateRollback).toHaveBeenCalledWith(expect.objectContaining({ + ResourcesToSkip: ['Xyz'], + })); +}); + +test('readCurrentTemplateWithNestedStacks() succesfully ignores stacks without metadata', async () => { + // GIVEN + givenStacks({ + 'MetadataRoot': { + template: { Resources: { WithMetadata: { Type: 'AWS::CloudFormation::Stack', @@ -776,10 +860,10 @@ test('readCurrentTemplateWithNestedStacks() succesfully ignores stacks without m }, }, }, - }); - - } else { - cfnStack.template = async () => ({ + }, + }, + '*': { + template: { Resources: { SomeResource: { Type: 'AWS::Something', @@ -788,10 +872,8 @@ test('readCurrentTemplateWithNestedStacks() succesfully ignores stacks without m }, }, }, - }); - } - - return cfnStack; + }, + }, }); const rootStack = testStack({ @@ -918,3 +1000,23 @@ function stackSummaryOf(logicalId: string, resourceType: string, physicalResourc LastUpdatedTimestamp: new Date(), }; } + +function givenStacks(stacks: Record) { + jest.spyOn(CloudFormationStack, 'lookup').mockImplementation(async (_, stackName) => { + let stack = stacks[stackName]; + if (!stack) { + stack = stacks['*']; + } + if (stack) { + const cfnStack = new FakeCloudformationStack({ + stackName, + stackId: `stack/${stackName}`, + stackStatus: stack.stackStatus, + }); + cfnStack.setTemplate(stack.template); + return cfnStack; + } else { + return new FakeCloudformationStack({ stackName }); + } + }); +} \ No newline at end of file diff --git a/packages/aws-cdk/test/api/fake-cloudformation-stack.ts b/packages/aws-cdk/test/api/fake-cloudformation-stack.ts index 1668ea0b55d33..918d6c4d5bb37 100644 --- a/packages/aws-cdk/test/api/fake-cloudformation-stack.ts +++ b/packages/aws-cdk/test/api/fake-cloudformation-stack.ts @@ -2,10 +2,12 @@ import { CloudFormation } from 'aws-sdk'; import { CloudFormationStack, Template } from '../../lib/api/util/cloudformation'; import { instanceMockFrom } from '../util'; +import { StackStatus } from '../../lib/api/util/cloudformation/stack-status'; export interface FakeCloudFormationStackProps { readonly stackName: string; - readonly stackId: string; + readonly stackId?: string; + readonly stackStatus?: string; } export class FakeCloudformationStack extends CloudFormationStack { @@ -29,7 +31,23 @@ export class FakeCloudformationStack extends CloudFormationStack { return Promise.resolve(this.__template); } - public get stackId(): string { + public get exists() { + return this.props.stackId !== undefined; + } + + public get stackStatus() { + const status = this.props.stackStatus ?? 'UPDATE_COMPLETE'; + return new StackStatus(status, 'The test said so'); + } + + public get stackId() { + if (!this.props.stackId) { + throw new Error('Cannot retrieve stackId from a non-existent stack'); + } return this.props.stackId; } + + public get outputs(): Record { + return {}; + } } diff --git a/packages/aws-cdk/test/api/stack-activity-monitor.test.ts b/packages/aws-cdk/test/api/stack-activity-monitor.test.ts index 6c5eddc7dd75e..a07ec99e40b38 100644 --- a/packages/aws-cdk/test/api/stack-activity-monitor.test.ts +++ b/packages/aws-cdk/test/api/stack-activity-monitor.test.ts @@ -29,6 +29,7 @@ test('prints 0/4 progress report, when addActivity is called with an "IN_PROGRES EventId: '', StackName: 'stack-name', }, + parentStackLogicalIds: [], }); }); @@ -53,6 +54,7 @@ test('prints 1/4 progress report, when addActivity is called with an "UPDATE_COM EventId: '', StackName: 'stack-name', }, + parentStackLogicalIds: [], }); }); @@ -77,6 +79,7 @@ test('prints 1/4 progress report, when addActivity is called with an "UPDATE_COM EventId: '', StackName: 'stack-name', }, + parentStackLogicalIds: [], }); }); @@ -101,6 +104,7 @@ test('prints 1/4 progress report, when addActivity is called with an "ROLLBACK_C EventId: '', StackName: 'stack-name', }, + parentStackLogicalIds: [], }); }); @@ -125,6 +129,7 @@ test('prints 0/4 progress report, when addActivity is called with an "UPDATE_FAI EventId: '', StackName: 'stack-name', }, + parentStackLogicalIds: [], }); }); @@ -149,6 +154,7 @@ test('does not print "Failed Resources:" list, when all deployments are successf EventId: '', StackName: 'stack-name', }, + parentStackLogicalIds: [], }); historyActivityPrinter.addActivity({ event: { @@ -160,6 +166,7 @@ test('does not print "Failed Resources:" list, when all deployments are successf EventId: '', StackName: 'stack-name', }, + parentStackLogicalIds: [], }); historyActivityPrinter.addActivity({ event: { @@ -171,6 +178,7 @@ test('does not print "Failed Resources:" list, when all deployments are successf EventId: '', StackName: 'stack-name', }, + parentStackLogicalIds: [], }); historyActivityPrinter.stop(); }); @@ -199,6 +207,7 @@ test('prints "Failed Resources:" list, when at least one deployment fails', () = EventId: '', StackName: 'stack-name', }, + parentStackLogicalIds: [], }); historyActivityPrinter.addActivity({ event: { @@ -210,6 +219,7 @@ test('prints "Failed Resources:" list, when at least one deployment fails', () = EventId: '', StackName: 'stack-name', }, + parentStackLogicalIds: [], }); historyActivityPrinter.stop(); }); @@ -242,6 +252,7 @@ test('print failed resources because of hook failures', () => { HookType: 'hook1', HookStatusReason: 'stack1 must obey certain rules', }, + parentStackLogicalIds: [], }); historyActivityPrinter.addActivity({ event: { @@ -254,6 +265,7 @@ test('print failed resources because of hook failures', () => { StackName: 'stack-name', ResourceStatusReason: 'The following hook(s) failed: hook1', }, + parentStackLogicalIds: [], }); historyActivityPrinter.stop(); }); diff --git a/packages/aws-cdk/test/cdk-toolkit.test.ts b/packages/aws-cdk/test/cdk-toolkit.test.ts index f67c35ad8dae7..d2c46cc15698d 100644 --- a/packages/aws-cdk/test/cdk-toolkit.test.ts +++ b/packages/aws-cdk/test/cdk-toolkit.test.ts @@ -64,7 +64,7 @@ import { instanceMockFrom, MockCloudExecutable, TestStackArtifact } from './util import { MockSdkProvider } from './util/mock-sdk'; import { Bootstrapper } from '../lib/api/bootstrap'; import { DeployStackResult } from '../lib/api/deploy-stack'; -import { Deployments, DeployStackOptions, DestroyStackOptions } from '../lib/api/deployments'; +import { Deployments, DeployStackOptions, DestroyStackOptions, RollbackStackOptions, RollbackStackResult } from '../lib/api/deployments'; import { HotswapMode } from '../lib/api/hotswap/common'; import { Template } from '../lib/api/util/cloudformation'; import { CdkToolkit, Tag } from '../lib/cdk-toolkit'; @@ -1224,6 +1224,31 @@ describe('synth', () => { expect(mockData.mock.calls.length).toEqual(1); expect(mockData.mock.calls[0][0]).toBeDefined(); }); + + test('rollback uses deployment role', async () => { + cloudExecutable = new MockCloudExecutable({ + stacks: [ + MockStack.MOCK_STACK_C, + ], + }); + + const mockedRollback = jest.spyOn(Deployments.prototype, 'rollbackStack').mockResolvedValue({ + success: true, + }); + + const toolkit = new CdkToolkit({ + cloudExecutable, + configuration: cloudExecutable.configuration, + sdkProvider: cloudExecutable.sdkProvider, + deployments: new Deployments({ sdkProvider: new MockSdkProvider() }), + }); + + await toolkit.rollback({ + selector: { patterns: [] }, + }); + + expect(mockedRollback).toHaveBeenCalled(); + }); }); class MockStack { @@ -1400,6 +1425,12 @@ class FakeCloudFormation extends Deployments { }); } + public rollbackStack(_options: RollbackStackOptions): Promise { + return Promise.resolve({ + success: true, + }); + } + public destroyStack(options: DestroyStackOptions): Promise { expect(options.stack).toBeDefined(); return Promise.resolve(); diff --git a/packages/aws-cdk/test/util/mock-sdk.ts b/packages/aws-cdk/test/util/mock-sdk.ts index 0d943fadb3dea..d280ef9f02942 100644 --- a/packages/aws-cdk/test/util/mock-sdk.ts +++ b/packages/aws-cdk/test/util/mock-sdk.ts @@ -269,11 +269,21 @@ type AwsCallInputOutput = // Determine the type of the mock handler from the type of the Input/Output type pair. // Don't need to worry about the 'never', TypeScript will propagate it upwards making it // impossible to specify the field that has 'never' anywhere in its type. -type MockHandlerType = +type HandlerType = AI extends [any, any] ? (input: AI[0]) => AI[1] : AI; // Any subset of the full type that synchronously returns the output structure is okay -export type SyncHandlerSubsetOf = {[K in keyof S]?: MockHandlerType>}; +export type SyncHandlerSubsetOf = {[K in keyof S]?: HandlerType>}; + +/** + * A jest Mock function we can pass into SdkProvider.stubXXX + * + * Use as follows: + * + * ```ts + * const mockDescribeStackEvents: MockedHandlerType = jest.fn(); + */ +export type MockedHandlerType = AwsCallInputOutput extends [infer IN, infer OUT] ? jest.Mock : never; /** * Fake AWS response. diff --git a/packages/aws-cdk/test/util/stack-monitor.test.ts b/packages/aws-cdk/test/util/stack-monitor.test.ts index e32098a03a9ee..e0d9bb673fbd7 100644 --- a/packages/aws-cdk/test/util/stack-monitor.test.ts +++ b/packages/aws-cdk/test/util/stack-monitor.test.ts @@ -124,11 +124,20 @@ describe('stack monitor, collecting errors from events', () => { return { StackEvents: [ addErrorToStackEvent( - event(100), { + event(102), { logicalResourceId: 'nestedStackLogicalResourceId', physicalResourceId: 'nestedStackPhysicalResourceId', resourceType: 'AWS::CloudFormation::Stack', resourceStatusReason: 'nested stack failed', + resourceStatus: 'UPDATE_FAILED', + }, + ), + addErrorToStackEvent( + event(100), { + logicalResourceId: 'nestedStackLogicalResourceId', + physicalResourceId: 'nestedStackPhysicalResourceId', + resourceType: 'AWS::CloudFormation::Stack', + resourceStatus: 'UPDATE_IN_PROGRESS', }, ), ], @@ -253,18 +262,28 @@ async function testMonitorWithEventCalls( let describeStackEvents = (jest.fn() as jest.Mock); let finished = false; + let error: Error | undefined = undefined; for (const invocation of beforeStopInvocations) { const invocation_ = invocation; // Capture loop variable in local because of closure semantics const isLast = invocation === beforeStopInvocations[beforeStopInvocations.length - 1]; describeStackEvents = describeStackEvents.mockImplementationOnce(request => { - const ret = invocation_(request); - if (isLast) { + try { + const ret = invocation_(request); + if (isLast) { + finished = true; + } + return ret; + } catch (e: any) { finished = true; + error = e; + throw e; } - return ret; }); } + if (error) { + throw error; + } for (const invocation of afterStopInvocations) { describeStackEvents = describeStackEvents.mockImplementationOnce(invocation); }