Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(cli): skip bundling for operations where stack is not needed #9889

Merged
merged 25 commits into from
Sep 21, 2020
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
2e6f65f
feat(cli): skip bundling for operations where stack is not needed
jogold Aug 21, 2020
6140dab
move custom default and test it
jogold Aug 21, 2020
cb40202
add tests in core
jogold Aug 21, 2020
f51f73a
README
jogold Aug 21, 2020
86c35c4
tests
jogold Aug 23, 2020
41c480e
Merge branch 'master' into skip-bundling
jogold Aug 24, 2020
ac2cc2d
Merge branch 'master' into skip-bundling
jogold Aug 26, 2020
cebbdb1
revert change in tag-aspect
jogold Aug 26, 2020
cf2f0e7
correct default
jogold Aug 26, 2020
f29fe68
Merge branch 'skip-bundling' of github.com:jogold/aws-cdk into skip-b…
jogold Aug 26, 2020
b9d3809
Merge branch 'master' into skip-bundling
jogold Aug 27, 2020
dc923ef
Merge branch 'master' into skip-bundling
jogold Aug 28, 2020
88efc5d
Merge branch 'master' into skip-bundling
jogold Sep 2, 2020
e00548b
Merge branch 'master' into skip-bundling
jogold Sep 2, 2020
5e44f79
remove CLI option
jogold Sep 2, 2020
bd704f4
dummy hash in case of BUNDLE
jogold Sep 2, 2020
9849358
revert changes in bin/cdk.ts
jogold Sep 2, 2020
23c5b60
Merge branch 'master' into skip-bundling
jogold Sep 7, 2020
8217cff
Merge branch 'master' into skip-bundling
jogold Sep 8, 2020
cb926b4
Merge branch 'master' into skip-bundling
jogold Sep 11, 2020
266e40b
Merge branch 'master' into skip-bundling
jogold Sep 16, 2020
6132e72
improve typing
jogold Sep 16, 2020
749f02b
Merge branch 'skip-bundling' of github.com:jogold/aws-cdk into skip-b…
jogold Sep 16, 2020
4e1457f
Merge branch 'master' into skip-bundling
jogold Sep 16, 2020
44b401b
Merge branch 'master' into skip-bundling
mergify[bot] Sep 21, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 19 additions & 10 deletions packages/@aws-cdk/core/lib/asset-staging.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { AssetHashType, AssetOptions } from './assets';
import { BundlingOptions } from './bundling';
import { Construct } from './construct-compat';
import { FileSystem, FingerprintOptions } from './fs';
import { Stack } from './stack';
import { Stage } from './stage';

/**
Expand Down Expand Up @@ -97,16 +98,24 @@ export class AssetStaging extends Construct {
const hashType = determineHashType(props.assetHashType, props.assetHash);

if (props.bundling) {
// Determine the source hash in advance of bundling if the asset hash type
// is SOURCE so that the bundler can opt to re-use its previous output.
const sourceHash = hashType === AssetHashType.SOURCE
? this.calculateHash(hashType, props.assetHash, props.bundling)
: undefined;

this.bundleDir = this.bundle(props.bundling, outdir, sourceHash);
this.assetHash = sourceHash ?? this.calculateHash(hashType, props.assetHash, props.bundling);
this.relativePath = renderAssetFilename(this.assetHash);
this.stagedPath = this.relativePath;
// Check if we actually have to bundle for this stack
const bundlingStacks: string[] = this.node.tryGetContext(cxapi.BUNDLING_STACKS) ?? ['*'];
const runBundling = bundlingStacks.includes(Stack.of(this).stackName) || bundlingStacks.includes('*');
if (runBundling) {
// Determine the source hash in advance of bundling if the asset hash type
// is SOURCE so that the bundler can opt to re-use its previous output.
const sourceHash = hashType === AssetHashType.SOURCE
? this.calculateHash(hashType, props.assetHash, props.bundling)
: undefined;

this.bundleDir = this.bundle(props.bundling, outdir, sourceHash);
this.assetHash = sourceHash ?? this.calculateHash(hashType, props.assetHash, props.bundling);
this.relativePath = renderAssetFilename(this.assetHash);
this.stagedPath = this.relativePath;
} else { // Bundling is skipped
this.assetHash = this.calculateHash(AssetHashType.CUSTOM, this.node.path); // Use node path as dummy hash
this.stagedPath = this.sourcePath;
}
} else {
this.assetHash = this.calculateHash(hashType, props.assetHash);

Expand Down
24 changes: 24 additions & 0 deletions packages/@aws-cdk/core/test/test.staging.ts
Original file line number Diff line number Diff line change
Expand Up @@ -580,6 +580,30 @@ export = {

test.done();
},

'bundling looks at bundling stacks in context'(test: Test) {
// GIVEN
const app = new App();
const stack = new Stack(app, 'MyStack');
stack.node.setContext(cxapi.BUNDLING_STACKS, ['OtherStack']);
const directory = path.join(__dirname, 'fs', 'fixtures', 'test1');

// WHEN
const asset = new AssetStaging(stack, 'Asset', {
sourcePath: directory,
bundling: {
image: BundlingDockerImage.fromRegistry('alpine'),
command: [DockerStubCommand.SUCCESS],
},
});

test.throws(() => readDockerStubInput());
test.equal(asset.assetHash, '3d96e735e26b857743a7c44523c9160c285c2d3ccf273d80fa38a1e674c32cb3'); // hash of MyStack/Asset
test.equal(asset.sourcePath, directory);
test.equal(stack.resolve(asset.stagedPath), directory);

test.done();
},
};

// Reads a docker stub and cleans the volume paths out of the stub.
Expand Down
5 changes: 5 additions & 0 deletions packages/@aws-cdk/cx-api/lib/app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,8 @@ export const DISABLE_ASSET_STAGING_CONTEXT = 'aws:cdk:disable-asset-staging';
* Omits stack traces from construct metadata entries.
*/
export const DISABLE_METADATA_STACK_TRACE = 'aws:cdk:disable-stack-trace';

/**
* Run bundling for stacks specified in this context key
*/
export const BUNDLING_STACKS = 'aws:cdk:bundling-stacks';
5 changes: 5 additions & 0 deletions packages/aws-cdk/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,11 @@ $ cdk doctor
- AWS_SDK_LOAD_CONFIG = 1
```

#### Bundling
By default asset bundling is skipped for `cdk list` and `cdk destroy`. For `cdk deploy`, `cdk diff`
and `cdk synthesize` the default is to bundle assets for all stacks unless `exclusively` is specified.
In this case, only the listed stacks will have their assets bundled.

### MFA support

If `mfa_serial` is found in the active profile of the shared ini file AWS CDK
Expand Down
18 changes: 13 additions & 5 deletions packages/aws-cdk/bin/cdk.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,13 @@ async function parseCommandLineArguments() {
.option('output', { type: 'string', alias: 'o', desc: 'Emits the synthesized cloud assembly into a directory (default: cdk.out)', requiresArg: true })
.option('no-color', { type: 'boolean', desc: 'Removes colors and other style from console output', default: false })
.command(['list [STACKS..]', 'ls [STACKS..]'], 'Lists all stacks in the app', yargs => yargs
.option('long', { type: 'boolean', default: false, alias: 'l', desc: 'Display environment information for each stack' }),
.option('long', { type: 'boolean', default: false, alias: 'l', desc: 'Display environment information for each stack' })
.option('bundling', { type: 'array', default: [], alias: 'b', desc: 'Run bundling only for given stacks (defaults to no stacks)' }),
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we even need a CLI option for this after all? I don't see why someone would override the defaults here...

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, seems confusing. my initial idea was to support this as a global feature with a "smart default" so that users will be able to override this behavior if they want.

)
.command(['synthesize [STACKS..]', 'synth [STACKS..]'], 'Synthesizes and prints the CloudFormation template for this stack', yargs => yargs
.option('exclusively', { type: 'boolean', alias: 'e', desc: 'Only synthesize requested stacks, don\'t include dependencies' }))
.option('exclusively', { type: 'boolean', alias: 'e', desc: 'Only synthesize requested stacks, don\'t include dependencies' })
.option('bundling', { type: 'array', alias: 'b', desc: 'Run bundling only for given stacks (defaults to STACKS if `exclusively` is used, all stacks otherwise)' }),
)
.command('bootstrap [ENVIRONMENTS..]', 'Deploys the CDK toolkit stack into an AWS environment', yargs => yargs
.option('bootstrap-bucket-name', { type: 'string', alias: ['b', 'toolkit-bucket-name'], desc: 'The name of the CDK toolkit bucket; bucket will be created and must not exist', default: undefined })
.option('bootstrap-kms-key-id', { type: 'string', desc: 'AWS KMS master key ID used for the SSE-KMS encryption', default: undefined })
Expand All @@ -93,16 +96,21 @@ async function parseCommandLineArguments() {
.option('parameters', { type: 'array', desc: 'Additional parameters passed to CloudFormation at deploy time (STACK:KEY=VALUE)', nargs: 1, requiresArg: true, default: {} })
.option('outputs-file', { type: 'string', alias: 'O', desc: 'Path to file where stack outputs will be written as JSON', requiresArg: true })
.option('previous-parameters', { type: 'boolean', default: true, desc: 'Use previous values for existing parameters (you must specify all parameters on every deployment if this is disabled)' })
.option('progress', { type: 'string', choices: [StackActivityProgress.BAR, StackActivityProgress.EVENTS], desc: 'Display mode for stack activity events.' }),
.option('progress', { type: 'string', choices: [StackActivityProgress.BAR, StackActivityProgress.EVENTS], desc: 'Display mode for stack activity events.' })
.option('bundling', { type: 'array', alias: 'b', desc: 'Run bundling only for given stacks (defaults to STACKS if `exclusively` is used, all stacks otherwise)' }),
)
.command('destroy [STACKS..]', 'Destroy the stack(s) named STACKS', yargs => yargs
.option('exclusively', { type: 'boolean', alias: 'e', desc: 'Only destroy requested stacks, don\'t include dependees' })
.option('force', { type: 'boolean', alias: 'f', desc: 'Do not ask for confirmation before destroying the stacks' }))
.option('force', { type: 'boolean', alias: 'f', desc: 'Do not ask for confirmation before destroying the stacks' })
.option('bundling', { type: 'array', default: [], alias: 'b', desc: 'Run bundling only for given stacks (defaults to no stacks)' }),
)
.command('diff [STACKS..]', 'Compares the specified stack with the deployed stack or a local template file, and returns with status 1 if any difference is found', yargs => yargs
.option('exclusively', { type: 'boolean', alias: 'e', desc: 'Only diff requested stacks, don\'t include dependencies' })
.option('context-lines', { type: 'number', desc: 'Number of context lines to include in arbitrary JSON diff rendering', default: 3, requiresArg: true })
.option('template', { type: 'string', desc: 'The path to the CloudFormation template to compare with', requiresArg: true })
.option('strict', { type: 'boolean', desc: 'Do not filter out AWS::CDK::Metadata resources', default: false }))
.option('strict', { type: 'boolean', desc: 'Do not filter out AWS::CDK::Metadata resources', default: false })
.option('bundling', { type: 'array', alias: 'b', desc: 'Run bundling only for given stacks (defaults to STACKS if `exclusively` is used, all stacks otherwise)' }),
)
.option('fail', { type: 'boolean', desc: 'Fail with exit code 1 in case of diff', default: false })
.command('metadata [STACK]', 'Returns all metadata associated with this stack')
.command('init [TEMPLATE]', 'Create a new, empty CDK project from a template. Invoked without TEMPLATE, the app template will be used.', yargs => yargs
Expand Down
3 changes: 3 additions & 0 deletions packages/aws-cdk/lib/api/cxapp/exec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ export async function execProgram(aws: SdkProvider, config: Configuration): Prom
context[cxapi.DISABLE_ASSET_STAGING_CONTEXT] = true;
}

const bundlingStacks = config.settings.get(['bundling']) ?? ['*'];
context[cxapi.BUNDLING_STACKS] = bundlingStacks;

debug('context:', context);
env[cxapi.CONTEXT_ENV] = JSON.stringify(context);

Expand Down
16 changes: 15 additions & 1 deletion packages/aws-cdk/lib/settings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@ export const TRANSIENT_CONTEXT_KEY = '$dontSaveContext';

const CONTEXT_KEY = 'context';

export type Arguments = { readonly [name: string]: unknown };
const CUSTOM_BUNDLING_DEFAULT_COMMANDS = ['deploy', 'diff', 'synth', 'synthesize'];

export type Arguments = { readonly [name: string]: unknown, readonly _: string[] };

/**
* All sources of settings combined
Expand Down Expand Up @@ -185,6 +187,17 @@ export class Settings {
const context = this.parseStringContextListToObject(argv);
const tags = this.parseStringTagsListToObject(argv);

// Custom `bundling` default. If we deploy, diff or synth a list of stacks
// exclusively we skip bundling for all other stacks.
let bundling = argv.bundling;
if (!bundling) {
if (CUSTOM_BUNDLING_DEFAULT_COMMANDS.includes(argv._[0]) && argv.exclusively) {
bundling = argv.STACKS ?? [];
} else {
bundling = ['*'];
}
}

return new Settings({
app: argv.app,
browser: argv.browser,
Expand All @@ -205,6 +218,7 @@ export class Settings {
staging: argv.staging,
output: argv.output,
progress: argv.progress,
bundling,
});
}

Expand Down
2 changes: 1 addition & 1 deletion packages/aws-cdk/test/context.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ test('surive no context in old file', async () => {
test('command line context is merged with stored context', async () => {
// GIVEN
await fs.writeJSON('cdk.context.json', { boo: 'far' });
const config = await new Configuration({ context: ['foo=bar'] } as any).load();
const config = await new Configuration({ context: ['foo=bar'], _: ['command'] } as any).load();

// WHEN
expect(config.context.all).toEqual({ foo: 'bar', boo: 'far' });
Expand Down
43 changes: 39 additions & 4 deletions packages/aws-cdk/test/settings.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,8 @@ test('can clear all values in all objects', () => {

test('can parse string context from command line arguments', () => {
// GIVEN
const settings1 = Settings.fromCommandLineArguments({ context: ['foo=bar'] });
const settings2 = Settings.fromCommandLineArguments({ context: ['foo='] });
const settings1 = Settings.fromCommandLineArguments({ context: ['foo=bar'], _: ['command'] });
const settings2 = Settings.fromCommandLineArguments({ context: ['foo='], _: ['command'] });

// THEN
expect(settings1.get(['context']).foo).toEqual( 'bar');
Expand All @@ -72,10 +72,45 @@ test('can parse string context from command line arguments', () => {

test('can parse string context from command line arguments with equals sign in value', () => {
// GIVEN
const settings1 = Settings.fromCommandLineArguments({ context: ['foo==bar='] });
const settings2 = Settings.fromCommandLineArguments({ context: ['foo=bar='] });
const settings1 = Settings.fromCommandLineArguments({ context: ['foo==bar='], _: ['command'] });
const settings2 = Settings.fromCommandLineArguments({ context: ['foo=bar='], _: ['command'] });

// THEN
expect(settings1.get(['context']).foo).toEqual( '=bar=');
expect(settings2.get(['context']).foo).toEqual( 'bar=');
});

test('custom default for bundling in case of deploy', () => {
// GIVEN
const settings = Settings.fromCommandLineArguments({
_: ['deploy'],
exclusively: true,
STACKS: ['cool-stack'],
});

// THEN
expect(settings.get(['bundling'])).toEqual(['cool-stack']);
});

test('bundling defaults to *', () => {
// GIVEN
const settings = Settings.fromCommandLineArguments({
_: ['command'],
});

// THEN
expect(settings.get(['bundling'])).toEqual(['*']);
});

test('deploy with bundling specified', () => {
// GIVEN
const settings = Settings.fromCommandLineArguments({
_: ['deploy'],
exclusively: true,
STACKS: ['cool-stack'],
bundling: ['other-stack', 'cool-stack'],
});

// THEN
expect(settings.get(['bundling'])).toEqual(['other-stack', 'cool-stack']);
});