diff --git a/packages/@aws-cdk/aws-s3/README.md b/packages/@aws-cdk/aws-s3/README.md index 84e176f62a6a5..a81b67e2b7bb4 100644 --- a/packages/@aws-cdk/aws-s3/README.md +++ b/packages/@aws-cdk/aws-s3/README.md @@ -349,7 +349,7 @@ bucket.virtualHostedUrlForObject('objectname', { regional: false }); // Virtual ### Object Ownership -You can use the two following properties to specify the bucket [object Ownership]. +You can use the two following properties to specify the bucket [object Ownership]. [object Ownership]: https://docs.aws.amazon.com/AmazonS3/latest/dev/about-object-ownership.html @@ -365,10 +365,28 @@ new s3.Bucket(this, 'MyBucket', { #### Bucket owner preferred -The bucket owner will own the object if the object is uploaded with the bucket-owner-full-control canned ACL. Without this setting and canned ACL, the object is uploaded and remains owned by the uploading account. +The bucket owner will own the object if the object is uploaded with the bucket-owner-full-control canned ACL. Without this setting and canned ACL, the object is uploaded and remains owned by the uploading account. ```ts new s3.Bucket(this, 'MyBucket', { objectOwnership: s3.ObjectOwnership.BUCKET_OWNER_PREFERRED, }); ``` + +### Bucket deletion + +When a bucket is removed from a stack (or the stack is deleted), the S3 +bucket will be removed according to its removal policy (which by default will +simply orphan the bucket and leave it in your AWS account). If the removal +policy is set to `RemovalPolicy.DESTROY`, the bucket will be deleted as long +as it does not contain any objects. + +To override this and force all objects to get deleted during bucket deletion, +enable the`autoDeleteObjects` option. + +```ts +const bucket = new Bucket(this, 'MyTempFileBucket', { + removalPolicy: RemovalPolicy.DESTROY, + autoDeleteObjects: true, +}); +``` diff --git a/packages/@aws-cdk/aws-s3/lib/auto-delete-objects-handler/index.ts b/packages/@aws-cdk/aws-s3/lib/auto-delete-objects-handler/index.ts new file mode 100644 index 0000000000000..d3a2e0e616eb2 --- /dev/null +++ b/packages/@aws-cdk/aws-s3/lib/auto-delete-objects-handler/index.ts @@ -0,0 +1,42 @@ +// eslint-disable-next-line import/no-extraneous-dependencies +import * as AWS from 'aws-sdk'; + +const s3 = new AWS.S3(); + +export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent) { + switch (event.RequestType) { + case 'Create': + case 'Update': + return; + case 'Delete': + return onDelete(event); + } +} + +/** + * Recursively delete all items in the bucket + * + * @param bucketName the bucket name + */ +async function emptyBucket(bucketName: string) { + const listedObjects = await s3.listObjectVersions({ Bucket: bucketName }).promise(); + const contents = [...listedObjects.Versions ?? [], ...listedObjects.DeleteMarkers ?? []]; + if (contents.length === 0) { + return; + }; + + const records = contents.map((record: any) => ({ Key: record.Key, VersionId: record.VersionId })); + await s3.deleteObjects({ Bucket: bucketName, Delete: { Objects: records } }).promise(); + + if (listedObjects?.IsTruncated) { + await emptyBucket(bucketName); + } +} + +async function onDelete(deleteEvent: AWSLambda.CloudFormationCustomResourceDeleteEvent) { + const bucketName = deleteEvent.ResourceProperties?.BucketName; + if (!bucketName) { + throw new Error('No BucketName was provided.'); + } + await emptyBucket(bucketName); +} diff --git a/packages/@aws-cdk/aws-s3/lib/bucket.ts b/packages/@aws-cdk/aws-s3/lib/bucket.ts index 30a6acf40d13c..f449ed43c93d7 100644 --- a/packages/@aws-cdk/aws-s3/lib/bucket.ts +++ b/packages/@aws-cdk/aws-s3/lib/bucket.ts @@ -1,17 +1,23 @@ import { EOL } from 'os'; +import * as path from 'path'; import * as events from '@aws-cdk/aws-events'; import * as iam from '@aws-cdk/aws-iam'; import * as kms from '@aws-cdk/aws-kms'; -import { Fn, IResource, Lazy, RemovalPolicy, Resource, Stack, Token } from '@aws-cdk/core'; +import { + Fn, IResource, Lazy, RemovalPolicy, Resource, Stack, Token, + CustomResource, CustomResourceProvider, CustomResourceProviderRuntime, +} from '@aws-cdk/core'; import { Construct } from 'constructs'; import { BucketPolicy } from './bucket-policy'; import { IBucketNotificationDestination } from './destination'; import { BucketNotifications } from './notifications-resource'; import * as perms from './perms'; import { LifecycleRule } from './rule'; -import { CfnBucket } from './s3.generated'; +import { CfnBucket, CfnBucketPolicy } from './s3.generated'; import { parseBucketArn, parseBucketName } from './util'; +const AUTO_DELETE_OBJECTS_RESOURCE_TYPE = 'Custom::S3AutoDeleteObjects'; + export interface IBucket extends IResource { /** * The ARN of the bucket. @@ -1041,6 +1047,16 @@ export interface BucketProps { */ readonly removalPolicy?: RemovalPolicy; + /** + * Whether all objects should be automatically deleted when the bucket is + * removed from the stack or when the stack is deleted. + * + * Requires the `removalPolicy` to be set to `RemovalPolicy.DESTROY`. + * + * @default false + */ + readonly autoDeleteObjects?: boolean; + /** * Whether this bucket should have versioning turned on or not. * @@ -1326,6 +1342,14 @@ export class Bucket extends BucketBase { if (props.publicReadAccess) { this.grantPublicAccess(); } + + if (props.autoDeleteObjects) { + if (props.removalPolicy !== RemovalPolicy.DESTROY) { + throw new Error('Cannot use \'autoDeleteObjects\' property on a bucket without setting removal policy to \'DESTROY\'.'); + } + + this.enableAutoDeleteObjects(); + } } /** @@ -1728,6 +1752,40 @@ export class Bucket extends BucketBase { }; }); } + + private enableAutoDeleteObjects() { + const provider = CustomResourceProvider.getOrCreateProvider(this, AUTO_DELETE_OBJECTS_RESOURCE_TYPE, { + codeDirectory: path.join(__dirname, 'auto-delete-objects-handler'), + runtime: CustomResourceProviderRuntime.NODEJS_12, + }); + + // Use a bucket policy to allow the custom resource to delete + // objects in the bucket + this.addToResourcePolicy(new iam.PolicyStatement({ + actions: [ + ...perms.BUCKET_READ_ACTIONS, // list objects + ...perms.BUCKET_DELETE_ACTIONS, // and then delete them + ], + resources: [ + this.bucketArn, + this.arnForObjects('*'), + ], + principals: [new iam.ArnPrincipal(provider.roleArn)], + })); + + const customResource = new CustomResource(this, 'AutoDeleteObjectsCustomResource', { + resourceType: AUTO_DELETE_OBJECTS_RESOURCE_TYPE, + serviceToken: provider.serviceToken, + properties: { + BucketName: this.bucketName, + }, + }); + + // Ensure bucket policy is deleted AFTER the custom resource otherwise + // we don't have permissions to list and delete in the bucket. + const policy = this.node.tryFindChild('Policy') as CfnBucketPolicy; + customResource.node.addDependency(policy); + } } /** diff --git a/packages/@aws-cdk/aws-s3/test/auto-delete-objects-handler.test.ts b/packages/@aws-cdk/aws-s3/test/auto-delete-objects-handler.test.ts new file mode 100644 index 0000000000000..e5a7072441974 --- /dev/null +++ b/packages/@aws-cdk/aws-s3/test/auto-delete-objects-handler.test.ts @@ -0,0 +1,168 @@ +const mockS3Client = { + listObjectVersions: jest.fn().mockReturnThis(), + deleteObjects: jest.fn().mockReturnThis(), + promise: jest.fn(), +}; + +import { handler } from '../lib/auto-delete-objects-handler'; + +jest.mock('aws-sdk', () => { + return { S3: jest.fn(() => mockS3Client) }; +}); + +beforeEach(() => { + mockS3Client.listObjectVersions.mockReturnThis(); + mockS3Client.deleteObjects.mockReturnThis(); +}); + +afterEach(() => { + jest.resetAllMocks(); +}); + +test('does nothing on create event', async () => { + // GIVEN + const event: Partial = { + RequestType: 'Create', + ResourceProperties: { + ServiceToken: 'Foo', + BucketName: 'MyBucket', + }, + }; + + // WHEN + await invokeHandler(event); + + // THEN + expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(0); + expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(0); +}); + +test('does nothing on update event', async () => { + // GIVEN + const event: Partial = { + RequestType: 'Update', + ResourceProperties: { + ServiceToken: 'Foo', + BucketName: 'MyBucket', + }, + }; + + // WHEN + await invokeHandler(event); + + // THEN + expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(0); + expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(0); +}); + +test('deletes no objects on delete event when bucket has no objects', async () => { + // GIVEN + mockS3Client.promise.mockResolvedValue({ Versions: [] }); // listObjectVersions() call + + // WHEN + const event: Partial = { + RequestType: 'Delete', + ResourceProperties: { + ServiceToken: 'Foo', + BucketName: 'MyBucket', + }, + }; + await invokeHandler(event); + + // THEN + expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(1); + expect(mockS3Client.listObjectVersions).toHaveBeenCalledWith({ Bucket: 'MyBucket' }); + expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(0); +}); + +test('deletes all objects on delete event', async () => { + // GIVEN + mockS3Client.promise.mockResolvedValue({ // listObjectVersions() call + Versions: [ + { Key: 'Key1', VersionId: 'VersionId1' }, + { Key: 'Key2', VersionId: 'VersionId2' }, + ], + }); + + // WHEN + const event: Partial = { + RequestType: 'Delete', + ResourceProperties: { + ServiceToken: 'Foo', + BucketName: 'MyBucket', + }, + }; + await invokeHandler(event); + + // THEN + expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(1); + expect(mockS3Client.listObjectVersions).toHaveBeenCalledWith({ Bucket: 'MyBucket' }); + expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(1); + expect(mockS3Client.deleteObjects).toHaveBeenCalledWith({ + Bucket: 'MyBucket', + Delete: { + Objects: [ + { Key: 'Key1', VersionId: 'VersionId1' }, + { Key: 'Key2', VersionId: 'VersionId2' }, + ], + }, + }); +}); + +test('delete event where bucket has many objects does recurse appropriately', async () => { + // GIVEN + mockS3Client.promise // listObjectVersions() call + .mockResolvedValueOnce({ + Versions: [ + { Key: 'Key1', VersionId: 'VersionId1' }, + { Key: 'Key2', VersionId: 'VersionId2' }, + ], + IsTruncated: true, + }) + .mockResolvedValueOnce(undefined) // deleteObjects() call + .mockResolvedValueOnce({ // listObjectVersions() call + Versions: [ + { Key: 'Key3', VersionId: 'VersionId3' }, + { Key: 'Key4', VersionId: 'VersionId4' }, + ], + }); + + // WHEN + const event: Partial = { + RequestType: 'Delete', + ResourceProperties: { + ServiceToken: 'Foo', + BucketName: 'MyBucket', + }, + }; + await invokeHandler(event); + + // THEN + expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(2); + expect(mockS3Client.listObjectVersions).toHaveBeenCalledWith({ Bucket: 'MyBucket' }); + expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(2); + expect(mockS3Client.deleteObjects).toHaveBeenNthCalledWith(1, { + Bucket: 'MyBucket', + Delete: { + Objects: [ + { Key: 'Key1', VersionId: 'VersionId1' }, + { Key: 'Key2', VersionId: 'VersionId2' }, + ], + }, + }); + expect(mockS3Client.deleteObjects).toHaveBeenNthCalledWith(2, { + Bucket: 'MyBucket', + Delete: { + Objects: [ + { Key: 'Key3', VersionId: 'VersionId3' }, + { Key: 'Key4', VersionId: 'VersionId4' }, + ], + }, + }); +}); + +// helper function to get around TypeScript expecting a complete event object, +// even though our tests only need some of the fields +async function invokeHandler(event: Partial) { + return handler(event as AWSLambda.CloudFormationCustomResourceEvent); +} diff --git a/packages/@aws-cdk/aws-s3/test/bucket.test.ts b/packages/@aws-cdk/aws-s3/test/bucket.test.ts index c9ea670dbb126..997a3abd4e56f 100644 --- a/packages/@aws-cdk/aws-s3/test/bucket.test.ts +++ b/packages/@aws-cdk/aws-s3/test/bucket.test.ts @@ -1,5 +1,5 @@ import { EOL } from 'os'; -import { expect, haveResource, haveResourceLike, SynthUtils, arrayWith, objectLike } from '@aws-cdk/assert'; +import { countResources, expect, haveResource, haveResourceLike, ResourcePart, SynthUtils, arrayWith, objectLike } from '@aws-cdk/assert'; import * as iam from '@aws-cdk/aws-iam'; import * as kms from '@aws-cdk/aws-kms'; import * as cdk from '@aws-cdk/core'; @@ -2316,4 +2316,115 @@ nodeunitShim({ }); test.done(); }, + + 'with autoDeleteObjects'(test: Test) { + const stack = new cdk.Stack(); + + new s3.Bucket(stack, 'MyBucket', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: true, + }); + + expect(stack).to(haveResource('AWS::S3::Bucket', { + UpdateReplacePolicy: 'Delete', + DeletionPolicy: 'Delete', + }, ResourcePart.CompleteDefinition)); + + expect(stack).to(haveResource('AWS::S3::BucketPolicy', { + Bucket: { + Ref: 'MyBucketF68F3FF0', + }, + 'PolicyDocument': { + 'Statement': [ + { + 'Action': [ + 's3:GetObject*', + 's3:GetBucket*', + 's3:List*', + 's3:DeleteObject*', + ], + 'Effect': 'Allow', + 'Principal': { + 'AWS': { + 'Fn::GetAtt': [ + 'CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092', + 'Arn', + ], + }, + }, + 'Resource': [ + { + 'Fn::GetAtt': [ + 'MyBucketF68F3FF0', + 'Arn', + ], + }, + { + 'Fn::Join': [ + '', + [ + { + 'Fn::GetAtt': [ + 'MyBucketF68F3FF0', + 'Arn', + ], + }, + '/*', + ], + ], + }, + ], + }, + ], + 'Version': '2012-10-17', + }, + })); + + expect(stack).to(haveResource('Custom::S3AutoDeleteObjects', { + 'Properties': { + 'ServiceToken': { + 'Fn::GetAtt': [ + 'CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F', + 'Arn', + ], + }, + 'BucketName': { + 'Ref': 'MyBucketF68F3FF0', + }, + }, + 'DependsOn': [ + 'MyBucketPolicyE7FBAC7B', + ], + }, ResourcePart.CompleteDefinition)); + + test.done(); + }, + + 'with autoDeleteObjects on multiple buckets'(test: Test) { + const stack = new cdk.Stack(); + + new s3.Bucket(stack, 'Bucket1', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: true, + }); + + new s3.Bucket(stack, 'Bucket2', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: true, + }); + + expect(stack).to(countResources('AWS::Lambda::Function', 1)); + + test.done(); + }, + + 'autoDeleteObjects throws if RemovalPolicy is not DESTROY'(test: Test) { + const stack = new cdk.Stack(); + + test.throws(() => new s3.Bucket(stack, 'MyBucket', { + autoDeleteObjects: true, + }), /Cannot use \'autoDeleteObjects\' property on a bucket without setting removal policy to \'DESTROY\'/); + + test.done(); + }, }); diff --git a/packages/@aws-cdk/aws-s3/test/integ.bucket-auto-delete-objects.expected.json b/packages/@aws-cdk/aws-s3/test/integ.bucket-auto-delete-objects.expected.json new file mode 100644 index 0000000000000..b4f691d566a7c --- /dev/null +++ b/packages/@aws-cdk/aws-s3/test/integ.bucket-auto-delete-objects.expected.json @@ -0,0 +1,172 @@ +{ + "Resources": { + "Bucket83908E77": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "BucketPolicyE9A3008A": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": { + "Ref": "Bucket83908E77" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*" + ], + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + } + }, + "Resource": [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + } + } + }, + "BucketAutoDeleteObjectsCustomResourceBAFD23C2": { + "Type": "Custom::S3AutoDeleteObjects", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F", + "Arn" + ] + }, + "BucketName": { + "Ref": "Bucket83908E77" + } + }, + "DependsOn": [ + "BucketPolicyE9A3008A" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ] + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + } + ] + } + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Ref": "AssetParameters94e60b8e13d1f4ded38f4813a8053fbe137c982a0b3c929daa8166d33f844316S3Bucket54B0BA6B" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters94e60b8e13d1f4ded38f4813a8053fbe137c982a0b3c929daa8166d33f844316S3VersionKey5D05A836" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters94e60b8e13d1f4ded38f4813a8053fbe137c982a0b3c929daa8166d33f844316S3VersionKey5D05A836" + } + ] + } + ] + } + ] + ] + } + }, + "Timeout": 900, + "MemorySize": 128, + "Handler": "__entrypoint__.handler", + "Role": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + }, + "Runtime": "nodejs12.x" + }, + "DependsOn": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092" + ] + } + }, + "Parameters": { + "AssetParameters94e60b8e13d1f4ded38f4813a8053fbe137c982a0b3c929daa8166d33f844316S3Bucket54B0BA6B": { + "Type": "String", + "Description": "S3 bucket for asset \"94e60b8e13d1f4ded38f4813a8053fbe137c982a0b3c929daa8166d33f844316\"" + }, + "AssetParameters94e60b8e13d1f4ded38f4813a8053fbe137c982a0b3c929daa8166d33f844316S3VersionKey5D05A836": { + "Type": "String", + "Description": "S3 key for asset version \"94e60b8e13d1f4ded38f4813a8053fbe137c982a0b3c929daa8166d33f844316\"" + }, + "AssetParameters94e60b8e13d1f4ded38f4813a8053fbe137c982a0b3c929daa8166d33f844316ArtifactHashCBB49470": { + "Type": "String", + "Description": "Artifact hash for asset \"94e60b8e13d1f4ded38f4813a8053fbe137c982a0b3c929daa8166d33f844316\"" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3/test/integ.bucket-auto-delete-objects.ts b/packages/@aws-cdk/aws-s3/test/integ.bucket-auto-delete-objects.ts new file mode 100644 index 0000000000000..70f3197aaa07f --- /dev/null +++ b/packages/@aws-cdk/aws-s3/test/integ.bucket-auto-delete-objects.ts @@ -0,0 +1,18 @@ +import { App, RemovalPolicy, Stack, StackProps } from '@aws-cdk/core'; +import { Construct } from 'constructs'; +import * as s3 from '../lib'; + +class TestStack extends Stack { + constructor(scope: Construct, id: string, props?: StackProps) { + super(scope, id, props); + + new s3.Bucket(this, 'Bucket', { + removalPolicy: RemovalPolicy.DESTROY, + autoDeleteObjects: true, + }); + } +} + +const app = new App(); +new TestStack(app, 'cdk-s3-bucket-auto-delete-objects'); +app.synth();