From 9402fb78981ff390790d7c75e80de554293f7999 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=B6hl=2C=20Lukas?= Date: Tue, 24 May 2022 14:30:48 +0200 Subject: [PATCH 1/7] feat(terraform): add ShowWithStruct function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Höhl, Lukas --- modules/terraform/show.go | 18 ++++++++++++++++++ modules/terraform/show_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/modules/terraform/show.go b/modules/terraform/show.go index d1121d5eb..3402d2904 100644 --- a/modules/terraform/show.go +++ b/modules/terraform/show.go @@ -28,3 +28,21 @@ func ShowE(t testing.TestingT, options *Options) (string, error) { } return RunTerraformCommandAndGetStdoutE(t, options, args...) } + +func ShowWithStruct(t testing.TestingT, options *Options) (*PlanStruct) { + out, err := ShowWithStructE(t, options) + require.NoError(t, err) + return out +} + +func ShowWithStructE(t testing.TestingT, options *Options) (*PlanStruct, error) { + json, err := ShowE(t, options) + if err != nil { + return nil, err + } + planStruct, err := parsePlanJson(json) + if err != nil { + return nil, err + } + return planStruct, nil +} \ No newline at end of file diff --git a/modules/terraform/show_test.go b/modules/terraform/show_test.go index c7e21a270..d844fff09 100644 --- a/modules/terraform/show_test.go +++ b/modules/terraform/show_test.go @@ -38,3 +38,33 @@ func TestShowWithInlinePlan(t *testing.T) { planJSON := Show(t, showOptions) require.Contains(t, planJSON, "null_resource.test[0]") } + +func TestShowWithStructInlinePlan(t *testing.T) { + t.Parallel() + + testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-basic-configuration", t.Name()) + require.NoError(t, err) + planFilePath := filepath.Join(testFolder, "plan.out") + + options := &Options{ + TerraformDir: testFolder, + PlanFilePath: planFilePath, + Vars: map[string]interface{}{ + "cnt": 1, + }, + } + + out := InitAndPlan(t, options) + require.Contains(t, out, fmt.Sprintf("Saved the plan to: %s", planFilePath)) + require.FileExists(t, planFilePath, "Plan file was not saved to expected location:", planFilePath) + + // show command does not accept Vars + showOptions := &Options{ + TerraformDir: testFolder, + PlanFilePath: planFilePath, + } + + // Test the JSON string + plan := ShowWithStruct(t, showOptions) + require.Contains(t, plan.ResourcePlannedValuesMap, "null_resource.test[0]") +} From 3f271655c8ee0ed5c1afdee99a005b3720447883 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=B6hl=2C=20Lukas?= Date: Fri, 27 May 2022 11:55:34 +0200 Subject: [PATCH 2/7] chore: format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Höhl, Lukas --- modules/terraform/show.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/terraform/show.go b/modules/terraform/show.go index 3402d2904..c21019488 100644 --- a/modules/terraform/show.go +++ b/modules/terraform/show.go @@ -29,7 +29,7 @@ func ShowE(t testing.TestingT, options *Options) (string, error) { return RunTerraformCommandAndGetStdoutE(t, options, args...) } -func ShowWithStruct(t testing.TestingT, options *Options) (*PlanStruct) { +func ShowWithStruct(t testing.TestingT, options *Options) *PlanStruct { out, err := ShowWithStructE(t, options) require.NoError(t, err) return out @@ -45,4 +45,4 @@ func ShowWithStructE(t testing.TestingT, options *Options) (*PlanStruct, error) return nil, err } return planStruct, nil -} \ No newline at end of file +} From 4e0bc5000b98b61b7bcf27fabc584ecfc32f11d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Jun 2022 19:00:12 -0500 Subject: [PATCH 3/7] Bump github.com/hashicorp/go-getter from 1.5.9 to 1.5.11 (#1110) Bumps [github.com/hashicorp/go-getter](https://github.com/hashicorp/go-getter) from 1.5.9 to 1.5.11. - [Release notes](https://github.com/hashicorp/go-getter/releases) - [Changelog](https://github.com/hashicorp/go-getter/blob/main/.goreleaser.yml) - [Commits](https://github.com/hashicorp/go-getter/compare/v1.5.9...v1.5.11) --- updated-dependencies: - dependency-name: github.com/hashicorp/go-getter dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Denis O --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7241cc1b6..3e5e95581 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/google/go-containerregistry v0.6.0 github.com/google/uuid v1.2.0 github.com/gruntwork-io/go-commons v0.8.0 - github.com/hashicorp/go-getter v1.5.9 + github.com/hashicorp/go-getter v1.5.11 github.com/hashicorp/go-multierror v1.1.0 github.com/hashicorp/go-version v1.3.0 github.com/hashicorp/hcl/v2 v2.9.1 diff --git a/go.sum b/go.sum index 24cd45229..2f9f3b499 100644 --- a/go.sum +++ b/go.sum @@ -482,8 +482,8 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-getter v1.5.9 h1:b7ahZW50iQiUek/at3CvZhPK1/jiV6CtKcsJiR6E4R0= -github.com/hashicorp/go-getter v1.5.9/go.mod h1:BrrV/1clo8cCYu6mxvboYg+KutTiFnXjMEgDD8+i7ZI= +github.com/hashicorp/go-getter v1.5.11 h1:wioTuNmaBU3IE9vdFtFMcmZWj0QzLc6DYaP6sNe5onY= +github.com/hashicorp/go-getter v1.5.11/go.mod h1:9i48BP6wpWweI/0/+FBjqLrp9S8XtwUGjiu0QkWHEaY= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= From acdc337dd41de98bf22a3b40ade0264a5c94fcaf Mon Sep 17 00:00:00 2001 From: Denis O Date: Thu, 2 Jun 2022 18:37:59 +0300 Subject: [PATCH 4/7] Add common list of regions to avoid in gcp test (#1133) --- modules/gcp/compute_test.go | 12 ++++++------ test/gcp/packer_gcp_basic_example_test.go | 7 ++++--- test/gcp/terraform_gcp_example_test.go | 4 +--- test/gcp/terraform_gcp_ig_example_test.go | 3 +-- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/modules/gcp/compute_test.go b/modules/gcp/compute_test.go index c29f87d3c..9d6fbe20d 100644 --- a/modules/gcp/compute_test.go +++ b/modules/gcp/compute_test.go @@ -22,12 +22,15 @@ const DEFAULT_MACHINE_TYPE = "f1-micro" const DEFAULT_IMAGE_FAMILY_PROJECT_NAME = "ubuntu-os-cloud" const DEFAULT_IMAGE_FAMILY_NAME = "family/ubuntu-1804-lts" +// Regions that don't support running f1-micro instances +var RegionsToAvoid = []string{"asia-east2", "southamerica-west1", "europe-west8"} + func TestGetPublicIpOfInstance(t *testing.T) { t.Parallel() instanceName := RandomValidGcpName() projectID := GetGoogleProjectIDFromEnvVar(t) - zone := GetRandomZone(t, projectID, nil, nil, []string{"southamerica-west1"}) + zone := GetRandomZone(t, projectID, nil, nil, RegionsToAvoid) createComputeInstance(t, projectID, zone, instanceName) defer deleteComputeInstance(t, projectID, zone, instanceName) @@ -74,8 +77,7 @@ func TestGetAndSetLabels(t *testing.T) { instanceName := RandomValidGcpName() projectID := GetGoogleProjectIDFromEnvVar(t) - // On October 22, 2018, GCP launched the asia-east2 region, which promptly failed all our tests, so blacklist asia-east2. - zone := GetRandomZone(t, projectID, nil, nil, []string{"asia-east2"}) + zone := GetRandomZone(t, projectID, nil, nil, RegionsToAvoid) createComputeInstance(t, projectID, zone, instanceName) defer deleteComputeInstance(t, projectID, zone, instanceName) @@ -111,9 +113,7 @@ func TestGetAndSetMetadata(t *testing.T) { projectID := GetGoogleProjectIDFromEnvVar(t) instanceName := RandomValidGcpName() - // The following zones do not have f1-micro instances available, so we avoid them - zonesToAvoid := []string{"asia-east2", "southamerica-west1"} - zone := GetRandomZone(t, projectID, nil, nil, zonesToAvoid) + zone := GetRandomZone(t, projectID, nil, nil, RegionsToAvoid) // Create a new Compute Instance createComputeInstance(t, projectID, zone, instanceName) diff --git a/test/gcp/packer_gcp_basic_example_test.go b/test/gcp/packer_gcp_basic_example_test.go index ec3da1fee..df3eb0911 100644 --- a/test/gcp/packer_gcp_basic_example_test.go +++ b/test/gcp/packer_gcp_basic_example_test.go @@ -22,6 +22,9 @@ var DefaultRetryablePackerErrors = map[string]string{ } var DefaultTimeBetweenPackerRetries = 15 * time.Second +// Regions that don't support n1-standard-1 instances +var RegionsToAvoid = []string{"asia-east2", "southamerica-west1", "europe-west8"} + const DefaultMaxPackerRetries = 3 // An example of how to test the Packer template in examples/packer-basic-example using Terratest. @@ -32,9 +35,7 @@ func TestPackerGCPBasicExample(t *testing.T) { projectID := gcp.GetGoogleProjectIDFromEnvVar(t) // Pick a random GCP zone to test in. This helps ensure your code works in all regions. - // zones that don't support n1-standard-1 instances - zonesToAvoid := []string{"asia-east2", "southamerica-west1"} - zone := gcp.GetRandomZone(t, projectID, nil, nil, zonesToAvoid) + zone := gcp.GetRandomZone(t, projectID, nil, nil, RegionsToAvoid) packerOptions := &packer.Options{ // The path to where the Packer template is located diff --git a/test/gcp/terraform_gcp_example_test.go b/test/gcp/terraform_gcp_example_test.go index adf5ca5d0..b934d58f8 100644 --- a/test/gcp/terraform_gcp_example_test.go +++ b/test/gcp/terraform_gcp_example_test.go @@ -109,9 +109,7 @@ func TestSshAccessToComputeInstance(t *testing.T) { // Setup values for our Terraform apply projectID := gcp.GetGoogleProjectIDFromEnvVar(t) randomValidGcpName := gcp.RandomValidGcpName() - // zones that don't support n1-standard-1 instances - zonesToAvoid := []string{"asia-east2", "southamerica-west1"} - zone := gcp.GetRandomZone(t, projectID, nil, nil, zonesToAvoid) + zone := gcp.GetRandomZone(t, projectID, nil, nil, RegionsToAvoid) terraformOptions := &terraform.Options{ // The path to where our Terraform code is located diff --git a/test/gcp/terraform_gcp_ig_example_test.go b/test/gcp/terraform_gcp_ig_example_test.go index ebf7e7b07..8ae1b8105 100644 --- a/test/gcp/terraform_gcp_ig_example_test.go +++ b/test/gcp/terraform_gcp_ig_example_test.go @@ -24,8 +24,7 @@ func TestTerraformGcpInstanceGroupExample(t *testing.T) { // Setup values for our Terraform apply projectId := gcp.GetGoogleProjectIDFromEnvVar(t) - // On October 22, 2018, GCP launched the asia-east2 region, which promptly failed all our tests, so blacklist asia-east2. - region := gcp.GetRandomRegion(t, projectId, nil, []string{"asia-east2"}) + region := gcp.GetRandomRegion(t, projectId, nil, RegionsToAvoid) randomValidGcpName := gcp.RandomValidGcpName() clusterSize := 3 From 2070f6817f58c6554d05f0773d82c9a1b0d4a182 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Mon, 6 Jun 2022 09:11:19 -0500 Subject: [PATCH 5/7] Handle new plan output in 1.2.0 (#1137) * Handle new output format in terraform 1.2 for resource counting * Avoid europe-southwest1 * Capture newlines as well * Version lock perl image to make tests pass * Fix regex to handle newlines between the strings --- modules/gcp/compute_test.go | 2 +- modules/k8s/job_test.go | 2 +- modules/terraform/count.go | 4 ++++ test/gcp/packer_gcp_basic_example_test.go | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/gcp/compute_test.go b/modules/gcp/compute_test.go index 9d6fbe20d..8c1242eb4 100644 --- a/modules/gcp/compute_test.go +++ b/modules/gcp/compute_test.go @@ -23,7 +23,7 @@ const DEFAULT_IMAGE_FAMILY_PROJECT_NAME = "ubuntu-os-cloud" const DEFAULT_IMAGE_FAMILY_NAME = "family/ubuntu-1804-lts" // Regions that don't support running f1-micro instances -var RegionsToAvoid = []string{"asia-east2", "southamerica-west1", "europe-west8"} +var RegionsToAvoid = []string{"asia-east2", "southamerica-west1", "europe-west8", "europe-southwest1"} func TestGetPublicIpOfInstance(t *testing.T) { t.Parallel() diff --git a/modules/k8s/job_test.go b/modules/k8s/job_test.go index d92d07af3..f579e18ed 100644 --- a/modules/k8s/job_test.go +++ b/modules/k8s/job_test.go @@ -145,7 +145,7 @@ spec: spec: containers: - name: pi - image: perl + image: "perl:5.34.1" command: ["perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"] restartPolicy: Never backoffLimit: 4 diff --git a/modules/terraform/count.go b/modules/terraform/count.go index 429a4ab77..d52e55ee1 100644 --- a/modules/terraform/count.go +++ b/modules/terraform/count.go @@ -22,6 +22,9 @@ const ( destroyRegexp = `Destroy complete! Resources: (\d+) destroyed\.` planWithChangesRegexp = `(\033\[1m)?Plan:(\033\[0m)? (\d+) to add, (\d+) to change, (\d+) to destroy\.` planWithNoChangesRegexp = `No changes\. (Infrastructure is up-to-date)|(Your infrastructure matches the configuration)\.` + + // '.' doesn't match newline by default in go. We must instruct the regex to match it with the 's' flag. + planWithNoInfraChangesRegexp = `(?s)You can apply this plan.+without changing any real infrastructure` ) const getResourceCountErrMessage = "Can't parse Terraform output" @@ -48,6 +51,7 @@ func GetResourceCountE(t testing.TestingT, cmdout string) (*ResourceCount, error {destroyRegexp, -1, -1, 1}, {planWithChangesRegexp, 3, 4, 5}, {planWithNoChangesRegexp, -1, -1, -1}, + {planWithNoInfraChangesRegexp, -1, -1, -1}, } for _, tc := range terraformCommandPatterns { diff --git a/test/gcp/packer_gcp_basic_example_test.go b/test/gcp/packer_gcp_basic_example_test.go index df3eb0911..1032db4e9 100644 --- a/test/gcp/packer_gcp_basic_example_test.go +++ b/test/gcp/packer_gcp_basic_example_test.go @@ -23,7 +23,7 @@ var DefaultRetryablePackerErrors = map[string]string{ var DefaultTimeBetweenPackerRetries = 15 * time.Second // Regions that don't support n1-standard-1 instances -var RegionsToAvoid = []string{"asia-east2", "southamerica-west1", "europe-west8"} +var RegionsToAvoid = []string{"asia-east2", "southamerica-west1", "europe-west8", "europe-southwest1"} const DefaultMaxPackerRetries = 3 From 94dd1e7bb2e80248a3322f12f4068e59559e6844 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 09:20:26 -0500 Subject: [PATCH 6/7] Bump github.com/hashicorp/go-getter from 1.5.11 to 1.6.1 (#1136) Bumps [github.com/hashicorp/go-getter](https://github.com/hashicorp/go-getter) from 1.5.11 to 1.6.1. - [Release notes](https://github.com/hashicorp/go-getter/releases) - [Changelog](https://github.com/hashicorp/go-getter/blob/main/.goreleaser.yml) - [Commits](https://github.com/hashicorp/go-getter/compare/v1.5.11...v1.6.1) --- updated-dependencies: - dependency-name: github.com/hashicorp/go-getter dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 3e5e95581..32f738bfd 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/google/go-containerregistry v0.6.0 github.com/google/uuid v1.2.0 github.com/gruntwork-io/go-commons v0.8.0 - github.com/hashicorp/go-getter v1.5.11 + github.com/hashicorp/go-getter v1.6.1 github.com/hashicorp/go-multierror v1.1.0 github.com/hashicorp/go-version v1.3.0 github.com/hashicorp/hcl/v2 v2.9.1 @@ -102,7 +102,7 @@ require ( golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect golang.org/x/mod v0.4.2 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.0.0-20210603125802-9665404d3644 // indirect + golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e // indirect golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect diff --git a/go.sum b/go.sum index 2f9f3b499..87098efd9 100644 --- a/go.sum +++ b/go.sum @@ -482,8 +482,8 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-getter v1.5.11 h1:wioTuNmaBU3IE9vdFtFMcmZWj0QzLc6DYaP6sNe5onY= -github.com/hashicorp/go-getter v1.5.11/go.mod h1:9i48BP6wpWweI/0/+FBjqLrp9S8XtwUGjiu0QkWHEaY= +github.com/hashicorp/go-getter v1.6.1 h1:NASsgP4q6tL94WH6nJxKWj8As2H/2kop/bB1d8JMyRY= +github.com/hashicorp/go-getter v1.6.1/go.mod h1:IZCrswsZPeWv9IkVnLElzRU/gz/QPi6pZHn4tv6vbwA= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= @@ -1069,8 +1069,9 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644 h1:CA1DEQ4NdKphKeL70tvsWNdT5oFh1lOjihRcEDROi0I= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e h1:w36l2Uw3dRan1K3TyXriXvY+6T56GNmlKGcqiQUJDfM= +golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 0dd0f81e8cee879995a10660f6e084d9334a8e53 Mon Sep 17 00:00:00 2001 From: Etiene Dalcol Date: Tue, 14 Jun 2022 17:29:19 +0200 Subject: [PATCH 7/7] Get default aws subnet ids (#1131) * add function for getting default aws subnet ids * check main route table when checking if subnet without route tables is public * fix aws TestGetVpcsE test * remove redundant true check * add check for recently created subnet on GetDefaultSubnetIDsForVpc test --- modules/aws/vpc.go | 76 +++++++++++++++++++++++++++++++++++++++-- modules/aws/vpc_test.go | 75 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 148 insertions(+), 3 deletions(-) diff --git a/modules/aws/vpc.go b/modules/aws/vpc.go index 096b6cab8..f3f5b78cf 100644 --- a/modules/aws/vpc.go +++ b/modules/aws/vpc.go @@ -24,6 +24,7 @@ type Vpc struct { type Subnet struct { Id string // The ID of the Subnet AvailabilityZone string // The Availability Zone the subnet is in + DefaultForAz bool // If the subnet is default for the Availability Zone Tags map[string]string // The tags associated with the subnet } @@ -34,6 +35,7 @@ const vpcResourceTypeFilterValue = "vpc" const subnetResourceTypeFilterValue = "subnet" const isDefaultFilterName = "isDefault" const isDefaultFilterValue = "true" +const defaultVPCName = "Default" // GetDefaultVpc fetches information about the default VPC in the given region. func GetDefaultVpc(t testing.TestingT, region string) *Vpc { @@ -117,7 +119,7 @@ func FindVpcName(vpc *ec2.Vpc) string { } if *vpc.IsDefault { - return "Default" + return defaultVPCName } return "" @@ -149,7 +151,7 @@ func GetSubnetsForVpcE(t testing.TestingT, vpcID string, region string) ([]Subne for _, ec2Subnet := range subnetOutput.Subnets { subnetTags := GetTagsForSubnet(t, *ec2Subnet.SubnetId, region) - subnet := Subnet{Id: aws.StringValue(ec2Subnet.SubnetId), AvailabilityZone: aws.StringValue(ec2Subnet.AvailabilityZone), Tags: subnetTags} + subnet := Subnet{Id: aws.StringValue(ec2Subnet.SubnetId), AvailabilityZone: aws.StringValue(ec2Subnet.AvailabilityZone), DefaultForAz: aws.BoolValue(ec2Subnet.DefaultForAz), Tags: subnetTags} subnets = append(subnets, subnet) } @@ -182,6 +184,34 @@ func GetTagsForVpcE(t testing.TestingT, vpcID string, region string) (map[string return tags, nil } +// GetDefaultSubnetIDsForVpc gets the ids of the subnets that are the default subnet for the AvailabilityZone +func GetDefaultSubnetIDsForVpc(t testing.TestingT, vpc Vpc) []string { + subnetIDs, err := GetDefaultSubnetIDsForVpcE(t, vpc) + require.NoError(t, err) + return subnetIDs +} + +// GetDefaultSubnetIDsForVpcE gets the ids of the subnets that are the default subnet for the AvailabilityZone +func GetDefaultSubnetIDsForVpcE(t testing.TestingT, vpc Vpc) ([]string, error) { + if vpc.Name != defaultVPCName { + // You cannot create a default subnet in a nondefault VPC + // https://docs.aws.amazon.com/vpc/latest/userguide/default-vpc.html + return nil, fmt.Errorf("Only default VPCs have default subnets but VPC with id %s is not default VPC", vpc.Id) + } + subnetIDs := []string{} + numSubnets := len(vpc.Subnets) + if numSubnets == 0 { + return nil, fmt.Errorf("Expected to find at least one subnet in vpc with ID %s but found zero", vpc.Id) + } + + for _, subnet := range vpc.Subnets { + if subnet.DefaultForAz { + subnetIDs = append(subnetIDs, subnet.Id) + } + } + return subnetIDs, nil +} + // GetTagsForSubnet gets the tags for the specified subnet. func GetTagsForSubnet(t testing.TestingT, subnetId string, region string) map[string]string { tags, err := GetTagsForSubnetE(t, subnetId, region) @@ -234,6 +264,14 @@ func IsPublicSubnetE(t testing.TestingT, subnetId string, region string) (bool, return false, err } + if len(rts.RouteTables) == 0 { + // Subnets not explicitly associated with any route table are implicitly associated with the main route table + rts, err = getImplicitRouteTableForSubnetE(t, subnetId, region) + if err != nil { + return false, err + } + } + for _, rt := range rts.RouteTables { for _, r := range rt.Routes { if strings.HasPrefix(aws.StringValue(r.GatewayId), "igw-") { @@ -245,6 +283,40 @@ func IsPublicSubnetE(t testing.TestingT, subnetId string, region string) (bool, return false, nil } +func getImplicitRouteTableForSubnetE(t testing.TestingT, subnetId string, region string) (*ec2.DescribeRouteTablesOutput, error) { + mainRouteFilterName := "association.main" + mainRouteFilterValue := "true" + subnetFilterName := "subnet-id" + + client, err := NewEc2ClientE(t, region) + if err != nil { + return nil, err + } + + subnetFilter := ec2.Filter{ + Name: &subnetFilterName, + Values: []*string{&subnetId}, + } + subnetOutput, err := client.DescribeSubnets(&ec2.DescribeSubnetsInput{Filters: []*ec2.Filter{&subnetFilter}}) + if err != nil { + return nil, err + } + numSubnets := len(subnetOutput.Subnets) + if numSubnets != 1 { + return nil, fmt.Errorf("Expected to find one subnet with id %s but found %s", subnetId, strconv.Itoa(numSubnets)) + } + + mainRouteFilter := ec2.Filter{ + Name: &mainRouteFilterName, + Values: []*string{&mainRouteFilterValue}, + } + vpcFilter := ec2.Filter{ + Name: aws.String(vpcIDFilterName), + Values: []*string{subnetOutput.Subnets[0].VpcId}, + } + return client.DescribeRouteTables(&ec2.DescribeRouteTablesInput{Filters: []*ec2.Filter{&mainRouteFilter, &vpcFilter}}) +} + // GetRandomPrivateCidrBlock gets a random CIDR block from the range of acceptable private IP addresses per RFC 1918 // (https://tools.ietf.org/html/rfc1918#section-3) // The routingPrefix refers to the "/28" in 1.2.3.4/28. diff --git a/modules/aws/vpc_test.go b/modules/aws/vpc_test.go index c4651cb3b..35e6a213c 100644 --- a/modules/aws/vpc_test.go +++ b/modules/aws/vpc_test.go @@ -1,6 +1,7 @@ package aws import ( + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -49,7 +50,7 @@ func TestGetVpcsE(t *testing.T) { // the default VPC has by default one subnet per availability zone // https://docs.aws.amazon.com/vpc/latest/userguide/default-vpc.html - assert.Equal(t, len(vpcs[0].Subnets), len(azs)) + assert.True(t, len(vpcs[0].Subnets) >= len(azs)) } func TestGetFirstTwoOctets(t *testing.T) { @@ -76,6 +77,46 @@ func TestIsPublicSubnet(t *testing.T) { assert.True(t, IsPublicSubnet(t, *subnet.SubnetId, region)) } +func TestGetDefaultSubnetIDsForVpc(t *testing.T) { + t.Parallel() + + region := GetRandomStableRegion(t, nil, nil) + defaultVpcBeforeSubnetCreation := GetDefaultVpc(t, region) + + // Creates a subnet in the default VPC with deferred deletion + // and fetches vpc object again + subnetName := fmt.Sprintf("%s-subnet", t.Name()) + subnet := createPrivateSubnetInDefaultVpc(t, defaultVpcBeforeSubnetCreation.Id, subnetName, region) + defer deleteSubnet(t, *subnet.SubnetId, region) + defaultVpc := GetDefaultVpc(t, region) + + defaultSubnetIDs := GetDefaultSubnetIDsForVpc(t, *defaultVpc) + assert.NotEmpty(t, defaultSubnetIDs) + // Checks that the amount of default subnets is smaller than + // total number of subnets in default vpc + assert.True(t, len(defaultSubnetIDs) < len(defaultVpc.Subnets)) + + availabilityZones := []string{} + for _, id := range defaultSubnetIDs { + // check if the recently created subnet does not come up here + assert.NotEqual(t, id, subnet.SubnetId) + // default subnets are by default public + // https://docs.aws.amazon.com/vpc/latest/userguide/default-vpc.html + assert.True(t, IsPublicSubnet(t, id, region)) + for _, subnet := range defaultVpc.Subnets { + if id == subnet.Id { + availabilityZones = append(availabilityZones, subnet.AvailabilityZone) + } + } + } + // only one default subnet is allowed per AZ + uniqueAZs := map[string]bool{} + for _, az := range availabilityZones { + uniqueAZs[az] = true + } + assert.Equal(t, len(defaultSubnetIDs), len(uniqueAZs)) +} + func TestGetTagsForVpc(t *testing.T) { t.Parallel() @@ -178,6 +219,38 @@ func createSubnet(t *testing.T, vpcId string, routeTableId string, region string return *createSubnetOutput.Subnet } +func createPrivateSubnetInDefaultVpc(t *testing.T, vpcId string, subnetName string, region string) ec2.Subnet { + ec2Client := NewEc2Client(t, region) + + createSubnetOutput, err := ec2Client.CreateSubnet(&ec2.CreateSubnetInput{ + CidrBlock: aws.String("172.31.172.0/24"), + VpcId: aws.String(vpcId), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String(subnetName), + }, + }, + }, + }, + }) + require.NoError(t, err) + + return *createSubnetOutput.Subnet +} + +func deleteSubnet(t *testing.T, subnetId string, region string) { + ec2Client := NewEc2Client(t, region) + + _, err := ec2Client.DeleteSubnet(&ec2.DeleteSubnetInput{ + SubnetId: aws.String(subnetId), + }) + require.NoError(t, err) +} + func createVpc(t *testing.T, region string) ec2.Vpc { ec2Client := NewEc2Client(t, region)