From 87646204b0a6972b342f1801416deb928beda543 Mon Sep 17 00:00:00 2001 From: pp Date: Sun, 13 Oct 2019 11:39:45 +0300 Subject: [PATCH] Add test for sandbox * Moved `sanbox_config` to `node_pool` resouce (Fix #240) * Created test for sandbox (Fix #252) * Updated docker image version --- .kitchen.yml | 7 ++ Makefile | 2 +- autogen/cluster.tf | 16 +-- build/int.cloudbuild.yaml | 2 +- build/lint.cloudbuild.yaml | 2 +- examples/simple_regional_beta/README.md | 10 +- examples/simple_regional_beta/main.tf | 35 +++--- examples/simple_regional_beta/variables.tf | 33 +++++- .../cluster.tf | 16 +-- modules/beta-private-cluster/cluster.tf | 16 +-- modules/beta-public-cluster/cluster.tf | 16 +-- test/fixtures/sandbox_enabled/example.tf | 39 +++++++ test/fixtures/sandbox_enabled/network.tf | 48 +++++++++ test/fixtures/sandbox_enabled/outputs.tf | 1 + test/fixtures/sandbox_enabled/variables.tf | 1 + .../sandbox_enabled/controls/gcloud.rb | 102 ++++++++++++++++++ test/integration/sandbox_enabled/inspec.yml | 17 +++ test/setup/main.tf | 13 ++- test/task_helper_functions.sh | 13 --- 19 files changed, 312 insertions(+), 77 deletions(-) create mode 100644 test/fixtures/sandbox_enabled/example.tf create mode 100644 test/fixtures/sandbox_enabled/network.tf create mode 120000 test/fixtures/sandbox_enabled/outputs.tf create mode 120000 test/fixtures/sandbox_enabled/variables.tf create mode 100644 test/integration/sandbox_enabled/controls/gcloud.rb create mode 100644 test/integration/sandbox_enabled/inspec.yml diff --git a/.kitchen.yml b/.kitchen.yml index 9f5df5a03e..61b8aae029 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -131,3 +131,10 @@ suites: systems: - name: workload_metadata_config backend: local + - name: "sandbox_enabled" + driver: + root_module_directory: test/fixtures/sandbox_enabled + verifier: + systems: + - name: sandbox_enabled + backend: local diff --git a/Makefile b/Makefile index 5039822a75..686914ac7e 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ # Make will use bash instead of sh SHELL := /usr/bin/env bash -DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.1.0 +DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.4.2 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools REGISTRY_URL := gcr.io/cloud-foundation-cicd diff --git a/autogen/cluster.tf b/autogen/cluster.tf index 296b2818df..035eeb0cf7 100644 --- a/autogen/cluster.tf +++ b/autogen/cluster.tf @@ -167,14 +167,6 @@ resource "google_container_cluster" "primary" { node_metadata = workload_metadata_config.value.node_metadata } } - - dynamic "sandbox_config" { - for_each = local.cluster_sandbox_enabled - - content { - sandbox_type = sandbox_config.value - } - } {% endif %} } } @@ -415,6 +407,14 @@ resource "google_container_node_pool" "pools" { node_metadata = workload_metadata_config.value.node_metadata } } + + dynamic "sandbox_config" { + for_each = local.cluster_sandbox_enabled + + content { + sandbox_type = sandbox_config.value + } + } {% endif %} } diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 85139efe7d..381939f3bc 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -38,4 +38,4 @@ tags: - 'integration' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.2' diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml index 3b7306297c..3cf4b8c12a 100644 --- a/build/lint.cloudbuild.yaml +++ b/build/lint.cloudbuild.yaml @@ -24,4 +24,4 @@ tags: - 'lint' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.2' diff --git a/examples/simple_regional_beta/README.md b/examples/simple_regional_beta/README.md index bd676115b9..899e919f4d 100644 --- a/examples/simple_regional_beta/README.md +++ b/examples/simple_regional_beta/README.md @@ -2,8 +2,7 @@ This example illustrates how to create a simple cluster with beta features. -[^]: (autogen_docs_start) - + ## Inputs | Name | Description | Type | Default | Required | @@ -11,13 +10,16 @@ This example illustrates how to create a simple cluster with beta features. | cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | -| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | | ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | | istio | Boolean to enable / disable Istio | string | `"true"` | no | | network | The VPC network to host the cluster in | string | n/a | yes | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | | project\_id | The project ID to host the cluster in | string | n/a | yes | | region | The region to host the cluster in | string | n/a | yes | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | | subnetwork | The subnetwork to host the cluster in | string | n/a | yes | ## Outputs @@ -40,7 +42,7 @@ This example illustrates how to create a simple cluster with beta features. | subnetwork | | | zones | List of zones in which the cluster resides | -[^]: (autogen_docs_end) + To provision this example, run the following from within this directory: - `terraform init` to get the plugins diff --git a/examples/simple_regional_beta/main.tf b/examples/simple_regional_beta/main.tf index fc95090ede..b75fdaa613 100644 --- a/examples/simple_regional_beta/main.tf +++ b/examples/simple_regional_beta/main.tf @@ -19,25 +19,28 @@ locals { } provider "google-beta" { - version = "~> 2.12.0" - credentials = file(var.credentials_path) - region = var.region + version = "~> 2.12.0" + region = var.region } module "gke" { - source = "../../modules/beta-public-cluster/" - project_id = var.project_id - name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" - regional = true - region = var.region - network = var.network - subnetwork = var.subnetwork - ip_range_pods = var.ip_range_pods - ip_range_services = var.ip_range_services - create_service_account = false - service_account = var.compute_engine_service_account - istio = var.istio - cloudrun = var.cloudrun + source = "../../modules/beta-public-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + create_service_account = false + service_account = var.compute_engine_service_account + istio = var.istio + cloudrun = var.cloudrun + node_metadata = var.node_metadata + sandbox_enabled = var.sandbox_enabled + remove_default_node_pool = var.remove_default_node_pool + node_pools = var.node_pools } data "google_client_config" "default" { diff --git a/examples/simple_regional_beta/variables.tf b/examples/simple_regional_beta/variables.tf index 1da408a790..ed16642774 100644 --- a/examples/simple_regional_beta/variables.tf +++ b/examples/simple_regional_beta/variables.tf @@ -18,10 +18,6 @@ variable "project_id" { description = "The project ID to host the cluster in" } -variable "credentials_path" { - description = "The path to the GCP credentials JSON file" -} - variable "cluster_name_suffix" { description = "A suffix to append to the default cluster name" default = "" @@ -60,3 +56,32 @@ variable "cloudrun" { description = "Boolean to enable / disable CloudRun" default = true } + +variable "node_metadata" { + description = "Specifies how node metadata is exposed to the workload running on the node" + default = "SECURE" + type = string +} + +variable "sandbox_enabled" { + type = bool + description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it)." + default = false +} + +variable "remove_default_node_pool" { + type = bool + description = "Remove default node pool while setting up the cluster" + default = false +} + +variable "node_pools" { + type = list(map(string)) + description = "List of maps containing node pools" + + default = [ + { + name = "default-node-pool" + }, + ] +} diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index cf1def945d..2348150d19 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -158,14 +158,6 @@ resource "google_container_cluster" "primary" { node_metadata = workload_metadata_config.value.node_metadata } } - - dynamic "sandbox_config" { - for_each = local.cluster_sandbox_enabled - - content { - sandbox_type = sandbox_config.value - } - } } } @@ -386,6 +378,14 @@ resource "google_container_node_pool" "pools" { node_metadata = workload_metadata_config.value.node_metadata } } + + dynamic "sandbox_config" { + for_each = local.cluster_sandbox_enabled + + content { + sandbox_type = sandbox_config.value + } + } } lifecycle { diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index c481c69a35..56f40ed17a 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -158,14 +158,6 @@ resource "google_container_cluster" "primary" { node_metadata = workload_metadata_config.value.node_metadata } } - - dynamic "sandbox_config" { - for_each = local.cluster_sandbox_enabled - - content { - sandbox_type = sandbox_config.value - } - } } } @@ -314,6 +306,14 @@ resource "google_container_node_pool" "pools" { node_metadata = workload_metadata_config.value.node_metadata } } + + dynamic "sandbox_config" { + for_each = local.cluster_sandbox_enabled + + content { + sandbox_type = sandbox_config.value + } + } } lifecycle { diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index a264e932b9..e37b2b3b31 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -158,14 +158,6 @@ resource "google_container_cluster" "primary" { node_metadata = workload_metadata_config.value.node_metadata } } - - dynamic "sandbox_config" { - for_each = local.cluster_sandbox_enabled - - content { - sandbox_type = sandbox_config.value - } - } } } @@ -309,6 +301,14 @@ resource "google_container_node_pool" "pools" { node_metadata = workload_metadata_config.value.node_metadata } } + + dynamic "sandbox_config" { + for_each = local.cluster_sandbox_enabled + + content { + sandbox_type = sandbox_config.value + } + } } lifecycle { diff --git a/test/fixtures/sandbox_enabled/example.tf b/test/fixtures/sandbox_enabled/example.tf new file mode 100644 index 0000000000..82522c4b0b --- /dev/null +++ b/test/fixtures/sandbox_enabled/example.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/simple_regional_beta" + + project_id = var.project_id + cluster_name_suffix = "-${random_string.suffix.result}" + region = var.region + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + compute_engine_service_account = var.compute_engine_service_account + istio = false + cloudrun = false + node_metadata = "UNSPECIFIED" + sandbox_enabled = true + remove_default_node_pool = true + node_pools = [ + { + name = "default-node-pool" + image_type = "COS_CONTAINERD" + }, + ] +} diff --git a/test/fixtures/sandbox_enabled/network.tf b/test/fixtures/sandbox_enabled/network.tf new file mode 100644 index 0000000000..5d34d43748 --- /dev/null +++ b/test/fixtures/sandbox_enabled/network.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +provider "google" { + project = var.project_id +} + +resource "google_compute_network" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} + diff --git a/test/fixtures/sandbox_enabled/outputs.tf b/test/fixtures/sandbox_enabled/outputs.tf new file mode 120000 index 0000000000..726bdc722f --- /dev/null +++ b/test/fixtures/sandbox_enabled/outputs.tf @@ -0,0 +1 @@ +../shared/outputs.tf \ No newline at end of file diff --git a/test/fixtures/sandbox_enabled/variables.tf b/test/fixtures/sandbox_enabled/variables.tf new file mode 120000 index 0000000000..c113c00a3d --- /dev/null +++ b/test/fixtures/sandbox_enabled/variables.tf @@ -0,0 +1 @@ +../shared/variables.tf \ No newline at end of file diff --git a/test/integration/sandbox_enabled/controls/gcloud.rb b/test/integration/sandbox_enabled/controls/gcloud.rb new file mode 100644 index 0000000000..eb0ffdaf46 --- /dev/null +++ b/test/integration/sandbox_enabled/controls/gcloud.rb @@ -0,0 +1,102 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is regional" do + expect(data['location']).to match(/^.*[1-9]$/) + end + + it "uses public nodes and master endpoint" do + expect(data['privateClusterConfig']).to eq nil + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => { + "disabled" => true, + }, + }) + end + end + + describe "node pool" do + let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } + + it "is the expected image type" do + expect(node_pools).to include( + including( + "config" => including( + "imageType" => "COS_CONTAINERD", + ), + ) + ) + end + + it "has the expected labels" do + expect(node_pools).to include( + including( + "config" => including( + "labels" => including( + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + "sandbox.gke.io/runtime" => "gvisor", + ), + ), + ) + ) + end + + it "has the expected network tags" do + expect(node_pools).to include( + including( + "config" => including( + "tags" => match_array([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]), + ), + ) + ) + end + + end + end +end diff --git a/test/integration/sandbox_enabled/inspec.yml b/test/integration/sandbox_enabled/inspec.yml new file mode 100644 index 0000000000..0454937a36 --- /dev/null +++ b/test/integration/sandbox_enabled/inspec.yml @@ -0,0 +1,17 @@ +name: sandbox_enabled +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string diff --git a/test/setup/main.tf b/test/setup/main.tf index f974c7408e..f7d54968a6 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -14,15 +14,18 @@ * limitations under the License. */ +resource "random_id" "random_project" { + byte_length = 2 +} + module "gke-project" { source = "terraform-google-modules/project-factory/google" version = "~> 3.0" - name = "ci-gke" - random_project_id = true - org_id = var.org_id - folder_id = var.folder_id - billing_account = var.billing_account + name = "ci-gke-${random_id.random_project.hex}" + org_id = var.org_id + folder_id = var.folder_id + billing_account = var.billing_account activate_apis = [ "bigquery-json.googleapis.com", diff --git a/test/task_helper_functions.sh b/test/task_helper_functions.sh index 70ab3db5c8..ddfbab53c7 100755 --- a/test/task_helper_functions.sh +++ b/test/task_helper_functions.sh @@ -49,16 +49,3 @@ function check_generate() { rm -Rf "${tempdir}" return $((rval)) } - -find_files() { - local pth="$1" - shift - find "${pth}" '(' \ - -path '*/.git' -o \ - -path '*/.terraform' -o \ - -path '*/.kitchen' -o \ - -path './autogen' -o \ - -path './test/fixtures/all_examples' -o \ - -path './test/fixtures/shared' ')' \ - -prune -o -type f "$@" -}