diff --git a/.kitchen.yml b/.kitchen.yml index 6bf414c21f..425e3d84b1 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -125,3 +125,16 @@ suites: systems: - name: workload_metadata_config backend: local + - name: "beta_cluster" + driver: + root_module_directory: test/fixtures/beta_cluster + verifier: + systems: + - name: gcloud + backend: local + controls: + - gcloud + - name: gcp + backend: gcp + controls: + - gcp diff --git a/Makefile b/Makefile index 5c184a3d36..95abe1e74d 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ SHELL := /usr/bin/env bash # Docker build config variables CREDENTIALS_PATH ?= /cft/workdir/credentials.json DOCKER_ORG := gcr.io/cloud-foundation-cicd -DOCKER_TAG_BASE_KITCHEN_TERRAFORM ?= 2.1.0 +DOCKER_TAG_BASE_KITCHEN_TERRAFORM ?= 2.3.0 DOCKER_REPO_BASE_KITCHEN_TERRAFORM := ${DOCKER_ORG}/cft/kitchen-terraform:${DOCKER_TAG_BASE_KITCHEN_TERRAFORM} # All is the first target in the file so it will get picked up when you just run 'make' on its own diff --git a/test/ci/beta-cluster.yml b/test/ci/beta-cluster.yml new file mode 100644 index 0000000000..dd4ce29302 --- /dev/null +++ b/test/ci/beta-cluster.yml @@ -0,0 +1,18 @@ +--- + +platform: linux + +inputs: +- name: pull-request + path: terraform-google-kubernetes-engine + +run: + path: make + args: ['test_integration'] + dir: terraform-google-kubernetes-engine + +params: + SUITE: "beta-cluster-local" + COMPUTE_ENGINE_SERVICE_ACCOUNT: "" + REGION: "us-east4" + ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]' diff --git a/test/fixtures/beta_cluster/main.tf b/test/fixtures/beta_cluster/main.tf new file mode 100644 index 0000000000..3478a29825 --- /dev/null +++ b/test/fixtures/beta_cluster/main.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +provider "google" { + version = "~> 2.9.0" + project = var.project_id + region = var.region +} + +provider "google-beta" { + version = "~> 2.9.0" + project = var.project_id + region = var.region +} + +locals { + name = "beta-cluster-${random_string.suffix.result}" +} + +resource "google_kms_key_ring" "db" { + location = var.region + name = "${local.name}-db" +} + +resource "google_kms_crypto_key" "db" { + name = local.name + key_ring = google_kms_key_ring.db.self_link +} + +module "this" { + source = "../../../modules/beta-public-cluster" + + name = local.name + project_id = var.project_id + regional = false + region = var.region + zones = slice(var.zones, 0, 1) + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + service_account = "create" + + // Beta features + istio = true + database_encryption = [{ + state = "ENCRYPTED" + key_name = google_kms_crypto_key.db.self_link + }] + cloudrun = true + enable_binary_authorization = true + pod_security_policy_config = [{ + enabled = true + }] + node_metadata = "EXPOSE" +} + +data "google_client_config" "default" { +} + diff --git a/test/fixtures/beta_cluster/network.tf b/test/fixtures/beta_cluster/network.tf new file mode 100644 index 0000000000..0a3f091958 --- /dev/null +++ b/test/fixtures/beta_cluster/network.tf @@ -0,0 +1,44 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +resource "google_compute_network" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} + diff --git a/test/fixtures/beta_cluster/outputs.tf b/test/fixtures/beta_cluster/outputs.tf new file mode 100644 index 0000000000..ee6192ef36 --- /dev/null +++ b/test/fixtures/beta_cluster/outputs.tf @@ -0,0 +1,84 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = var.project_id +} + +output "region" { + value = module.this.region +} + +output "cluster_name" { + description = "Cluster name" + value = module.this.name +} + +output "network" { + value = google_compute_network.main.name +} + +output "subnetwork" { + value = google_compute_subnetwork.main.name +} + +output "location" { + value = module.this.location +} + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = google_compute_subnetwork.main.secondary_ip_range[0].range_name +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = google_compute_subnetwork.main.secondary_ip_range[1].range_name +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.this.zones +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = module.this.master_version +} + +output "kubernetes_endpoint" { + sensitive = true + value = module.this.endpoint +} + +output "client_token" { + sensitive = true + value = base64encode(data.google_client_config.default.access_token) +} + +output "ca_certificate" { + description = "The cluster CA certificate" + value = module.this.ca_certificate +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.this.service_account +} + +output "database_encryption_key_name" { + value = google_kms_crypto_key.db.self_link +} diff --git a/test/fixtures/beta_cluster/terraform.tfvars b/test/fixtures/beta_cluster/terraform.tfvars new file mode 120000 index 0000000000..0f20d6aff1 --- /dev/null +++ b/test/fixtures/beta_cluster/terraform.tfvars @@ -0,0 +1 @@ +../deploy_service/terraform.tfvars \ No newline at end of file diff --git a/test/fixtures/beta_cluster/variables.tf b/test/fixtures/beta_cluster/variables.tf new file mode 120000 index 0000000000..c28fc18c01 --- /dev/null +++ b/test/fixtures/beta_cluster/variables.tf @@ -0,0 +1 @@ +../deploy_service/variables.tf \ No newline at end of file diff --git a/test/integration/beta_cluster/controls/gcloud.rb b/test/integration/beta_cluster/controls/gcloud.rb new file mode 100644 index 0000000000..c079a50197 --- /dev/null +++ b/test/integration/beta_cluster/controls/gcloud.rb @@ -0,0 +1,204 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') +service_account = attribute('service_account') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud beta --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is zonal" do + expect(data['location']).to match(/^(.*)[1-9]-[a-z]$/) + end + + it "is single zoned" do + expect(data['locations'].size).to eq 1 + end + + it "uses public nodes and master endpoint" do + expect(data['privateClusterConfig']).to eq nil + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => { + "disabled" => true, + }, + "istioConfig" => {}, + "cloudRunConfig" => {}, + }) + end + + it "has the expected binaryAuthorization settings" do + expect(data['binaryAuthorization']).to eq({ + "enabled" => true, + }) + end + + it "has the expected nodeMetadata conseal config" do + expect(data['nodeConfig']['workloadMetadataConfig']).to eq({ + "nodeMetadata" => 'EXPOSE', + }) + end + + it "has the expected podSecurityPolicyConfig settings" do + expect(data['podSecurityPolicyConfig']).to eq({ + "enabled" => true, + }) + end + + it "has the expected databaseEncryption settings" do + expect(data['databaseEncryption']).to eq({ + "state" => 'ENCRYPTED', + "keyName" => attribute('database_encryption_key_name'), + }) + end + end + + describe "default node pool" do + let(:default_node_pool) { data['nodePools'].select { |p| p['name'] == "default-pool" }.first } + + it "has no initial node count" do + expect(default_node_pool['initialNodeCount']).to eq nil + end + + it "does not have autoscaling enabled" do + expect(default_node_pool['autoscaling']).to eq nil + end + end + + describe "node pool" do + let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } + + it "uses an automatically created service account" do + expect(node_pools).to include( + including( + "config" => including( + "serviceAccount" => service_account, + ), + ), + ) + end + + it "has autoscaling enabled" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "enabled" => true, + ), + ) + ) + end + + it "has the expected minimum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "minNodeCount" => 1, + ), + ) + ) + end + + it "has the expected maximum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "maxNodeCount" => 100, + ), + ) + ) + end + + it "is the expected machine type" do + expect(node_pools).to include( + including( + "config" => including( + "machineType" => "n1-standard-2", + ), + ) + ) + end + + it "has the expected disk size" do + expect(node_pools).to include( + including( + "config" => including( + "diskSizeGb" => 100, + ), + ) + ) + end + + it "has the expected labels" do + expect(node_pools).to include( + including( + "config" => including( + "labels" => including( + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + ), + ), + ) + ) + end + + it "has the expected network tags" do + expect(node_pools).to include( + including( + "config" => including( + "tags" => match_array([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]), + ), + ) + ) + end + + it "has autorepair enabled" do + expect(node_pools).to include( + including( + "management" => including( + "autoRepair" => true, + ), + ) + ) + end + end + end +end diff --git a/test/integration/beta_cluster/controls/gcp.rb b/test/integration/beta_cluster/controls/gcp.rb new file mode 100644 index 0000000000..8e4cf6f96c --- /dev/null +++ b/test/integration/beta_cluster/controls/gcp.rb @@ -0,0 +1,31 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +control "gcp" do + title "Native InSpec Resources" + + service_account = attribute("service_account") + project_id = attribute("project_id") + + if service_account.start_with? "projects/" + service_account_name = service_account + else + service_account_name = "projects/#{project_id}/serviceAccounts/#{service_account}" + end + + describe google_service_account name: service_account_name do + its("display_name") { should eq "Terraform-managed service account for cluster #{attribute("cluster_name")}" } + its("project_id") { should eq project_id } + end +end diff --git a/test/integration/beta_cluster/inspec.yml b/test/integration/beta_cluster/inspec.yml new file mode 100644 index 0000000000..66062ea35d --- /dev/null +++ b/test/integration/beta_cluster/inspec.yml @@ -0,0 +1,33 @@ +name: beta_cluster +depends: + - name: inspec-gcp + git: https://github.com/inspec/inspec-gcp.git + tag: v0.10.0 +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: master_kubernetes_version + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string + - name: service_account + required: true + type: string + - name: service_account + required: true + type: string + - name: database_encryption_key_name + required: true + type: string