diff --git a/CHANGELOG.md b/CHANGELOG.md index d75c2ce5b7..aca3cd5f8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,8 +20,9 @@ Extending the adopted spec, each change should have a link to its corresponding * `simple_regional_with_networking` example. [#195] * `release_channel` variable for beta submodules. [#271] * The `node_locations` attribute to the `node_pools` object for beta submodules. [#290] -* `private_zonal_with_nteworking` example. [#308] +* `private_zonal_with_networking` example. [#308] * `regional_private_node_pool_oauth_scopes` example. [#321] +* The `cluster_autoscaling` variable for beta submodules. [#93] ### Changed @@ -314,6 +315,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [#108]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/108 [#106]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/106 [#94]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/94 +[#93]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/93 [#89]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/89 [#80]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/80 [#77]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/77 diff --git a/README.md b/README.md index 4c15b0f311..1cebf678c5 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | | network | The VPC network to host the cluster in (required) | string | n/a | yes | diff --git a/autogen/cluster.tf.tmpl b/autogen/cluster.tf.tmpl index ffb565ff21..dc19ed2d90 100644 --- a/autogen/cluster.tf.tmpl +++ b/autogen/cluster.tf.tmpl @@ -62,6 +62,18 @@ resource "google_container_cluster" "primary" { monitoring_service = var.monitoring_service {% if beta_cluster %} + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility default_max_pods_per_node = var.default_max_pods_per_node @@ -89,7 +101,7 @@ resource "google_container_cluster" "primary" { } {% endif %} dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config + for_each = local.master_authorized_networks_config content { dynamic "cidr_blocks" { for_each = master_authorized_networks_config.value.cidr_blocks diff --git a/autogen/main.tf.tmpl b/autogen/main.tf.tmpl index 11e0ec224f..3ac28cc16a 100644 --- a/autogen/main.tf.tmpl +++ b/autogen/main.tf.tmpl @@ -50,6 +50,17 @@ locals { node_version = var.regional ? local.node_version_regional : local.node_version_zonal {% if beta_cluster %} release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + + autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{ + resource_type = "cpu" + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores + }, { + resource_type = "memory" + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb + }] : [] + {% endif %} @@ -116,6 +127,10 @@ locals { # /BETA features {% endif %} + master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ + cidr_blocks : var.master_authorized_networks + }] + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) diff --git a/autogen/outputs.tf.tmpl b/autogen/outputs.tf.tmpl index 3db3358196..4a2df6f198 100644 --- a/autogen/outputs.tf.tmpl +++ b/autogen/outputs.tf.tmpl @@ -75,7 +75,7 @@ output "monitoring_service" { output "master_authorized_networks_config" { description = "Networks from which access to master is permitted" - value = var.master_authorized_networks_config + value = google_container_cluster.primary.master_authorized_networks_config } output "master_version" { diff --git a/autogen/variables.tf.tmpl b/autogen/variables.tf.tmpl index 5b87f08e05..3295199e48 100644 --- a/autogen/variables.tf.tmpl +++ b/autogen/variables.tf.tmpl @@ -78,9 +78,9 @@ variable "node_version" { default = "" } -variable "master_authorized_networks_config" { - type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) - description = "The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } @@ -174,8 +174,26 @@ variable "node_pools_metadata" { default-node-pool = {} } } - {% if beta_cluster %} + +variable "cluster_autoscaling" { + type = object({ + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + }) + default = { + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" diff --git a/cluster.tf b/cluster.tf index 70d932ccfe..573b4cc5d8 100644 --- a/cluster.tf +++ b/cluster.tf @@ -49,7 +49,7 @@ resource "google_container_cluster" "primary" { monitoring_service = var.monitoring_service dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config + for_each = local.master_authorized_networks_config content { dynamic "cidr_blocks" { for_each = master_authorized_networks_config.value.cidr_blocks diff --git a/docs/upgrading_to_v6.0.md b/docs/upgrading_to_v6.0.md new file mode 100644 index 0000000000..b7e9b43374 --- /dev/null +++ b/docs/upgrading_to_v6.0.md @@ -0,0 +1,37 @@ +# Upgrading to v6.0 + +The v6.0 release of *kubernetes-engine* is a backwards incompatible +release. + +## Dropped support +Due to changes in GKE, the module has dropped support for setting the `kubernetes_dashboard` variable. + +Additionally, support for Google provider versions older than v2.18 has been removed. + +## Migration Instructions + +### Master Authorized Networks +Previously, setting up master authorized networks required setting a nested config within `master_authorized_networks_config`. +Now, to set up master authorized networks you can simply pass a list of authorized networks. + +```diff + module "kubernetes_engine_private_cluster" { + source = "terraform-google-modules/kubernetes-engine/google" +- version = "~> 5.0" ++ version = "~> 6.0" + +- master_authorized_networks_config = [ ++ master_authorized_networks = [ + { +- cidr_blocks = [ +- { +- cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range +- display_name = "VPC" +- }, +- ] ++ cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range ++ display_name = "VPC" + }, + ] + } +``` diff --git a/examples/node_pool/README.md b/examples/node_pool/README.md index 237b3f0b6f..7b8867b9c8 100644 --- a/examples/node_pool/README.md +++ b/examples/node_pool/README.md @@ -7,6 +7,7 @@ This example illustrates how to create a cluster with multiple custom node-pool | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index b2f1d010ed..15a7c124ae 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -36,6 +36,7 @@ module "gke" { create_service_account = false remove_default_node_pool = true disable_legacy_metadata_endpoints = false + cluster_autoscaling = var.cluster_autoscaling node_pools = [ { diff --git a/examples/node_pool/variables.tf b/examples/node_pool/variables.tf index 040c78d2c4..49570f241f 100644 --- a/examples/node_pool/variables.tf +++ b/examples/node_pool/variables.tf @@ -52,3 +52,20 @@ variable "compute_engine_service_account" { description = "Service account to associate to the nodes in the cluster" } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + }) + default = { + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} diff --git a/examples/node_pool_update_variant/main.tf b/examples/node_pool_update_variant/main.tf index 9b29a5f0fe..d6f7f270b7 100644 --- a/examples/node_pool_update_variant/main.tf +++ b/examples/node_pool_update_variant/main.tf @@ -46,14 +46,10 @@ module "gke" { enable_private_nodes = true master_ipv4_cidr_block = "172.16.0.0/28" - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range - display_name = "VPC" - }, - ] + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" }, ] diff --git a/examples/node_pool_update_variant_beta/main.tf b/examples/node_pool_update_variant_beta/main.tf index 37b595f793..a127e9aae2 100644 --- a/examples/node_pool_update_variant_beta/main.tf +++ b/examples/node_pool_update_variant_beta/main.tf @@ -47,14 +47,10 @@ module "gke" { enable_private_nodes = true master_ipv4_cidr_block = "172.16.0.0/28" - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range - display_name = "VPC" - }, - ] + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" }, ] diff --git a/examples/private_zonal_with_networking/main.tf b/examples/private_zonal_with_networking/main.tf index 82db39a591..b7bb59c9d2 100644 --- a/examples/private_zonal_with_networking/main.tf +++ b/examples/private_zonal_with_networking/main.tf @@ -71,14 +71,10 @@ module "gke" { enable_private_nodes = true master_ipv4_cidr_block = "172.16.0.0/28" - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range - display_name = "VPC" - }, - ] + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" }, ] } diff --git a/examples/regional_private_node_pool_oauth_scopes/main.tf b/examples/regional_private_node_pool_oauth_scopes/main.tf index e39e3299f6..b843cb7234 100644 --- a/examples/regional_private_node_pool_oauth_scopes/main.tf +++ b/examples/regional_private_node_pool_oauth_scopes/main.tf @@ -33,14 +33,10 @@ module "gke" { remove_default_node_pool = true disable_legacy_metadata_endpoints = true - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = module.gke-network.subnets_ips[0] - display_name = "VPC" - }, - ] + cidr_block = module.gke-network.subnets_ips[0] + display_name = "VPC" }, ] diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf index 0aeca18009..a69cc3ddbb 100644 --- a/examples/safer_cluster/main.tf +++ b/examples/safer_cluster/main.tf @@ -49,16 +49,14 @@ module "gke" { ip_range_services = local.svc_range_name compute_engine_service_account = var.compute_engine_service_account master_ipv4_cidr_block = "172.16.0.0/28" - master_authorized_networks_config = [ + + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = "10.60.0.0/17" - display_name = "VPC" - }, - ] + cidr_block = "10.60.0.0/17" + display_name = "VPC" }, ] + istio = true cloudrun = true } diff --git a/examples/simple_regional_private/main.tf b/examples/simple_regional_private/main.tf index f17a3728a6..10a55795df 100644 --- a/examples/simple_regional_private/main.tf +++ b/examples/simple_regional_private/main.tf @@ -45,14 +45,10 @@ module "gke" { enable_private_nodes = true master_ipv4_cidr_block = "172.16.0.0/28" - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range - display_name = "VPC" - }, - ] + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" }, ] } diff --git a/examples/simple_regional_private_beta/main.tf b/examples/simple_regional_private_beta/main.tf index 4e1d405940..eec167a3e4 100644 --- a/examples/simple_regional_private_beta/main.tf +++ b/examples/simple_regional_private_beta/main.tf @@ -44,14 +44,10 @@ module "gke" { enable_private_nodes = true master_ipv4_cidr_block = "172.16.0.0/28" - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range - display_name = "VPC" - }, - ] + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" }, ] diff --git a/examples/simple_zonal_private/main.tf b/examples/simple_zonal_private/main.tf index ae1a90a6cc..0b2a3c4180 100644 --- a/examples/simple_zonal_private/main.tf +++ b/examples/simple_zonal_private/main.tf @@ -46,14 +46,10 @@ module "gke" { enable_private_nodes = true master_ipv4_cidr_block = "172.16.0.0/28" - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range - display_name = "VPC" - }, - ] + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" }, ] } diff --git a/examples/stub_domains_private/main.tf b/examples/stub_domains_private/main.tf index 3f268e75a4..b263922b2a 100644 --- a/examples/stub_domains_private/main.tf +++ b/examples/stub_domains_private/main.tf @@ -40,14 +40,10 @@ module "gke" { enable_private_endpoint = false enable_private_nodes = true - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range - display_name = "VPC" - }, - ] + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" }, ] diff --git a/examples/workload_metadata_config/main.tf b/examples/workload_metadata_config/main.tf index 3d2254c2da..fbcfc949ae 100644 --- a/examples/workload_metadata_config/main.tf +++ b/examples/workload_metadata_config/main.tf @@ -48,14 +48,10 @@ module "gke" { master_ipv4_cidr_block = "172.16.0.0/28" node_metadata = "SECURE" - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range - display_name = "VPC" - }, - ] + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" }, ] } diff --git a/main.tf b/main.tf index 754fcefe04..409cbfe55b 100644 --- a/main.tf +++ b/main.tf @@ -82,6 +82,10 @@ locals { cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ + cidr_blocks : var.master_authorized_networks + }] + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 22227f6a1e..6df69df5bd 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -141,6 +141,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | @@ -170,7 +171,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 3d0c0eac34..10fcf764a3 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -55,6 +55,18 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility default_max_pods_per_node = var.default_max_pods_per_node @@ -81,7 +93,7 @@ resource "google_container_cluster" "primary" { } } dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config + for_each = local.master_authorized_networks_config content { dynamic "cidr_blocks" { for_each = master_authorized_networks_config.value.cidr_blocks diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index c0a2f6c892..9afc2502b8 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -46,6 +46,17 @@ locals { node_version = var.regional ? local.node_version_regional : local.node_version_zonal release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{ + resource_type = "cpu" + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores + }, { + resource_type = "memory" + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb + }] : [] + + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 @@ -102,6 +113,10 @@ locals { # /BETA features + master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ + cidr_blocks : var.master_authorized_networks + }] + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) diff --git a/modules/beta-private-cluster-update-variant/outputs.tf b/modules/beta-private-cluster-update-variant/outputs.tf index 32e37ef5a3..8ea1ded043 100644 --- a/modules/beta-private-cluster-update-variant/outputs.tf +++ b/modules/beta-private-cluster-update-variant/outputs.tf @@ -75,7 +75,7 @@ output "monitoring_service" { output "master_authorized_networks_config" { description = "Networks from which access to master is permitted" - value = var.master_authorized_networks_config + value = google_container_cluster.primary.master_authorized_networks_config } output "master_version" { diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 0f581e56cc..62c47d002d 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -78,9 +78,9 @@ variable "node_version" { default = "" } -variable "master_authorized_networks_config" { - type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) - description = "The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } @@ -175,6 +175,24 @@ variable "node_pools_metadata" { } } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + }) + default = { + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index c5c6b079ef..b03a4ea921 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -141,6 +141,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | @@ -170,7 +171,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 5b75a98b2f..f2789bfd03 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -55,6 +55,18 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility default_max_pods_per_node = var.default_max_pods_per_node @@ -81,7 +93,7 @@ resource "google_container_cluster" "primary" { } } dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config + for_each = local.master_authorized_networks_config content { dynamic "cidr_blocks" { for_each = master_authorized_networks_config.value.cidr_blocks diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index c0a2f6c892..9afc2502b8 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -46,6 +46,17 @@ locals { node_version = var.regional ? local.node_version_regional : local.node_version_zonal release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{ + resource_type = "cpu" + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores + }, { + resource_type = "memory" + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb + }] : [] + + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 @@ -102,6 +113,10 @@ locals { # /BETA features + master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ + cidr_blocks : var.master_authorized_networks + }] + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf index 32e37ef5a3..8ea1ded043 100644 --- a/modules/beta-private-cluster/outputs.tf +++ b/modules/beta-private-cluster/outputs.tf @@ -75,7 +75,7 @@ output "monitoring_service" { output "master_authorized_networks_config" { description = "Networks from which access to master is permitted" - value = var.master_authorized_networks_config + value = google_container_cluster.primary.master_authorized_networks_config } output "master_version" { diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 0f581e56cc..62c47d002d 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -78,9 +78,9 @@ variable "node_version" { default = "" } -variable "master_authorized_networks_config" { - type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) - description = "The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } @@ -175,6 +175,24 @@ variable "node_pools_metadata" { } } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + }) + default = { + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index b0e2cfc85d..b6ab1f39d3 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -136,6 +136,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | @@ -162,7 +163,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | | network | The VPC network to host the cluster in (required) | string | n/a | yes | diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index e675d41c23..5867718699 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -55,6 +55,18 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility default_max_pods_per_node = var.default_max_pods_per_node @@ -81,7 +93,7 @@ resource "google_container_cluster" "primary" { } } dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config + for_each = local.master_authorized_networks_config content { dynamic "cidr_blocks" { for_each = master_authorized_networks_config.value.cidr_blocks diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 84e43e33fb..5cff8bdd4e 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -46,6 +46,17 @@ locals { node_version = var.regional ? local.node_version_regional : local.node_version_zonal release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{ + resource_type = "cpu" + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores + }, { + resource_type = "memory" + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb + }] : [] + + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 upstream_nameservers_config = length(var.upstream_nameservers) > 0 @@ -102,6 +113,10 @@ locals { # /BETA features + master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ + cidr_blocks : var.master_authorized_networks + }] + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf index 32e37ef5a3..8ea1ded043 100644 --- a/modules/beta-public-cluster/outputs.tf +++ b/modules/beta-public-cluster/outputs.tf @@ -75,7 +75,7 @@ output "monitoring_service" { output "master_authorized_networks_config" { description = "Networks from which access to master is permitted" - value = var.master_authorized_networks_config + value = google_container_cluster.primary.master_authorized_networks_config } output "master_version" { diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index e27a7d0414..1a1b9c54b4 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -78,9 +78,9 @@ variable "node_version" { default = "" } -variable "master_authorized_networks_config" { - type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) - description = "The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } @@ -175,6 +175,24 @@ variable "node_pools_metadata" { } } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + }) + default = { + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 1786d3b46c..8d26153072 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -157,7 +157,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 27653ab689..7027d766ce 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -49,7 +49,7 @@ resource "google_container_cluster" "primary" { monitoring_service = var.monitoring_service dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config + for_each = local.master_authorized_networks_config content { dynamic "cidr_blocks" { for_each = master_authorized_networks_config.value.cidr_blocks diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index 7826dfff18..7dea99f25c 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -82,6 +82,10 @@ locals { cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ + cidr_blocks : var.master_authorized_networks + }] + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) diff --git a/modules/private-cluster-update-variant/outputs.tf b/modules/private-cluster-update-variant/outputs.tf index 0a4e6c9ada..6d5e1e3909 100644 --- a/modules/private-cluster-update-variant/outputs.tf +++ b/modules/private-cluster-update-variant/outputs.tf @@ -75,7 +75,7 @@ output "monitoring_service" { output "master_authorized_networks_config" { description = "Networks from which access to master is permitted" - value = var.master_authorized_networks_config + value = google_container_cluster.primary.master_authorized_networks_config } output "master_version" { diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 82c268aecf..9c75edb5ec 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -78,9 +78,9 @@ variable "node_version" { default = "" } -variable "master_authorized_networks_config" { - type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) - description = "The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } @@ -174,7 +174,6 @@ variable "node_pools_metadata" { default-node-pool = {} } } - variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 0f5c3df935..60fbe8de76 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -157,7 +157,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 47b50f6db2..acb6f29a68 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -49,7 +49,7 @@ resource "google_container_cluster" "primary" { monitoring_service = var.monitoring_service dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config + for_each = local.master_authorized_networks_config content { dynamic "cidr_blocks" { for_each = master_authorized_networks_config.value.cidr_blocks diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index 7826dfff18..7dea99f25c 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -82,6 +82,10 @@ locals { cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ + cidr_blocks : var.master_authorized_networks + }] + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) diff --git a/modules/private-cluster/outputs.tf b/modules/private-cluster/outputs.tf index 0a4e6c9ada..6d5e1e3909 100644 --- a/modules/private-cluster/outputs.tf +++ b/modules/private-cluster/outputs.tf @@ -75,7 +75,7 @@ output "monitoring_service" { output "master_authorized_networks_config" { description = "Networks from which access to master is permitted" - value = var.master_authorized_networks_config + value = google_container_cluster.primary.master_authorized_networks_config } output "master_version" { diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 82c268aecf..9c75edb5ec 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -78,9 +78,9 @@ variable "node_version" { default = "" } -variable "master_authorized_networks_config" { - type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) - description = "The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } @@ -174,7 +174,6 @@ variable "node_pools_metadata" { default-node-pool = {} } } - variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index 2d99d26f36..06e81e12c9 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -221,7 +221,7 @@ For simplicity, we suggest using `roles/container.admin` and | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | Additional CIDR of private networks that can access the master. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. By default, the private master endpoint is accessible by the nodes in the cluster's VPC and by Google's internal production jobs managing the cluster. | object | `` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster | string | n/a | yes | diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf index 76c40b3ebe..4b2c00109d 100644 --- a/modules/safer-cluster/main.tf +++ b/modules/safer-cluster/main.tf @@ -36,7 +36,7 @@ module "gke" { // https://cloud.google.com/kubernetes-engine/versioning-and-upgrades node_version = "" - master_authorized_networks_config = var.master_authorized_networks_config + master_authorized_networks = var.master_authorized_networks subnetwork = var.subnetwork ip_range_pods = var.ip_range_pods diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index fe47be7be6..40fa320dc6 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -77,9 +77,9 @@ variable "node_version" { default = "" } -variable "master_authorized_networks_config" { - type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) - description = "Additional CIDR of private networks that can access the master. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. By default, the private master endpoint is accessible by the nodes in the cluster's VPC and by Google's internal production jobs managing the cluster." +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } diff --git a/outputs.tf b/outputs.tf index 0a4e6c9ada..6d5e1e3909 100644 --- a/outputs.tf +++ b/outputs.tf @@ -75,7 +75,7 @@ output "monitoring_service" { output "master_authorized_networks_config" { description = "Networks from which access to master is permitted" - value = var.master_authorized_networks_config + value = google_container_cluster.primary.master_authorized_networks_config } output "master_version" { diff --git a/test/fixtures/node_pool/example.tf b/test/fixtures/node_pool/example.tf index 82dd01035c..8c787f94d6 100644 --- a/test/fixtures/node_pool/example.tf +++ b/test/fixtures/node_pool/example.tf @@ -26,5 +26,13 @@ module "example" { ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name compute_engine_service_account = var.compute_engine_service_accounts[0] + + cluster_autoscaling = { + enabled = true + max_cpu_cores = 20 + min_cpu_cores = 5 + max_memory_gb = 30 + min_memory_gb = 10 + } } diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb index 69a15e8293..c08b61f1c6 100644 --- a/test/integration/node_pool/controls/gcloud.rb +++ b/test/integration/node_pool/controls/gcloud.rb @@ -33,6 +33,30 @@ end end + describe "cluster-autoscaling" do + it "has the expected cluster autoscaling settings" do + expect(data['autoscaling']).to eq({ + "autoprovisioningNodePoolDefaults" => { + "oauthScopes" => %w(https://www.googleapis.com/auth/logging.write https://www.googleapis.com/auth/monitoring), + "serviceAccount" => "default" + }, + "enableNodeAutoprovisioning" => true, + "resourceLimits" => [ + { + "maximum" => "20", + "minimum" => "5", + "resourceType" => "cpu" + }, + { + "maximum" => "30", + "minimum" => "10", + "resourceType" => "memory" + } + ] + }) + end + end + describe "node pools" do let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } diff --git a/test/setup/make_source.sh b/test/setup/make_source.sh deleted file mode 100755 index 4af7cd63cd..0000000000 --- a/test/setup/make_source.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -echo "#!/usr/bin/env bash" > ../source.sh - -project_ids=$(terraform output project_ids) -echo "export TF_VAR_project_ids='$project_ids'" >> ../source.sh - -registry_project_id=$(terraform output registry_project_id) -echo "export TF_VAR_registry_project_id='$registry_project_id'" >> ../source.sh - -sa_json=$(terraform output sa_key) -# shellcheck disable=SC2086 -echo "export SERVICE_ACCOUNT_JSON='$(echo $sa_json | base64 --decode)'" >> ../source.sh - -compute_engine_service_accounts=$(terraform output compute_engine_service_accounts) -echo "export TF_VAR_compute_engine_service_accounts='$compute_engine_service_accounts'" >> ../source.sh diff --git a/variables.tf b/variables.tf index b9fdf45738..904cd6ddab 100644 --- a/variables.tf +++ b/variables.tf @@ -78,9 +78,9 @@ variable "node_version" { default = "" } -variable "master_authorized_networks_config" { - type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) - description = "The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } @@ -174,7 +174,6 @@ variable "node_pools_metadata" { default-node-pool = {} } } - variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name"