From 31c5188f50c76a319808d24399497a685b14f229 Mon Sep 17 00:00:00 2001
From: Yecheng Fu <cofyc.jackson@gmail.com>
Date: Thu, 22 Aug 2019 16:33:55 +0800
Subject: [PATCH] support zonal/multi-zonal clusters

---
 deploy/gcp/examples/multi-zonal.tfvars        | 13 ++++
 deploy/gcp/examples/single-zonal.tfvars       | 13 ++++
 deploy/gcp/examples/tidb-customized.tfvars    | 25 ++++++++
 deploy/gcp/main.tf                            |  9 ++-
 deploy/gcp/tidbclusters.tf                    |  8 ++-
 deploy/gcp/variables.tf                       | 60 +++++++++++++++----
 deploy/modules/gcp/tidb-cluster/data.tf       |  4 +-
 deploy/modules/gcp/tidb-cluster/main.tf       |  2 +-
 deploy/modules/gcp/tidb-cluster/variables.tf  |  5 ++
 deploy/modules/gcp/tidb-operator/main.tf      | 16 +++--
 deploy/modules/gcp/tidb-operator/outputs.tf   |  2 +-
 deploy/modules/gcp/tidb-operator/variables.tf | 14 +++--
 12 files changed, 146 insertions(+), 25 deletions(-)
 create mode 100644 deploy/gcp/examples/multi-zonal.tfvars
 create mode 100644 deploy/gcp/examples/single-zonal.tfvars
 create mode 100644 deploy/gcp/examples/tidb-customized.tfvars

diff --git a/deploy/gcp/examples/multi-zonal.tfvars b/deploy/gcp/examples/multi-zonal.tfvars
new file mode 100644
index 00000000000..17aa938cc04
--- /dev/null
+++ b/deploy/gcp/examples/multi-zonal.tfvars
@@ -0,0 +1,13 @@
+#
+# This will create a zonal cluster in zone us-central1-b with one additional zone.
+# Work nodes will be created in primary zone us-central1-b and additional zone us-central1-c.
+#
+gke_name = "multi-zonal"
+vpc_name = "multi-zonal"
+location = "us-central1-b"
+pd_instance_type   = "n1-standard-2"
+tikv_instance_type = "n1-highmem-4"
+tidb_instance_type = "n1-standard-8"
+node_locations = [
+   "us-central1-c"
+]
diff --git a/deploy/gcp/examples/single-zonal.tfvars b/deploy/gcp/examples/single-zonal.tfvars
new file mode 100644
index 00000000000..4b073e6e2c0
--- /dev/null
+++ b/deploy/gcp/examples/single-zonal.tfvars
@@ -0,0 +1,13 @@
+#
+# This will create a zonal cluster in zone us-central1-b without additional zones.
+# Work nodes will be created in a single zone only.
+#
+gke_name = "single-zonal"
+vpc_name = "single-zonal"
+location = "us-central1-b"
+pd_instance_type   = "n1-standard-2"
+tikv_instance_type = "n1-highmem-4"
+tidb_instance_type = "n1-standard-8"
+pd_count = 3
+tikv_count = 3
+tidb_count = 3
diff --git a/deploy/gcp/examples/tidb-customized.tfvars b/deploy/gcp/examples/tidb-customized.tfvars
new file mode 100644
index 00000000000..a1a525ba3e4
--- /dev/null
+++ b/deploy/gcp/examples/tidb-customized.tfvars
@@ -0,0 +1,25 @@
+pd_instance_type   = "n1-standard-2"
+tikv_instance_type = "n1-highmem-4"
+tidb_instance_type = "n1-standard-8"
+
+# specify tidb version
+tidb_version = "3.0.2"
+
+# override tidb cluster values
+override_values = <<EOF
+pd:
+  hostNetwork: true
+tikv:
+  resources:
+     requests:
+       cpu: "1"
+       memory: 1Gi
+       storage: 10Gi
+  hostNetwork: true
+tidb:
+  resources:
+    requests:
+      cpu: "1"
+      memory: 1Gi
+  hostNetwork: true
+EOF
diff --git a/deploy/gcp/main.tf b/deploy/gcp/main.tf
index 2935751a443..6bb309f0863 100644
--- a/deploy/gcp/main.tf
+++ b/deploy/gcp/main.tf
@@ -1,3 +1,8 @@
+locals {
+  # Create a regional cluster in current region by default.
+  location = var.location != "" ? var.location : var.GCP_REGION
+}
+
 provider "google" {
   credentials = file(var.GCP_CREDENTIALS_PATH)
   region      = var.GCP_REGION
@@ -39,10 +44,12 @@ module "tidb-operator" {
   vpc_name                      = var.vpc_name
   subnetwork_name               = module.vpc.private_subnetwork_name
   gcp_project                   = var.GCP_PROJECT
-  gcp_region                    = var.GCP_REGION
+  location                      = local.location
+  node_locations                = var.node_locations
   kubeconfig_path               = local.kubeconfig
   tidb_operator_version         = var.tidb_operator_version
   maintenance_window_start_time = var.maintenance_window_start_time
+  operator_helm_values          = var.operator_helm_values
 }
 
 module "bastion" {
diff --git a/deploy/gcp/tidbclusters.tf b/deploy/gcp/tidbclusters.tf
index 6588e6ab831..7485786021d 100644
--- a/deploy/gcp/tidbclusters.tf
+++ b/deploy/gcp/tidbclusters.tf
@@ -2,6 +2,7 @@ data "local_file" "kubeconfig" {
   depends_on = [module.tidb-operator.cluster_id]
   filename   = module.tidb-operator.kubeconfig_path
 }
+
 resource "local_file" "kubeconfig" {
   depends_on = [module.tidb-operator.cluster_id]
   content    = data.local_file.kubeconfig.content
@@ -26,14 +27,19 @@ module "default-tidb-cluster" {
   gke_cluster_name           = module.tidb-operator.gke_cluster_name
   cluster_name               = var.default_tidb_cluster_name
   cluster_version            = var.tidb_version
-  kubeconfig_path            = module.tidb-operator.kubeconfig_path
+  kubeconfig_path            = local_file.kubeconfig.filename
   tidb_cluster_chart_version = coalesce(var.tidb_operator_chart_version, var.tidb_operator_version)
   pd_instance_type           = var.pd_instance_type
   tikv_instance_type         = var.tikv_instance_type
   tidb_instance_type         = var.tidb_instance_type
+  pd_image_type              = var.pd_image_type
+  tikv_image_type            = var.tikv_image_type
+  tidb_image_type            = var.tidb_image_type
   monitor_instance_type      = var.monitor_instance_type
   pd_node_count              = var.pd_count
   tikv_node_count            = var.tikv_count
   tidb_node_count            = var.tidb_count
   monitor_node_count         = var.monitor_count
+  tikv_local_ssd_count 	     = var.tikv_local_ssd_count
+  override_values            = var.override_values
 }
diff --git a/deploy/gcp/variables.tf b/deploy/gcp/variables.tf
index e0f25ccbba1..fcb6fb5c2cc 100644
--- a/deploy/gcp/variables.tf
+++ b/deploy/gcp/variables.tf
@@ -24,6 +24,12 @@ variable "tidb_operator_chart_version" {
   default     = ""
 }
 
+variable "operator_helm_values" {
+  description = "Operator helm values"
+  type        = string
+  default     = ""
+}
+
 variable "create_vpc" {
   default = true
 }
@@ -33,26 +39,31 @@ variable "gke_name" {
   default     = "tidb-cluster"
 }
 
-variable "default_tidb_cluster_name" {
-  description = "The name that will be given to the default tidb cluster created."
-  default     = "tidb-cluster"
+variable "location" {
+  description = "The GKE cluster location. If you specify a zone (such as us-central1-a), the cluster will be a zonal cluster with a single cluster master. If you specify a region (such as us-west1), the cluster will be a regional cluster with multiple masters spread across zones in the region. If not specified, the cluster will be a regional cluster in GCP_REGION."
+  type        = string
 }
 
-variable "vpc_name" {
-  description = "The name of the VPC network"
-  default     = "tidb-cluster"
+variable "node_locations" {
+  description = "The list of zones in which the cluster's nodes should be located. These must be in the same region as the cluster zone for zonal clusters, or in the region of a regional cluster. In a multi-zonal cluster, the number of nodes specified in initial_node_count is created in all specified zones as well as the primary zone. If specified for a regional cluster, nodes will be created in only these zones."
+  type        = list(string)
+  default = []
 }
 
-variable "pd_replica_count" {
-  default = 3
+variable "gke_version" {
+  description = "Kubernetes version to use for the GKE cluster"
+  type        = string
+  default     = "latest"
 }
 
-variable "tikv_replica_count" {
-  default = 3
+variable "default_tidb_cluster_name" {
+  description = "The name that will be given to the default tidb cluster created."
+  default     = "tidb-cluster"
 }
 
-variable "tidb_replica_count" {
-  default = 3
+variable "vpc_name" {
+  description = "The name of the VPC network"
+  default     = "tidb-cluster"
 }
 
 variable "pd_count" {
@@ -80,6 +91,26 @@ variable "tikv_instance_type" {}
 
 variable "tidb_instance_type" {}
 
+variable "pd_image_type" {
+  description = "PD image type, avaiable: UBUNTU/COS"
+  default     = "COS"
+}
+
+variable "tidb_image_type" {
+  description = "TiDB image type, avaiable: UBUNTU/COS"
+  default     = "COS"
+}
+
+variable "tikv_image_type" {
+  description = "TiKV image type, avaiable: UBUNTU/COS"
+  default     = "COS"
+}
+
+variable "tikv_local_ssd_count" {
+  description = "TiKV node pool local ssd count (cannot be changed after the node pool is created)"
+  default = 1
+}
+
 variable "monitor_instance_type" {
   default = "n1-standard-2"
 }
@@ -92,3 +123,8 @@ variable "maintenance_window_start_time" {
   description = "The time in HH:MM GMT format to define the start of the daily maintenance window"
   default     = "01:00"
 }
+
+variable "override_values" {
+  description = "YAML formatted values that will be passed in to the tidb-cluster helm release"
+  default     = ""
+}
diff --git a/deploy/modules/gcp/tidb-cluster/data.tf b/deploy/modules/gcp/tidb-cluster/data.tf
index 0efb7629ba7..8bf3a18c34a 100644
--- a/deploy/modules/gcp/tidb-cluster/data.tf
+++ b/deploy/modules/gcp/tidb-cluster/data.tf
@@ -19,9 +19,11 @@ data "external" "monitor_port" {
 }
 
 locals {
+  # Location examples: us-central1 (region), us-central1-b (zone), us-central1-c (zone)
+  cluster_location_args = "%{if length(split("-", var.gke_cluster_location)) == 3}--zone ${var.gke_cluster_location}%{else}--region ${var.gke_cluster_location}%{endif}"
   # TODO Update related code when node locations is avaiable in attributes of cluster resource.
   cmd_get_cluster_locations = <<EOT
-gcloud --project ${var.gcp_project} container clusters list --filter='name=${var.gke_cluster_name}' --format='json[no-heading](locations)' --region ${var.gke_cluster_location} | jq '.[0] | .locations |= join(",")'
+gcloud --project ${var.gcp_project} container clusters list --filter='name=${var.gke_cluster_name}' --format='json[no-heading](locations)' ${local.cluster_location_args} | jq '.[0] | .locations |= join(",")'
 EOT
 }
 
diff --git a/deploy/modules/gcp/tidb-cluster/main.tf b/deploy/modules/gcp/tidb-cluster/main.tf
index e7c0bfef78e..e3588ffac89 100644
--- a/deploy/modules/gcp/tidb-cluster/main.tf
+++ b/deploy/modules/gcp/tidb-cluster/main.tf
@@ -51,7 +51,7 @@ resource "google_container_node_pool" "tikv_pool" {
     image_type   = var.tikv_image_type
     // This value cannot be changed (instead a new node pool is needed)
     // 1 SSD is 375 GiB
-    local_ssd_count = 1
+    local_ssd_count = var.tikv_local_ssd_count
 
     taint {
       effect = "NO_SCHEDULE"
diff --git a/deploy/modules/gcp/tidb-cluster/variables.tf b/deploy/modules/gcp/tidb-cluster/variables.tf
index 3087da68342..c8c37c9ccc1 100644
--- a/deploy/modules/gcp/tidb-cluster/variables.tf
+++ b/deploy/modules/gcp/tidb-cluster/variables.tf
@@ -59,3 +59,8 @@ variable "tikv_image_type" {
   description = "TiKV image type, avaiable: UBUNTU/COS"
   default = "COS"
 }
+
+variable "tikv_local_ssd_count" {
+  description = "TiKV node pool local ssd count (cannot be changed after the node pool is created)"
+  default = 1
+}
diff --git a/deploy/modules/gcp/tidb-operator/main.tf b/deploy/modules/gcp/tidb-operator/main.tf
index 82b164f8e93..c879c0040b3 100644
--- a/deploy/modules/gcp/tidb-operator/main.tf
+++ b/deploy/modules/gcp/tidb-operator/main.tf
@@ -2,7 +2,8 @@ resource "google_container_cluster" "cluster" {
   name       = var.gke_name
   network    = var.vpc_name
   subnetwork = var.subnetwork_name
-  location   = var.gcp_region
+  location   = var.location
+  node_locations = var.node_locations
   project    = var.gcp_project
 
   master_auth {
@@ -42,11 +43,18 @@ resource "google_container_cluster" "cluster" {
   }
 }
 
+locals {
+   # TODO better way to detect whether the cluster is zonal or regional
+   cmd_get_cluster_credentials = length(split("-", var.location)) == 3 ?  "gcloud --project ${var.gcp_project} container clusters get-credentials ${google_container_cluster.cluster.name} --zone ${var.location}" : "gcloud --project ${var.gcp_project} container clusters get-credentials ${google_container_cluster.cluster.name} --region ${var.location}"
+}
+
 resource "null_resource" "get-credentials" {
   depends_on = [google_container_cluster.cluster]
+  triggers = {
+    command = local.cmd_get_cluster_credentials
+  }
   provisioner "local-exec" {
-    command = "gcloud --project ${var.gcp_project} container clusters get-credentials ${google_container_cluster.cluster.name} --region ${var.gcp_region}"
-
+    command = local.cmd_get_cluster_credentials
     environment = {
       KUBECONFIG = var.kubeconfig_path
     }
@@ -70,7 +78,7 @@ provider "helm" {
   # service_account = "tiller"
   install_tiller = false # currently this doesn't work, so we install tiller in the local-exec provisioner. See https://github.com/terraform-providers/terraform-provider-helm/issues/148
   kubernetes {
-    config_path = local_file.kubeconfig.filename
+    config_path = var.kubeconfig_path
   }
 }
 
diff --git a/deploy/modules/gcp/tidb-operator/outputs.tf b/deploy/modules/gcp/tidb-operator/outputs.tf
index 84a9d10740f..61f8dbcdc4c 100644
--- a/deploy/modules/gcp/tidb-operator/outputs.tf
+++ b/deploy/modules/gcp/tidb-operator/outputs.tf
@@ -15,5 +15,5 @@ output "gke_cluster_location" {
 }
 
 output "kubeconfig_path" {
-  value = local_file.kubeconfig.filename
+  value = data.local_file.kubeconfig.filename
 }
diff --git a/deploy/modules/gcp/tidb-operator/variables.tf b/deploy/modules/gcp/tidb-operator/variables.tf
index a94e2c3212e..f4a11a37060 100644
--- a/deploy/modules/gcp/tidb-operator/variables.tf
+++ b/deploy/modules/gcp/tidb-operator/variables.tf
@@ -11,14 +11,20 @@ variable "subnetwork_name" {
   description = "The name of the subnetwork in which to place the cluster"
 }
 
-variable "gcp_region" {
-  description = "The GCP region"
-}
-
 variable "gcp_project" {
   description = "The GCP project name"
 }
 
+variable "location" {
+  description = "The GKE cluster location. If you specify a zone (such as us-central1-a), the cluster will be a zonal cluster with a single cluster master. If you specify a region (such as us-west1), the cluster will be a regional cluster with multiple masters spread across zones in the region."
+  type = string
+}
+
+variable "node_locations" {
+  description = "The list of zones in which the cluster's nodes should be located. These must be in the same region as the cluster zone for zonal clusters, or in the region of a regional cluster. In a multi-zonal cluster, the number of nodes specified in initial_node_count is created in all specified zones as well as the primary zone. If specified for a regional cluster, nodes will be created in only these zones."
+  type = list(string)
+}
+
 variable "gke_version" {
   description = "Kubernetes version to use for the GKE cluster"
   type        = string