diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 9124f74..9d631bb 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -13,7 +13,7 @@ locals { example = "basic" project_name = "tf-${substr(md5(join("-", [local.example, md5(local.identifier)])), 0, 5)}-${local.identifier}" username = substr(lower("tf-${local.identifier}"), 0, 32) - image = "sles-15" + image = "sle-micro-60" ip = chomp(data.http.myip.response_body) ssh_key = var.key rke2_version = "stable" @@ -38,7 +38,7 @@ resource "random_pet" "server" { module "access" { source = "rancher/access/aws" - version = "v3.0.1" + version = "v3.1.5" vpc_name = "${local.project_name}-vpc" vpc_public = true security_group_name = "${local.project_name}-sg" @@ -51,7 +51,7 @@ module "server" { module.access, ] source = "rancher/server/aws" - version = "v1.1.0" + version = "v1.3.1" image_type = local.image server_name = "${local.project_name}-${random_pet.server.id}" server_type = "small" @@ -61,9 +61,10 @@ module "server" { cloudinit_use_strategy = "default" # use the default cloudinit config server_access_addresses = { # you must include ssh access here to enable setup "runner" = { - port = 22 - protocol = "tcp" - cidrs = ["${local.ip}/32"] + port = 22 + protocol = "tcp" + cidrs = ["${local.ip}/32"] + ip_family = "ipv4" } } server_user = { @@ -89,7 +90,7 @@ module "config" { module.download, ] source = "rancher/rke2-config/local" - version = "v0.1.3" + version = "v0.1.4" local_file_path = local.local_file_path } diff --git a/examples/manifest/main.tf b/examples/manifest/main.tf new file mode 100644 index 0000000..901c80f --- /dev/null +++ b/examples/manifest/main.tf @@ -0,0 +1,121 @@ +provider "aws" { + default_tags { + tags = { + Id = local.identifier + Owner = local.email + } + } +} + +locals { + identifier = var.identifier # this is a random unique string that can be used to identify resources in the cloud provider + email = "terraform-ci@suse.com" + example = "manifest" + project_name = "tf-${substr(md5(join("-", [local.example, md5(local.identifier)])), 0, 5)}-${local.identifier}" + username = substr(lower("tf-${local.identifier}"), 0, 32) + image = "sle-micro-60" + ip = chomp(data.http.myip.response_body) + ssh_key = var.key + rke2_version = "stable" + local_file_path = "${path.root}/data/${local.identifier}" +} + +data "http" "myip" { + url = "https://ipinfo.io/ip" + retry { + attempts = 2 + min_delay_ms = 1000 + } +} + +resource "random_pet" "server" { + keepers = { + # regenerate the pet name when the identifier changes + identifier = local.identifier + } + length = 1 +} + +module "access" { + source = "rancher/access/aws" + version = "v3.1.5" + vpc_name = "${local.project_name}-vpc" + vpc_public = true + security_group_name = "${local.project_name}-sg" + security_group_type = "egress" + load_balancer_use_strategy = "skip" +} + +module "server" { + depends_on = [ + module.access, + ] + source = "rancher/server/aws" + version = "v1.3.1" + image_type = local.image + server_name = "${local.project_name}-${random_pet.server.id}" + server_type = "small" + subnet_name = keys(module.access.subnets)[0] + security_group_name = module.access.security_group.tags_all.Name + direct_access_use_strategy = "ssh" # either the subnet needs to be public or you must add an eip + cloudinit_use_strategy = "default" # use the default cloudinit config + server_access_addresses = { # you must include ssh access here to enable setup + "runner" = { + port = 22 + protocol = "tcp" + cidrs = ["${local.ip}/32"] + ip_family = "ipv4" + } + } + server_user = { + user = local.username + aws_keypair_use_strategy = "skip" # we will use cloud-init to add a keypair directly + ssh_key_name = "" # not creating or selecting a key, but this field is still required + public_ssh_key = local.ssh_key # ssh key to add via cloud-init + user_workfolder = "/home/${local.username}" + timeout = 5 + } +} + +module "download" { + source = "rancher/rke2-download/github" + version = "v0.1.1" + path = local.local_file_path +} + +module "config" { + depends_on = [ + module.access, + module.server, + module.download, + ] + source = "rancher/rke2-config/local" + version = "v0.1.4" + local_file_path = local.local_file_path + cni = ["none"] # install cilium with helm chart in manifests directory +} + +# everything before this module is not necessary, you can generate the resources manually or use other methods +module "this" { + depends_on = [ + module.access, + module.server, + module.download, + module.config, + ] + source = "../../" # change this to "rancher/rke2-install/null" per https://registry.terraform.io/modules/rancher/rke2-install/null/latest + # version = "v0.2.7" # when using this example you will need to set the version + ssh_ip = module.server.server.public_ip + ssh_user = local.username + release = local.rke2_version + local_file_path = local.local_file_path + local_manifests_path = "${path.root}/manifests" + retrieve_kubeconfig = true + remote_workspace = module.server.image.workfolder + identifier = md5(join("-", [ + # if any of these things change, redeploy rke2 + module.server.server.id, + local.rke2_version, + module.config.yaml_config, + ])) +} diff --git a/examples/manifest/manifests/cilium.yaml b/examples/manifest/manifests/cilium.yaml new file mode 100644 index 0000000..d0e32bc --- /dev/null +++ b/examples/manifest/manifests/cilium.yaml @@ -0,0 +1,8 @@ +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: cilium + namespace: kube-system +spec: + bootstrap: true + chart: https://raw.githubusercontent.com/cilium/charts/master/cilium-1.16.1.tgz diff --git a/examples/manifest/outputs.tf b/examples/manifest/outputs.tf new file mode 100644 index 0000000..3f0f062 --- /dev/null +++ b/examples/manifest/outputs.tf @@ -0,0 +1,14 @@ +output "server" { + value = module.server.server +} +output "image" { + value = module.server.image +} +output "access" { + value = module.access +} +output "kubeconfig" { + value = module.this.kubeconfig + description = "Kubernetes config file contents for the cluster." + sensitive = true +} diff --git a/examples/manifest/variables.tf b/examples/manifest/variables.tf new file mode 100644 index 0000000..d7a85de --- /dev/null +++ b/examples/manifest/variables.tf @@ -0,0 +1,7 @@ +variable "identifier" { + type = string +} +variable "key" { + type = string +} + diff --git a/examples/manifest/versions.tf b/examples/manifest/versions.tf new file mode 100644 index 0000000..f9264ce --- /dev/null +++ b/examples/manifest/versions.tf @@ -0,0 +1,33 @@ +terraform { + required_version = ">= 1.5.0, < 1.6" + required_providers { + local = { + source = "hashicorp/local" + version = ">= 2.4" + } + aws = { + source = "hashicorp/aws" + version = ">= 5.11" + } + random = { + source = "hashicorp/random" + version = ">= 3.1" + } + http = { + source = "hashicorp/http" + version = ">= 3.4" + } + acme = { # used in the access module + source = "vancluever/acme" + version = ">= 2.0" + } + github = { + source = "integrations/github" + version = "6.2.1" + } + } +} +provider "acme" { + server_url = "https://acme-staging-v02.api.letsencrypt.org/directory" +} +provider "github" {} diff --git a/flake.lock b/flake.lock index 0db5a22..c0137b3 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1723151389, - "narHash": "sha256-9AVY0ReCmSGXHrlx78+1RrqcDgVSRhHUKDVV1LLBy28=", + "lastModified": 1724395761, + "narHash": "sha256-zRkDV/nbrnp3Y8oCADf5ETl1sDrdmAW6/bBVJ8EbIdQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "13fe00cb6c75461901f072ae62b5805baef9f8b2", + "rev": "ae815cee91b417be55d43781eb4b73ae1ecc396c", "type": "github" }, "original": { diff --git a/main.tf b/main.tf index e092d9c..3fa7ad8 100644 --- a/main.tf +++ b/main.tf @@ -9,6 +9,7 @@ locals { identifier = var.identifier local_file_path = var.local_file_path local_path = (local.local_file_path == "" ? "${abspath(path.root)}/rke2" : local.local_file_path) + local_manifests_path = var.local_manifests_path remote_workspace = ((var.remote_workspace == "~" || var.remote_workspace == "") ? "/home/${local.ssh_user}" : var.remote_workspace) # https://github.com/hashicorp/terraform/issues/30243 remote_path = (var.remote_file_path == "" ? "${local.remote_workspace}/rke2_artifacts" : var.remote_file_path) retrieve_kubeconfig = var.retrieve_kubeconfig @@ -195,6 +196,49 @@ resource "time_sleep" "ten_s_after_install" { ] create_duration = "10s" } + +# copy manifests to remote server after install, but before start +resource "terraform_data" "copy_manifests" { + count = (local.local_manifests_path == "" ? 0 : 1) + depends_on = [ + terraform_data.copy_to_remote, + null_resource.configure, + null_resource.install_prep, + time_sleep.ten_s_before_install, + null_resource.install, + time_sleep.ten_s_after_install, + ] + triggers_replace = local.identifier + connection { + type = "ssh" + user = local.ssh_user + script_path = "${local.remote_workspace}/copy_manifests_terraform" + agent = true + host = local.ssh_ip + } + provisioner "remote-exec" { + inline = [<<-EOT + echo "Connected!" + EOT + ] + } + provisioner "file" { + source = local.local_manifests_path + destination = "${local.remote_path}/manifests" + } + provisioner "remote-exec" { + inline = [<<-EOT + set -x + set -e + ls -lah ${local.remote_path}/manifests + sudo install -d /var/lib/rancher/rke2/server/manifests + sudo cp ${local.remote_path}/manifests/* /var/lib/rancher/rke2/server/manifests + ls -lah /var/lib/rancher/rke2/server/manifests + EOT + ] + } +} + # optionally run a script on the server before starting rke2 # this can be used to mitigate OS specific issues or configuration resource "null_resource" "prep" { @@ -206,6 +250,7 @@ resource "null_resource" "prep" { time_sleep.ten_s_before_install, null_resource.install, time_sleep.ten_s_after_install, + terraform_data.copy_manifests, ] triggers = { id = local.identifier, @@ -254,6 +299,7 @@ resource "time_sleep" "ten_s_before_start" { time_sleep.ten_s_before_install, null_resource.install, time_sleep.ten_s_after_install, + terraform_data.copy_manifests, null_resource.prep, ] create_duration = "10s" @@ -268,6 +314,7 @@ resource "null_resource" "start" { time_sleep.ten_s_before_install, null_resource.install, time_sleep.ten_s_after_install, + terraform_data.copy_manifests, null_resource.prep, time_sleep.ten_s_before_start, ] @@ -308,9 +355,13 @@ resource "null_resource" "get_kubeconfig" { terraform_data.copy_to_remote, null_resource.configure, null_resource.install_prep, + time_sleep.ten_s_before_install, null_resource.install, - null_resource.start, + time_sleep.ten_s_after_install, + terraform_data.copy_manifests, null_resource.prep, + time_sleep.ten_s_before_start, + null_resource.start, ] triggers = { id = local.identifier, @@ -357,10 +408,14 @@ data "local_sensitive_file" "kubeconfig" { terraform_data.copy_to_remote, null_resource.configure, null_resource.install_prep, + time_sleep.ten_s_before_install, null_resource.install, + time_sleep.ten_s_after_install, + terraform_data.copy_manifests, + null_resource.prep, + time_sleep.ten_s_before_start, null_resource.start, null_resource.get_kubeconfig, - null_resource.prep, ] filename = "${local.local_path}/kubeconfig" } diff --git a/tests/manifest_test.go b/tests/manifest_test.go new file mode 100644 index 0000000..912f4b7 --- /dev/null +++ b/tests/manifest_test.go @@ -0,0 +1,48 @@ +package test + +import ( + "fmt" + "os" + "testing" + + "github.com/gruntwork-io/terratest/modules/random" + "github.com/gruntwork-io/terratest/modules/ssh" + "github.com/gruntwork-io/terratest/modules/terraform" + + "github.com/stretchr/testify/assert" +) + +func TestManifest(t *testing.T) { + t.Parallel() + id := os.Getenv("IDENTIFIER") + if id == "" { + id = random.UniqueId() + } + directory := "manifest" + region := "us-west-1" + owner := "terraform-ci@suse.com" + terraformVars := map[string]interface{}{} + terraformOptions, keyPair := setup(t, directory, region, owner, id, terraformVars) + delete(terraformOptions.Vars, "key_name") + + sshAgent := ssh.SshAgentWithKeyPair(t, keyPair.KeyPair) + defer sshAgent.Stop() + terraformOptions.SshAgent = sshAgent + + defer teardown(t, directory, keyPair) + defer terraform.Destroy(t, terraformOptions) + terraform.InitAndApply(t, terraformOptions) + + out := terraform.OutputAll(t, terraformOptions) + t.Logf("out: %v", out) + outputServer, ok := out["server"].(map[string]interface{}) + assert.True(t, ok, fmt.Sprintf("Wrong data type for 'server', expected map[string], got %T", out["server"])) + outputImage, ok := out["image"].(map[string]interface{}) + assert.True(t, ok, fmt.Sprintf("Wrong data type for 'image', expected map[string], got %T", out["image"])) + outputKubeconfig, ok := out["kubeconfig"].(string) + assert.True(t, ok, fmt.Sprintf("Wrong data type for 'kubeconfig', expected string, got %T", out["kubeconfig"])) + + assert.NotEmpty(t, outputKubeconfig, "The 'kubeconfig' is empty") + assert.NotEmpty(t, outputServer["public_ip"], "The 'server.public_ip' is empty") + assert.NotEmpty(t, outputImage["id"], "The 'image.id' is empty") +} diff --git a/tests/util_test.go b/tests/util_test.go index 8aa8720..f3b5602 100644 --- a/tests/util_test.go +++ b/tests/util_test.go @@ -3,6 +3,7 @@ package test import ( "context" "fmt" + //"log" "os" "path/filepath" @@ -90,8 +91,9 @@ func setup(t *testing.T, directory string, region string, owner string, id strin "AWS_DEFAULT_REGION": region, }, RetryableTerraformErrors: retryableTerraformErrors, - NoColor: true, - MaxRetries: 20, + NoColor: true, + MaxRetries: 20, + Upgrade: true, }) return terraformOptions, keyPair } diff --git a/variables.tf b/variables.tf index 5474b56..9309252 100644 --- a/variables.tf +++ b/variables.tf @@ -27,6 +27,13 @@ variable "role" { EOT default = "server" } +variable "local_manifests_path" { + type = string + description = <<-EOT + The local path to a directory with manifests that will be copied to the /var/lib/rancher/rke2/server/manifests directory on the server. + EOT + default = "" +} variable "local_file_path" { type = string description = <<-EOT