Skip to content

Commit

Permalink
feat: add manifests after install before start (#162)
Browse files Browse the repository at this point in the history
Signed-off-by: matttrach <matt.trachier@suse.com>
  • Loading branch information
matttrach committed Aug 28, 2024
1 parent 43f20b1 commit d9352d3
Show file tree
Hide file tree
Showing 11 changed files with 310 additions and 14 deletions.
15 changes: 8 additions & 7 deletions examples/basic/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ locals {
example = "basic"
project_name = "tf-${substr(md5(join("-", [local.example, md5(local.identifier)])), 0, 5)}-${local.identifier}"
username = substr(lower("tf-${local.identifier}"), 0, 32)
image = "sles-15"
image = "sle-micro-60"
ip = chomp(data.http.myip.response_body)
ssh_key = var.key
rke2_version = "stable"
Expand All @@ -38,7 +38,7 @@ resource "random_pet" "server" {

module "access" {
source = "rancher/access/aws"
version = "v3.0.1"
version = "v3.1.5"
vpc_name = "${local.project_name}-vpc"
vpc_public = true
security_group_name = "${local.project_name}-sg"
Expand All @@ -51,7 +51,7 @@ module "server" {
module.access,
]
source = "rancher/server/aws"
version = "v1.1.0"
version = "v1.3.1"
image_type = local.image
server_name = "${local.project_name}-${random_pet.server.id}"
server_type = "small"
Expand All @@ -61,9 +61,10 @@ module "server" {
cloudinit_use_strategy = "default" # use the default cloudinit config
server_access_addresses = { # you must include ssh access here to enable setup
"runner" = {
port = 22
protocol = "tcp"
cidrs = ["${local.ip}/32"]
port = 22
protocol = "tcp"
cidrs = ["${local.ip}/32"]
ip_family = "ipv4"
}
}
server_user = {
Expand All @@ -89,7 +90,7 @@ module "config" {
module.download,
]
source = "rancher/rke2-config/local"
version = "v0.1.3"
version = "v0.1.4"
local_file_path = local.local_file_path
}

Expand Down
121 changes: 121 additions & 0 deletions examples/manifest/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
provider "aws" {
default_tags {
tags = {
Id = local.identifier
Owner = local.email
}
}
}

locals {
identifier = var.identifier # this is a random unique string that can be used to identify resources in the cloud provider
email = "terraform-ci@suse.com"
example = "manifest"
project_name = "tf-${substr(md5(join("-", [local.example, md5(local.identifier)])), 0, 5)}-${local.identifier}"
username = substr(lower("tf-${local.identifier}"), 0, 32)
image = "sle-micro-60"
ip = chomp(data.http.myip.response_body)
ssh_key = var.key
rke2_version = "stable"
local_file_path = "${path.root}/data/${local.identifier}"
}

data "http" "myip" {
url = "https://ipinfo.io/ip"
retry {
attempts = 2
min_delay_ms = 1000
}
}

resource "random_pet" "server" {
keepers = {
# regenerate the pet name when the identifier changes
identifier = local.identifier
}
length = 1
}

module "access" {
source = "rancher/access/aws"
version = "v3.1.5"
vpc_name = "${local.project_name}-vpc"
vpc_public = true
security_group_name = "${local.project_name}-sg"
security_group_type = "egress"
load_balancer_use_strategy = "skip"
}

module "server" {
depends_on = [
module.access,
]
source = "rancher/server/aws"
version = "v1.3.1"
image_type = local.image
server_name = "${local.project_name}-${random_pet.server.id}"
server_type = "small"
subnet_name = keys(module.access.subnets)[0]
security_group_name = module.access.security_group.tags_all.Name
direct_access_use_strategy = "ssh" # either the subnet needs to be public or you must add an eip
cloudinit_use_strategy = "default" # use the default cloudinit config
server_access_addresses = { # you must include ssh access here to enable setup
"runner" = {
port = 22
protocol = "tcp"
cidrs = ["${local.ip}/32"]
ip_family = "ipv4"
}
}
server_user = {
user = local.username
aws_keypair_use_strategy = "skip" # we will use cloud-init to add a keypair directly
ssh_key_name = "" # not creating or selecting a key, but this field is still required
public_ssh_key = local.ssh_key # ssh key to add via cloud-init
user_workfolder = "/home/${local.username}"
timeout = 5
}
}

module "download" {
source = "rancher/rke2-download/github"
version = "v0.1.1"
path = local.local_file_path
}

module "config" {
depends_on = [
module.access,
module.server,
module.download,
]
source = "rancher/rke2-config/local"
version = "v0.1.4"
local_file_path = local.local_file_path
cni = ["none"] # install cilium with helm chart in manifests directory
}

# everything before this module is not necessary, you can generate the resources manually or use other methods
module "this" {
depends_on = [
module.access,
module.server,
module.download,
module.config,
]
source = "../../" # change this to "rancher/rke2-install/null" per https://registry.terraform.io/modules/rancher/rke2-install/null/latest
# version = "v0.2.7" # when using this example you will need to set the version
ssh_ip = module.server.server.public_ip
ssh_user = local.username
release = local.rke2_version
local_file_path = local.local_file_path
local_manifests_path = "${path.root}/manifests"
retrieve_kubeconfig = true
remote_workspace = module.server.image.workfolder
identifier = md5(join("-", [
# if any of these things change, redeploy rke2
module.server.server.id,
local.rke2_version,
module.config.yaml_config,
]))
}
8 changes: 8 additions & 0 deletions examples/manifest/manifests/cilium.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: cilium
namespace: kube-system
spec:
bootstrap: true
chart: https://raw.githubusercontent.com/cilium/charts/master/cilium-1.16.1.tgz
14 changes: 14 additions & 0 deletions examples/manifest/outputs.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
output "server" {
value = module.server.server
}
output "image" {
value = module.server.image
}
output "access" {
value = module.access
}
output "kubeconfig" {
value = module.this.kubeconfig
description = "Kubernetes config file contents for the cluster."
sensitive = true
}
7 changes: 7 additions & 0 deletions examples/manifest/variables.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
variable "identifier" {
type = string
}
variable "key" {
type = string
}

33 changes: 33 additions & 0 deletions examples/manifest/versions.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
terraform {
required_version = ">= 1.5.0, < 1.6"
required_providers {
local = {
source = "hashicorp/local"
version = ">= 2.4"
}
aws = {
source = "hashicorp/aws"
version = ">= 5.11"
}
random = {
source = "hashicorp/random"
version = ">= 3.1"
}
http = {
source = "hashicorp/http"
version = ">= 3.4"
}
acme = { # used in the access module
source = "vancluever/acme"
version = ">= 2.0"
}
github = {
source = "integrations/github"
version = "6.2.1"
}
}
}
provider "acme" {
server_url = "https://acme-staging-v02.api.letsencrypt.org/directory"
}
provider "github" {}
6 changes: 3 additions & 3 deletions flake.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

59 changes: 57 additions & 2 deletions main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ locals {
identifier = var.identifier
local_file_path = var.local_file_path
local_path = (local.local_file_path == "" ? "${abspath(path.root)}/rke2" : local.local_file_path)
local_manifests_path = var.local_manifests_path
remote_workspace = ((var.remote_workspace == "~" || var.remote_workspace == "") ? "/home/${local.ssh_user}" : var.remote_workspace) # https://github.com/hashicorp/terraform/issues/30243
remote_path = (var.remote_file_path == "" ? "${local.remote_workspace}/rke2_artifacts" : var.remote_file_path)
retrieve_kubeconfig = var.retrieve_kubeconfig
Expand Down Expand Up @@ -195,6 +196,49 @@ resource "time_sleep" "ten_s_after_install" {
]
create_duration = "10s"
}

# copy manifests to remote server after install, but before start
resource "terraform_data" "copy_manifests" {
count = (local.local_manifests_path == "" ? 0 : 1)
depends_on = [
terraform_data.copy_to_remote,
null_resource.configure,
null_resource.install_prep,
time_sleep.ten_s_before_install,
null_resource.install,
time_sleep.ten_s_after_install,
]
triggers_replace = local.identifier
connection {
type = "ssh"
user = local.ssh_user
script_path = "${local.remote_workspace}/copy_manifests_terraform"
agent = true
host = local.ssh_ip
}
provisioner "remote-exec" {
inline = [<<-EOT
echo "Connected!"
EOT
]
}
provisioner "file" {
source = local.local_manifests_path
destination = "${local.remote_path}/manifests"
}
provisioner "remote-exec" {
inline = [<<-EOT
set -x
set -e
ls -lah ${local.remote_path}/manifests
sudo install -d /var/lib/rancher/rke2/server/manifests
sudo cp ${local.remote_path}/manifests/* /var/lib/rancher/rke2/server/manifests
ls -lah /var/lib/rancher/rke2/server/manifests
EOT
]
}
}

# optionally run a script on the server before starting rke2
# this can be used to mitigate OS specific issues or configuration
resource "null_resource" "prep" {
Expand All @@ -206,6 +250,7 @@ resource "null_resource" "prep" {
time_sleep.ten_s_before_install,
null_resource.install,
time_sleep.ten_s_after_install,
terraform_data.copy_manifests,
]
triggers = {
id = local.identifier,
Expand Down Expand Up @@ -254,6 +299,7 @@ resource "time_sleep" "ten_s_before_start" {
time_sleep.ten_s_before_install,
null_resource.install,
time_sleep.ten_s_after_install,
terraform_data.copy_manifests,
null_resource.prep,
]
create_duration = "10s"
Expand All @@ -268,6 +314,7 @@ resource "null_resource" "start" {
time_sleep.ten_s_before_install,
null_resource.install,
time_sleep.ten_s_after_install,
terraform_data.copy_manifests,
null_resource.prep,
time_sleep.ten_s_before_start,
]
Expand Down Expand Up @@ -308,9 +355,13 @@ resource "null_resource" "get_kubeconfig" {
terraform_data.copy_to_remote,
null_resource.configure,
null_resource.install_prep,
time_sleep.ten_s_before_install,
null_resource.install,
null_resource.start,
time_sleep.ten_s_after_install,
terraform_data.copy_manifests,
null_resource.prep,
time_sleep.ten_s_before_start,
null_resource.start,
]
triggers = {
id = local.identifier,
Expand Down Expand Up @@ -357,10 +408,14 @@ data "local_sensitive_file" "kubeconfig" {
terraform_data.copy_to_remote,
null_resource.configure,
null_resource.install_prep,
time_sleep.ten_s_before_install,
null_resource.install,
time_sleep.ten_s_after_install,
terraform_data.copy_manifests,
null_resource.prep,
time_sleep.ten_s_before_start,
null_resource.start,
null_resource.get_kubeconfig,
null_resource.prep,
]
filename = "${local.local_path}/kubeconfig"
}
Loading

0 comments on commit d9352d3

Please sign in to comment.