From 27133a507ffcbe24bf1058f3d15588bb42427589 Mon Sep 17 00:00:00 2001 From: Jim Martens Date: Fri, 3 Nov 2023 08:48:16 +0100 Subject: [PATCH] Added module to create k8s cluster with single or multiple nodes --- terraform/.terraform/modules/modules.json | 2 +- terraform/domains.tf | 26 ++- terraform/hetzner.tf | 35 ++-- terraform/ingress-apps.tf | 6 +- terraform/main.tf | 70 ++++++-- terraform/modules/domain/main.tf | 9 - terraform/modules/domain/versions.tf | 8 + terraform/modules/k8s-cluster/ha.tf | 0 terraform/modules/k8s-cluster/helm.tf | 18 ++ terraform/modules/k8s-cluster/inwx.tf | 14 ++ terraform/modules/k8s-cluster/main.tf | 37 ++++ terraform/modules/k8s-cluster/manager-node.tf | 160 ++++++++++++++++++ terraform/modules/k8s-cluster/outputs.tf | 0 .../modules/k8s-cluster/templates/add-node.sh | 8 + .../templates/cloud-init-k8s.tftpl | 26 ++- .../k8s-cluster/templates/join-worker.sh | 12 ++ .../modules/k8s-cluster/templates/join.sh | 12 ++ .../templates/microk8s-config.tftpl | 20 +++ terraform/modules/k8s-cluster/variables.tf | 130 ++++++++++++++ terraform/modules/k8s-cluster/versions.tf | 17 ++ terraform/modules/k8s-cluster/worker-node.tf | 94 ++++++++++ terraform/nameserver_records.tf | 28 +-- terraform/uberspace.tf | 20 +-- terraform/variables.tf | 11 ++ terraform/versions.tf | 19 +++ 25 files changed, 687 insertions(+), 95 deletions(-) create mode 100644 terraform/modules/domain/versions.tf create mode 100644 terraform/modules/k8s-cluster/ha.tf create mode 100644 terraform/modules/k8s-cluster/helm.tf create mode 100644 terraform/modules/k8s-cluster/inwx.tf create mode 100644 terraform/modules/k8s-cluster/main.tf create mode 100644 terraform/modules/k8s-cluster/manager-node.tf create mode 100644 terraform/modules/k8s-cluster/outputs.tf create mode 100644 terraform/modules/k8s-cluster/templates/add-node.sh rename terraform/{ => modules/k8s-cluster}/templates/cloud-init-k8s.tftpl (70%) create mode 100644 terraform/modules/k8s-cluster/templates/join-worker.sh create mode 100644 terraform/modules/k8s-cluster/templates/join.sh create mode 100644 terraform/modules/k8s-cluster/templates/microk8s-config.tftpl create mode 100644 terraform/modules/k8s-cluster/variables.tf create mode 100644 terraform/modules/k8s-cluster/versions.tf create mode 100644 terraform/modules/k8s-cluster/worker-node.tf create mode 100644 terraform/versions.tf diff --git a/terraform/.terraform/modules/modules.json b/terraform/.terraform/modules/modules.json index 02c2538..7737b47 100644 --- a/terraform/.terraform/modules/modules.json +++ b/terraform/.terraform/modules/modules.json @@ -1 +1 @@ -{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"api_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"argocd_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"drone_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"gitea_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"helm-repo_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"keycloak_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"nextcloud_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"personal_website_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"statping_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"wahlrecht_domain","Source":"./modules/domain","Dir":"modules/domain"}]} \ No newline at end of file +{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"api_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"argocd_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"drone_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"gitea_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"helm-repo_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"keycloak_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"nextcloud_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"personal_website_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"prometheus_k8s_test_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"statping_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"test_cluster","Source":"./modules/k8s-cluster","Dir":"modules/k8s-cluster"},{"Key":"wahlrecht_domain","Source":"./modules/domain","Dir":"modules/domain"}]} \ No newline at end of file diff --git a/terraform/domains.tf b/terraform/domains.tf index b4977ff..bed4be2 100644 --- a/terraform/domains.tf +++ b/terraform/domains.tf @@ -17,7 +17,7 @@ resource "inwx_domain_contact" "admin" { } resource "inwx_domain" "twomartens_de" { - name = "2martens.de" + name = local.domain nameservers = [ "ns.inwx.de", "ns2.inwx.de", @@ -59,24 +59,24 @@ resource "inwx_domain" "twomartens_eu" { // NS records resource "inwx_nameserver_record" "twomartens_de_ns1" { - domain = "2martens.de" - name = "2martens.de" + domain = local.domain + name = local.domain content = "ns.inwx.de" type = "NS" ttl = 86400 } resource "inwx_nameserver_record" "twomartens_de_ns2" { - domain = "2martens.de" - name = "2martens.de" + domain = local.domain + name = local.domain content = "ns2.inwx.de" type = "NS" ttl = 86400 } resource "inwx_nameserver_record" "twomartens_de_ns3" { - domain = "2martens.de" - name = "2martens.de" + domain = local.domain + name = local.domain content = "ns3.inwx.eu" type = "NS" ttl = 86400 @@ -109,11 +109,15 @@ resource "inwx_nameserver_record" "twomartens_eu_ns3" { // SOAs resource "inwx_nameserver_record" "twomartens_de_soa" { - domain = "2martens.de" - name = "2martens.de" + domain = local.domain + name = local.domain content = "ns.inwx.de hostmaster.inwx.de 2023103008 10800 3600 604800 3600" type = "SOA" ttl = 86400 + + lifecycle { + ignore_changes = [content] + } } resource "inwx_nameserver_record" "twomartens_eu_soa" { @@ -122,6 +126,10 @@ resource "inwx_nameserver_record" "twomartens_eu_soa" { content = "ns.inwx.de hostmaster.inwx.de 2023103001 10800 3600 604800 3600" type = "SOA" ttl = 86400 + + lifecycle { + ignore_changes = [content] + } } // redirect 2martens.eu to 2martens.de diff --git a/terraform/hetzner.tf b/terraform/hetzner.tf index c64e7d9..16a8b12 100644 --- a/terraform/hetzner.tf +++ b/terraform/hetzner.tf @@ -1,25 +1,5 @@ // data sources -data "hcloud_location" "falkenstein" { - name = "fsn1" -} - -data "hcloud_location" "nuremberg" { - name = "nbg1" -} - -data "hcloud_datacenter" "falkenstein" { - name = "fsn1-dc14" -} - -data "hcloud_datacenter" "nuremberg" { - name = "nbg1-dc3" -} - -data "hcloud_ssh_key" "macos" { - name = "2martens@Jims-Air.fritz.box" -} - data "hcloud_placement_group" "default" { name = "default" } @@ -198,9 +178,16 @@ resource "hcloud_server" "server_k8s_test" { } firewall_ids = [hcloud_firewall.basic-firewall.id, hcloud_firewall.k8s-firewall.id] ssh_keys = [data.hcloud_ssh_key.macos.id] - user_data = templatefile("${path.module}/templates/cloud-init-k8s.tftpl", { + user_data = templatefile("${path.module}/modules/k8s-cluster/templates/cloud-init-k8s.tftpl", { node_ip : "10.0.0.2", - cluster_token : "DkZaYY5JUAptUtq4HbXVb2x4HHD3HP7U" + admin_public_ssh_key : format("%s %s", data.hcloud_ssh_key.macos.public_key, data.hcloud_ssh_key.macos.name), + terraform_public_ssh_key : var.terraform_public_ssh_key, + microk8s_channel : "1.28-strict/stable", + admin_user : "2martensAdmin", + "microk8s_config" : templatefile("${path.module}/modules/k8s-cluster/templates/microk8s-config.tftpl", { + node_ip : "10.0.0.2", + api_server_domain : "" + }) }) shutdown_before_deletion = true @@ -219,13 +206,13 @@ resource "hcloud_server" "server_k8s_test" { module "drone_domain" { source = "./modules/domain" - domain = "2martens.de" + domain = local.domain subdomain = "ci" ipv4 = "49.12.69.146" ipv6 = "2a01:4f8:c012:1fef::1" } resource "inwx_nameserver_record" "twomartens_de_ci-ownercheck_txt" { - domain = "2martens.de" + domain = local.domain name = "ownercheck.ci.2martens.de" content = "e1eb600b" type = "TXT" diff --git a/terraform/ingress-apps.tf b/terraform/ingress-apps.tf index 8b6200a..5ac3cae 100644 --- a/terraform/ingress-apps.tf +++ b/terraform/ingress-apps.tf @@ -1,6 +1,6 @@ module "argocd_domain" { source = "./modules/domain" - domain = "2martens.de" + domain = local.domain subdomain = "argocd" ipv4 = "49.13.77.70" ipv6 = "2a01:4f8:c012:4d25::1" @@ -8,7 +8,7 @@ module "argocd_domain" { module "api_domain" { source = "./modules/domain" - domain = "2martens.de" + domain = local.domain subdomain = "api" ipv4 = "49.13.77.70" ipv6 = "2a01:4f8:c012:4d25::1" @@ -16,7 +16,7 @@ module "api_domain" { module "prometheus_k8s_test_domain" { source = "./modules/domain" - domain = "2martens.de" + domain = local.domain subdomain = "prometheus.k8s.test" ipv4 = "49.13.77.70" ipv6 = "2a01:4f8:c012:4d25::1" diff --git a/terraform/main.tf b/terraform/main.tf index 9228b83..6d1a4a7 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,19 +1,55 @@ -terraform { - cloud { - organization = "2martens" +data "hcloud_location" "falkenstein" { + name = "fsn1" +} - workspaces { - name = "cloud-configuration" - } - } - required_providers { - inwx = { - source = "inwx/inwx" - version = "1.3.2" - } - hcloud = { - source = "hetznercloud/hcloud" - version = "1.44.1" - } - } +data "hcloud_location" "nuremberg" { + name = "nbg1" +} + +data "hcloud_datacenter" "falkenstein" { + name = "fsn1-dc14" +} + +data "hcloud_datacenter" "nuremberg" { + name = "nbg1-dc3" +} + +data "hcloud_ssh_key" "macos" { + name = "2martens@Jims-Air.fritz.box" +} + +locals { + domain = "2martens.de" +} + +module "test_cluster" { + source = "./modules/k8s-cluster" + + admin_ssh_key = data.hcloud_ssh_key.macos + admin_user = "2martensAdmin" + terraform_public_ssh_key = var.terraform_public_ssh_key + terraform_private_ssh_key = var.terraform_private_ssh_key + basic_firewall_id = hcloud_firewall.basic-firewall.id + k8s_firewall_id = hcloud_firewall.k8s-firewall.id + locations = [{ + id = data.hcloud_location.falkenstein.id, + name = data.hcloud_location.falkenstein.name, + datacenter_name = data.hcloud_datacenter.falkenstein.name, + network_zone = data.hcloud_location.falkenstein.network_zone + }, { + id = data.hcloud_location.nuremberg.id, + name = data.hcloud_location.nuremberg.name, + datacenter_name = data.hcloud_datacenter.nuremberg.name, + network_zone = data.hcloud_location.nuremberg.network_zone + }] + domain = local.domain + network_id = hcloud_network.kubernetes-network.id + private_node_ips = ["10.0.0.4", "10.0.0.5", "10.0.0.6"] + server_subnet_id = hcloud_network_subnet.k8s-network-subnet.id + cluster_role = "test" + number_nodes = 1 + server_type = "cax21" + image_name = "ubuntu-22.04" + create_loadbalancer = false + loadbalancer_ip = "10.0.0.3" } \ No newline at end of file diff --git a/terraform/modules/domain/main.tf b/terraform/modules/domain/main.tf index 7add14a..b841ebe 100644 --- a/terraform/modules/domain/main.tf +++ b/terraform/modules/domain/main.tf @@ -1,12 +1,3 @@ -terraform { - required_providers { - inwx = { - source = "inwx/inwx" - version = "1.3.2" - } - } -} - resource "inwx_nameserver_record" "a" { domain = var.domain name = var.subdomain != "" ? format("%s.%s", var.subdomain, var.domain) : var.domain diff --git a/terraform/modules/domain/versions.tf b/terraform/modules/domain/versions.tf new file mode 100644 index 0000000..09192be --- /dev/null +++ b/terraform/modules/domain/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + inwx = { + source = "inwx/inwx" + version = "1.3.2" + } + } +} diff --git a/terraform/modules/k8s-cluster/ha.tf b/terraform/modules/k8s-cluster/ha.tf new file mode 100644 index 0000000..e69de29 diff --git a/terraform/modules/k8s-cluster/helm.tf b/terraform/modules/k8s-cluster/helm.tf new file mode 100644 index 0000000..cf772fa --- /dev/null +++ b/terraform/modules/k8s-cluster/helm.tf @@ -0,0 +1,18 @@ +resource "null_resource" "install_argocd" { + connection { + host = hcloud_server_network.manager_private[0].ip + user = "root" + type = "ssh" + private_key = var.terraform_private_ssh_key + timeout = "2m" + } + + provisioner "remote-exec" { + inline = [ + "helm repo add 2martens https://repo.2martens.de/charts/", + "helm install 2martens/argocd", + ] + } + + depends_on = [null_resource.join_nodes, hcloud_server.manager] +} \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/inwx.tf b/terraform/modules/k8s-cluster/inwx.tf new file mode 100644 index 0000000..3b8521d --- /dev/null +++ b/terraform/modules/k8s-cluster/inwx.tf @@ -0,0 +1,14 @@ +resource "inwx_nameserver_record" "kube_api_server_a" { + domain = var.domain + name = local.kube_api_server_domain + content = var.create_loadbalancer ? hcloud_load_balancer.kubernetes[0].ipv4 : hcloud_primary_ip.ipv4_manager_address[0].ip_address + type = "A" + ttl = 3600 +} +resource "inwx_nameserver_record" "kube_api_server_aaaa" { + domain = var.domain + name = local.kube_api_server_domain + content = var.create_loadbalancer ? hcloud_load_balancer.kubernetes[0].ipv6 : hcloud_primary_ip.ipv6_manager_address[0].ip_address + type = "AAAA" + ttl = 3600 +} \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/main.tf b/terraform/modules/k8s-cluster/main.tf new file mode 100644 index 0000000..01fb878 --- /dev/null +++ b/terraform/modules/k8s-cluster/main.tf @@ -0,0 +1,37 @@ +data "hcloud_placement_group" "default" { + name = "default" +} + +locals { + kube_api_server_domain = format("%s.%s.%s", "k8s", var.cluster_role, var.domain) +} + +resource "hcloud_load_balancer" "kubernetes" { + count = var.create_loadbalancer ? 1 : 0 + + load_balancer_type = var.loadbalancer_type + name = format("%s_%s", "k8s", var.cluster_role) + location = var.locations[count.index].name + network_zone = var.locations[count.index].network_zone + algorithm { + type = "round_robin" + } +} + +resource "hcloud_load_balancer_network" "private" { + count = var.create_loadbalancer ? 1 : 0 + + load_balancer_id = hcloud_load_balancer.kubernetes[count.index].id + subnet_id = var.server_subnet_id + ip = var.loadbalancer_ip +} + +resource "hcloud_load_balancer_target" "nodes" { + count = var.create_loadbalancer ? var.number_nodes : 0 + + type = "server" + load_balancer_id = hcloud_load_balancer.kubernetes[0].id + server_id = hcloud_server.manager[count.index].id + use_private_ip = true +} + diff --git a/terraform/modules/k8s-cluster/manager-node.tf b/terraform/modules/k8s-cluster/manager-node.tf new file mode 100644 index 0000000..a575321 --- /dev/null +++ b/terraform/modules/k8s-cluster/manager-node.tf @@ -0,0 +1,160 @@ +resource "hcloud_primary_ip" "ipv4_manager_address" { + count = var.create_loadbalancer ? 0 : var.number_nodes + + name = format("%s_%s_%s_%d", "k8s", var.cluster_role, "ipv4", count.index) + datacenter = element(var.locations.*.datacenter_name, count.index) + type = "ipv4" + assignee_type = "server" + auto_delete = false +} + +resource "hcloud_primary_ip" "ipv6_manager_address" { + count = var.create_loadbalancer ? 0 : var.number_nodes + + name = format("%s_%s_%s_%d", "k8s", var.cluster_role, "ipv6", count.index) + datacenter = element(var.locations.*.datacenter_name, count.index) + type = "ipv6" + assignee_type = "server" + auto_delete = false +} + +resource "hcloud_server_network" "manager_private" { + count = var.number_nodes + + server_id = hcloud_server.manager[count.index].id + subnet_id = var.server_subnet_id + ip = var.private_node_ips[count.index] +} + +resource "hcloud_server" "manager" { + count = var.number_nodes + + name = format("%s_%s_%s_%d", "k8s", var.cluster_role, "node", count.index) + image = var.image_name + allow_deprecated_images = false + server_type = var.server_type + location = element(var.locations.*.name, count.index) + public_net { + ipv4_enabled = var.create_loadbalancer ? false : true + ipv4 = var.create_loadbalancer ? null : hcloud_primary_ip.ipv4_manager_address[count.index].id + ipv6_enabled = var.create_loadbalancer ? false : true + ipv6 = var.create_loadbalancer ? null : hcloud_primary_ip.ipv6_manager_address[count.index].id + } + ignore_remote_firewall_ids = false + keep_disk = false + placement_group_id = data.hcloud_placement_group.default.id + labels = { + "kubernetes" : "yes", + "manager" : "yes" + } + firewall_ids = [var.basic_firewall_id, var.k8s_firewall_id] + ssh_keys = [var.admin_ssh_key.id] + user_data = templatefile("${path.module}/templates/cloud-init-k8s.tftpl", { + node_ip : var.private_node_ips[count.index], + admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name), + terraform_public_ssh_key : var.terraform_public_ssh_key, + microk8s_channel : var.microk8s_channel, + admin_user : var.admin_user, + microk8s_config : templatefile("${path.module}/templates/microk8s-config.tftpl", { + node_ip : var.private_node_ips[count.index], + api_server_domain : local.kube_api_server_domain + }) + }) + shutdown_before_deletion = true + + network { + network_id = var.network_id + ip = var.private_node_ips[count.index] + alias_ips = [] + } + + lifecycle { + ignore_changes = [ssh_keys, user_data] + } +} + +resource "random_id" "cluster_token" { + byte_length = 16 +} + +resource "null_resource" "setup_tokens" { + count = var.number_nodes > 1 ? 1 : 0 + + triggers = { + rerun = random_id.cluster_token.hex + } + + connection { + host = hcloud_server_network.manager_private[0].ip + user = "root" + type = "ssh" + private_key = var.terraform_private_ssh_key + timeout = "2m" + } + + provisioner "local-exec" { + interpreter = ["bash", "-c"] + command = < /tmp/current_joining_node.txt + echo "0" > /tmp/current_joining_worker_node.txt + EOT + } + + provisioner "file" { + content = templatefile("${path.module}/templates/add-node.sh", + { + cluster_token = random_id.cluster_token.hex + cluster_token_ttl_seconds = var.cluster_token_ttl_seconds + }) + destination = "/usr/local/bin/add-node.sh" + } + + provisioner "remote-exec" { + inline = [ + "sh /usr/local/bin/add-node.sh", + ] + } +} + + +resource "null_resource" "join_nodes" { + count = var.number_nodes - 1 < 1 ? 0 : var.number_nodes - 1 + + triggers = { + rerun = random_id.cluster_token.hex + } + connection { + host = element(hcloud_server_network.manager_private.*.ip, count.index + 1) + user = "root" + type = "ssh" + private_key = var.terraform_private_ssh_key + timeout = "20m" + } + + provisioner "local-exec" { + interpreter = ["bash", "-c"] + command = "while [[ $(cat /tmp/current_joining_node.txt) != \"${count.index + 1}\" ]]; do echo \"${count.index + 1} is waiting...\";sleep 5;done" + } + + provisioner "file" { + content = templatefile("${path.module}/templates/join.sh", + { + cluster_token = random_id.cluster_token.hex + main_node_ip = hcloud_server_network.manager_private[0].ip + }) + destination = "/usr/local/bin/join.sh" + } + + provisioner "remote-exec" { + inline = [ + "sh /usr/local/bin/join.sh" + ] + } + + provisioner "local-exec" { + interpreter = ["bash", "-c"] + command = "echo \"${count.index + 2}\" > /tmp/current_joining_node.txt" + } + + depends_on = [null_resource.setup_tokens] +} \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/outputs.tf b/terraform/modules/k8s-cluster/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/terraform/modules/k8s-cluster/templates/add-node.sh b/terraform/modules/k8s-cluster/templates/add-node.sh new file mode 100644 index 0000000..4985659 --- /dev/null +++ b/terraform/modules/k8s-cluster/templates/add-node.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +until microk8s.status --wait-ready; + do sleep 5; echo "waiting for worker status.."; +done + +sleep 10 +microk8s add-node --token ${cluster_token} --token-ttl ${cluster_token_ttl_seconds} diff --git a/terraform/templates/cloud-init-k8s.tftpl b/terraform/modules/k8s-cluster/templates/cloud-init-k8s.tftpl similarity index 70% rename from terraform/templates/cloud-init-k8s.tftpl rename to terraform/modules/k8s-cluster/templates/cloud-init-k8s.tftpl index 2e2a0db..0e19520 100644 --- a/terraform/templates/cloud-init-k8s.tftpl +++ b/terraform/modules/k8s-cluster/templates/cloud-init-k8s.tftpl @@ -1,16 +1,24 @@ #cloud-config users: - - name: 2martensAdmin + - name: ${admin_user} groups: users, admin sudo: ALL=(ALL) NOPASSWD:ALL shell: /bin/bash ssh_authorized_keys: - - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBjYkz73HbsFtRAOfS5QB9xDbOajU9C4gpDp+OjQkLir 2martens@Jims-Air.fritz.box" + - ${admin_public_ssh_key} + - name: terraform + groups: users, admin + sudo: ALL=(ALL) NOPASSWD:ALL + ssh_authorized_keys: + - ${terraform_public_ssh_key} packages: - fail2ban - ufw package_update: true package_upgrade: true +snap: + commands: + - snap install kubectl --classic runcmd: - printf "[sshd]\nenabled = true\nbanaction = iptables-multiport" > /etc/fail2ban/jail.local - systemctl enable fail2ban @@ -30,12 +38,14 @@ runcmd: - sed -i -e '/^\(#\|\)AllowTcpForwarding/s/^.*$/AllowTcpForwarding no/' /etc/ssh/sshd_config - sed -i -e '/^\(#\|\)AllowAgentForwarding/s/^.*$/AllowAgentForwarding no/' /etc/ssh/sshd_config - sed -i -e '/^\(#\|\)AuthorizedKeysFile/s/^.*$/AuthorizedKeysFile .ssh\/authorized_keys/' /etc/ssh/sshd_config - - sed -i '$a AllowUsers 2martensAdmin' /etc/ssh/sshd_config + - sed -i '$a AllowUsers ${admin_user} terraform' /etc/ssh/sshd_config - curl https://packages.hetzner.com/hcloud/deb/hc-utils_0.0.4-1_all.deb -o /tmp/hc-utils_0.0.4-1_all.deb -s - apt install /tmp/hc-utils_0.0.4-1_all.deb - mkdir -p /var/snap/microk8s/common/ - - curl https://repo.2martens.de/files/microk8s-config.yaml -o /var/snap/microk8s/common/.microk8s.yaml - - sed -i -e '/\$NODE_IP/s//${node_ip}/' /var/snap/microk8s/common/.microk8s.yaml - - sed -i -e '/\$CLUSTER_TOKEN/s//${cluster_token}/' /var/snap/microk8s/common/.microk8s.yaml - - snap install microk8s --channel 1.28-strict/stable - - reboot \ No newline at end of file + - echo ${microk8s_config} > /var/snap/microk8s/common/.microk8s.yaml + - snap install microk8s --channel ${microk8s_channel} + - mkdir -p /home/terraform/.kube + - microk8s kubectl config view --raw > /home/terraform/.kube/config + - chown -R terraform:terraform /home/terraform/.kube + - chmod go-r /home/terraform/.kube/config + - microk8s.status --wait-ready \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/templates/join-worker.sh b/terraform/modules/k8s-cluster/templates/join-worker.sh new file mode 100644 index 0000000..16b74a9 --- /dev/null +++ b/terraform/modules/k8s-cluster/templates/join-worker.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +until microk8s.status --wait-ready; + do sleep 3; echo "waiting for worker status.."; +done + + +if microk8s status | grep "datastore master nodes: 127.0.0.1:19001" > /dev/null 2>&1; then + microk8s join ${main_node_ip}:25000/${cluster_token} --worker +else + echo "Join process already done. Nothing to do" +fi \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/templates/join.sh b/terraform/modules/k8s-cluster/templates/join.sh new file mode 100644 index 0000000..693b664 --- /dev/null +++ b/terraform/modules/k8s-cluster/templates/join.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +until microk8s.status --wait-ready; + do sleep 3; echo "waiting for worker status.."; +done + + +if microk8s status | grep "datastore master nodes: 127.0.0.1:19001" > /dev/null 2>&1; then + microk8s join "${main_node_ip}:25000/${cluster_token}" +else + echo "Join process already done. Nothing to do" +fi \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/templates/microk8s-config.tftpl b/terraform/modules/k8s-cluster/templates/microk8s-config.tftpl new file mode 100644 index 0000000..f5c164b --- /dev/null +++ b/terraform/modules/k8s-cluster/templates/microk8s-config.tftpl @@ -0,0 +1,20 @@ +--- +version: 0.1.0 +extraKubeAPIServerArgs: + --advertise-address=${node_ip} + --authorization-mode=RBAC,Node +extraKubeletArgs: + --node-ip=${node_ip} + --cluster-dns=10.152.183.10 + --cluster-domain=cluster.local +extraSANs: + - ${api_server_domain} +addons: + - name: dns + - name: cis-hardening + - name: community + - name: helm3 + - name: ingress + - name: ha-cluster + - name: rbac + - name: cert-manager \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/variables.tf b/terraform/modules/k8s-cluster/variables.tf new file mode 100644 index 0000000..486230b --- /dev/null +++ b/terraform/modules/k8s-cluster/variables.tf @@ -0,0 +1,130 @@ +variable "admin_ssh_key" { + description = "SSH key to provide to each server for the admin" + type = object({ + id : number, + name : string + public_key : string + }) +} + +variable "admin_user" { + description = "Linux username for the admin user on each server" + type = string +} + +variable "terraform_private_ssh_key" { + description = "Private SSH key to be used by Terraform to connect with remote machines" + type = string + sensitive = true +} + +variable "terraform_public_ssh_key" { + description = "Public SSH key for the private key used by Terraform to connect with remote machines" + type = string +} + +variable "locations" { + description = "Locations in which the resources should be created. Singular resources will always be created in the first. Must be all in the same network zone." + type = list(object({ + id : number, + name : string, + datacenter_name : string, + network_zone : string + })) +} + +variable "domain" { + description = "The registered domain that should be used as the base for all subdomains." + type = string +} + +variable "network_id" { + description = "ID of the private network used to connect servers with each other and the loadbalancer" + type = number +} + +variable "server_subnet_id" { + description = "ID of the subnet of the private network that is supposed to house the servers and loadbalancer" + type = string +} + +variable "private_node_ips" { + description = "One IP in the private network subnet for each manager node created." + type = list(string) +} + +variable "private_worker_node_ips" { + description = "One IP in the private network subnet for each worker node created." + type = list(string) + default = [] +} + +variable "basic_firewall_id" { + description = "ID of the basic firewall for each server" + type = number +} + +variable "k8s_firewall_id" { + description = "ID of the K8s specific firewall" + type = number +} + +variable "cluster_role" { + description = "The role the cluster serves. For example test, qs, prod. Will be used for names and subdomains and must be unique within the Hetzner project." + type = string + default = "test" +} + +variable "number_nodes" { + description = "Number of manager nodes in the cluster." + type = number + default = 1 +} + +variable "number_worker_nodes" { + description = "Number of worker nodes in the cluster." + type = number + default = 0 +} + +variable "image_name" { + description = "Name of the image to use for the servers." + type = string + default = "ubuntu-22.04" +} + +variable "server_type" { + description = "Hetzner server type like cax21 (default)" + type = string + default = "cax21" +} + +variable "create_loadbalancer" { + description = "Specifies if a loadbalancer is created as gateway in front of the k8s servers. If false, the first server acts as gateway." + type = bool + default = false +} + +variable "loadbalancer_type" { + description = "Type of the loadbalancer used. By default lb11 is used." + type = string + default = "lb11" +} + +variable "loadbalancer_ip" { + description = "The private IP of the loadbalancer. Variable contains valid IP but must be overriden if a loadbalancer is used." + type = string + default = "0.0.0.0" +} + +variable "cluster_token_ttl_seconds" { + description = "Number of seconds the join token is valid" + type = number + default = 3600 +} + +variable "microk8s_channel" { + description = "The snap channel used for the microk8s installation" + type = string + default = "1.28-strict/stable" +} \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/versions.tf b/terraform/modules/k8s-cluster/versions.tf new file mode 100644 index 0000000..62ed723 --- /dev/null +++ b/terraform/modules/k8s-cluster/versions.tf @@ -0,0 +1,17 @@ +terraform { + required_providers { + helm = { + source = "hashicorp/helm" + version = "2.11.0" + } + inwx = { + source = "inwx/inwx" + version = "1.3.2" + } + hcloud = { + source = "hetznercloud/hcloud" + version = "1.44.1" + } + } +} + diff --git a/terraform/modules/k8s-cluster/worker-node.tf b/terraform/modules/k8s-cluster/worker-node.tf new file mode 100644 index 0000000..82272a5 --- /dev/null +++ b/terraform/modules/k8s-cluster/worker-node.tf @@ -0,0 +1,94 @@ +resource "hcloud_server_network" "worker_private" { + count = var.number_worker_nodes + + server_id = hcloud_server.worker[count.index].id + subnet_id = var.server_subnet_id + ip = var.private_worker_node_ips[count.index] +} + +resource "hcloud_server" "worker" { + count = var.number_worker_nodes + + name = format("%s_%s_%s_%d", "k8s", var.cluster_role, "node", count.index) + image = var.image_name + allow_deprecated_images = false + server_type = var.server_type + location = element(var.locations.*.name, count.index) + public_net { + ipv4_enabled = false + ipv6_enabled = false + } + ignore_remote_firewall_ids = false + keep_disk = false + placement_group_id = data.hcloud_placement_group.default.id + labels = { + "kubernetes" : "yes", + "worker" : "yes" + } + firewall_ids = [var.basic_firewall_id, var.k8s_firewall_id] + ssh_keys = [var.admin_ssh_key.id] + user_data = templatefile("${path.module}/templates/cloud-init-k8s.tftpl", { + node_ip : var.private_worker_node_ips[count.index], + admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name), + terraform_public_ssh_key : var.terraform_public_ssh_key, + microk8s_channel : var.microk8s_channel, + admin_user : var.admin_user, + microk8s_config : templatefile("${path.module}/templates/microk8s-config.tftpl", { + node_ip : var.private_worker_node_ips[count.index], + api_server_domain : local.kube_api_server_domain + }) + }) + shutdown_before_deletion = true + + network { + network_id = var.network_id + ip = var.private_worker_node_ips[count.index] + alias_ips = [] + } + + lifecycle { + ignore_changes = [ssh_keys, user_data] + } +} + +resource "null_resource" "join_workers" { + count = var.number_worker_nodes + + triggers = { + rerun = random_id.cluster_token.hex + } + connection { + host = element(hcloud_server_network.worker_private.*.ip, count.index) + user = "root" + type = "ssh" + private_key = var.terraform_private_ssh_key + timeout = "20m" + } + + provisioner "local-exec" { + interpreter = ["bash", "-c"] + command = "while [[ $(cat /tmp/current_joining_worker_node.txt) != \"${count.index}\" ]]; do echo \"${count.index} is waiting...\";sleep 5;done" + } + + provisioner "file" { + content = templatefile("${path.module}/templates/join-worker.sh", + { + cluster_token = random_id.cluster_token.hex + main_node_ip = hcloud_server_network.manager_private[0].ip + }) + destination = "/usr/local/bin/join-worker.sh" + } + + provisioner "remote-exec" { + inline = [ + "sh /usr/local/bin/join-worker.sh" + ] + } + + provisioner "local-exec" { + interpreter = ["bash", "-c"] + command = "echo \"${count.index + 1}\" > /tmp/current_joining_worker_node.txt" + } + + depends_on = [null_resource.setup_tokens, null_resource.join_nodes] +} \ No newline at end of file diff --git a/terraform/nameserver_records.tf b/terraform/nameserver_records.tf index 3b49f81..b4a75ef 100644 --- a/terraform/nameserver_records.tf +++ b/terraform/nameserver_records.tf @@ -1,6 +1,6 @@ // A records resource "inwx_nameserver_record" "twomartens_de_allris_a" { - domain = "2martens.de" + domain = local.domain name = "allris.2martens.de" content = "185.26.156.105" type = "A" @@ -8,7 +8,7 @@ resource "inwx_nameserver_record" "twomartens_de_allris_a" { } resource "inwx_nameserver_record" "twomartens_de_autoconfig_a" { - domain = "2martens.de" + domain = local.domain name = "autoconfig.2martens.de" content = "185.26.156.22" type = "A" @@ -16,7 +16,7 @@ resource "inwx_nameserver_record" "twomartens_de_autoconfig_a" { } resource "inwx_nameserver_record" "twomartens_de_autodiscover_a" { - domain = "2martens.de" + domain = local.domain name = "autodiscover.2martens.de" content = "185.26.156.22" type = "A" @@ -24,7 +24,7 @@ resource "inwx_nameserver_record" "twomartens_de_autodiscover_a" { } resource "inwx_nameserver_record" "twomartens_de_jim_a" { - domain = "2martens.de" + domain = local.domain name = "jim.2martens.de" content = "185.26.156.65" type = "A" @@ -32,7 +32,7 @@ resource "inwx_nameserver_record" "twomartens_de_jim_a" { } resource "inwx_nameserver_record" "twomartens_de_pad_a" { - domain = "2martens.de" + domain = local.domain name = "pad.2martens.de" content = "185.26.156.132" type = "A" @@ -40,7 +40,7 @@ resource "inwx_nameserver_record" "twomartens_de_pad_a" { } resource "inwx_nameserver_record" "twomartens_de_short_a" { - domain = "2martens.de" + domain = local.domain name = "short.2martens.de" content = "95.143.172.56" type = "A" @@ -50,7 +50,7 @@ resource "inwx_nameserver_record" "twomartens_de_short_a" { // AAAA records resource "inwx_nameserver_record" "twomartens_de_allris_aaaa" { - domain = "2martens.de" + domain = local.domain name = "allris.2martens.de" content = "2a00:d0c0:200:0:b9:1a:9c:61" type = "AAAA" @@ -58,7 +58,7 @@ resource "inwx_nameserver_record" "twomartens_de_allris_aaaa" { } resource "inwx_nameserver_record" "twomartens_de_autoconfig_aaaa" { - domain = "2martens.de" + domain = local.domain name = "autoconfig.2martens.de" content = "2a00:d0c0:200:0:b9:1a:9c16:5d" type = "AAAA" @@ -66,7 +66,7 @@ resource "inwx_nameserver_record" "twomartens_de_autoconfig_aaaa" { } resource "inwx_nameserver_record" "twomartens_de_autodiscover_aaaa" { - domain = "2martens.de" + domain = local.domain name = "autodiscover.2martens.de" content = "2a00:d0c0:200:0:b9:1a:9c16:5d" type = "AAAA" @@ -74,7 +74,7 @@ resource "inwx_nameserver_record" "twomartens_de_autodiscover_aaaa" { } resource "inwx_nameserver_record" "twomartens_de_jim_aaaa" { - domain = "2martens.de" + domain = local.domain name = "jim.2martens.de" content = "2a00:d0c0:200:0:b9:1a:9c:40" type = "AAAA" @@ -82,7 +82,7 @@ resource "inwx_nameserver_record" "twomartens_de_jim_aaaa" { } resource "inwx_nameserver_record" "twomartens_de_pad_aaaa" { - domain = "2martens.de" + domain = local.domain name = "pad.2martens.de" content = "2a00:d0c0:200:0:c0bf:e0ff:fec1:af72" type = "AAAA" @@ -90,7 +90,7 @@ resource "inwx_nameserver_record" "twomartens_de_pad_aaaa" { } resource "inwx_nameserver_record" "twomartens_de_short_aaaa" { - domain = "2martens.de" + domain = local.domain name = "short.2martens.de" content = "2001:1a50:11:0:3854:d4ff:fe62:e0d8" type = "AAAA" @@ -100,7 +100,7 @@ resource "inwx_nameserver_record" "twomartens_de_short_aaaa" { // CNAME records resource "inwx_nameserver_record" "twomartens_de_cdn_cname" { - domain = "2martens.de" + domain = local.domain name = "cdn.2martens.de" content = "d1fvxyvcoii67h.cloudfront.net" type = "CNAME" @@ -108,7 +108,7 @@ resource "inwx_nameserver_record" "twomartens_de_cdn_cname" { } resource "inwx_nameserver_record" "twomartens_de_aws-validations-cdn_cname" { - domain = "2martens.de" + domain = local.domain name = "_4741e2d9c8a605950eebfa048029e4ef.cdn.2martens.de" content = "_cf71e85bb40cc12717f2718e4590ad23.acm-validations.aws" type = "CNAME" diff --git a/terraform/uberspace.tf b/terraform/uberspace.tf index df23e5f..13cfbe7 100644 --- a/terraform/uberspace.tf +++ b/terraform/uberspace.tf @@ -1,7 +1,7 @@ // Identity management module "keycloak_domain" { source = "./modules/domain" - domain = "2martens.de" + domain = local.domain subdomain = "id" ipv4 = "95.143.172.184" ipv6 = "2001:1a50:11:0:98b3:92ff:fe3e:bba1" @@ -14,7 +14,7 @@ module "keycloak_domain" { // Angular frontends module "wahlrecht_domain" { source = "./modules/domain" - domain = "2martens.de" + domain = local.domain subdomain = "wahlrecht" ipv4 = "185.26.156.43" ipv6 = "2a00:d0c0:200:0:470:9fff:fe83:16bc" @@ -23,7 +23,7 @@ module "wahlrecht_domain" { // CI/CD module "gitea_domain" { source = "./modules/domain" - domain = "2martens.de" + domain = local.domain subdomain = "git" ipv4 = "185.26.156.91" ipv6 = "2a00:d0c0:200:0:b9:1a:9c:5a" @@ -31,7 +31,7 @@ module "gitea_domain" { module "helm-repo_domain" { source = "./modules/domain" - domain = "2martens.de" + domain = local.domain subdomain = "repo" ipv4 = "185.26.156.49" ipv6 = "2a00:d0c0:200:0:2ca6:bff:fe78:832f" @@ -40,7 +40,7 @@ module "helm-repo_domain" { // Monitoring module "statping_domain" { source = "./modules/domain" - domain = "2martens.de" + domain = local.domain subdomain = "status" ipv4 = "185.26.156.33" ipv6 = "2a00:d0c0:200:0:f8e2:6fff:fec2:7a92" @@ -49,7 +49,7 @@ module "statping_domain" { // Personal Website module "personal_website_domain" { source = "./modules/domain" - domain = "2martens.de" + domain = local.domain subdomain = "" ipv4 = "185.26.156.65" ipv6 = "2a00:d0c0:200:0:b9:1a:9c:40" @@ -59,8 +59,8 @@ module "personal_website_domain" { mxSpf = "v=spf1 include:spf.uberspace.de" } resource "inwx_nameserver_record" "twomartens_de_google-verification_txt" { - domain = "2martens.de" - name = "2martens.de" + domain = local.domain + name = local.domain content = "google-site-verification=nUFiHQFxBpBMdX96ELH3TcfyIfXf2ZlwMFYtXjVq5lo" type = "TXT" ttl = 3600 @@ -69,7 +69,7 @@ resource "inwx_nameserver_record" "twomartens_de_google-verification_txt" { // Nextcloud module "nextcloud_domain" { source = "./modules/domain" - domain = "2martens.de" + domain = local.domain subdomain = "cloud" ipv4 = "185.26.156.194" ipv6 = "2a00:d0c0:200:0:b9:1a:9c:95" @@ -81,7 +81,7 @@ module "nextcloud_domain" { // domain key resource "inwx_nameserver_record" "twomartens_de_uberspace-domainkey_txt" { - domain = "2martens.de" + domain = local.domain name = "uberspace._domainkey.id.2martens.de" content = "v=DKIM1;t=s;n=core;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqs3bFL8W77z7VJHCFbsJ3o68Y1uBbVsbIS1y35CMfRv6wp7si7aIG2ZeKqzdh2dTMsvtvSMaYVq1gT/EtRmTbU/BYC21ff8sqcVqn/ll5nNsk5jKWXTYAlTQp4LBQN7icw94ZNGr5/SDYcnv2nsBYFf2GUviObWXGHX4RaBFNj9NVUNWNin/HicvW+LsbfYq37QtlhjmUn9K96VCwKcTV1mx+Ek0osErYefcOVNawqWIlVRQDLkHZhk1StLsOpRqV+qEjhzTk4n4ZiNtLJG1D9CpHl24d5DKsQDyVFdEfqHimFTSNgiitlkuXg+i+NMiRA9G3gNJtvw8uvfN8f+stQIDAQAB" type = "TXT" diff --git a/terraform/variables.tf b/terraform/variables.tf index 791a3aa..4dc9bac 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -36,4 +36,15 @@ variable "admin_email" { variable "hcloud_token" { type = string sensitive = true +} + +variable "terraform_private_ssh_key" { + description = "Private SSH key to be used by Terraform to connect with remote machines" + type = string + sensitive = true +} + +variable "terraform_public_ssh_key" { + description = "Public SSH key for the private key used by Terraform to connect with remote machines" + type = string } \ No newline at end of file diff --git a/terraform/versions.tf b/terraform/versions.tf new file mode 100644 index 0000000..9228b83 --- /dev/null +++ b/terraform/versions.tf @@ -0,0 +1,19 @@ +terraform { + cloud { + organization = "2martens" + + workspaces { + name = "cloud-configuration" + } + } + required_providers { + inwx = { + source = "inwx/inwx" + version = "1.3.2" + } + hcloud = { + source = "hetznercloud/hcloud" + version = "1.44.1" + } + } +} \ No newline at end of file