Added module to create k8s cluster with single or multiple nodes

terraform/.terraform/modules/modules.json

@@ -1 +1 @@
-{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"api_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"argocd_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"drone_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"gitea_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"helm-repo_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"keycloak_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"nextcloud_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"personal_website_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"statping_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"wahlrecht_domain","Source":"./modules/domain","Dir":"modules/domain"}]}
+{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"api_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"argocd_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"drone_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"gitea_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"helm-repo_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"keycloak_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"nextcloud_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"personal_website_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"prometheus_k8s_test_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"statping_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"test_cluster","Source":"./modules/k8s-cluster","Dir":"modules/k8s-cluster"},{"Key":"wahlrecht_domain","Source":"./modules/domain","Dir":"modules/domain"}]}

terraform/domains.tf

@@ -17,7 +17,7 @@ resource "inwx_domain_contact" "admin" {
 }
 
 resource "inwx_domain" "twomartens_de" {
-  name = "2martens.de"
+  name = local.domain
   nameservers = [
     "ns.inwx.de",
     "ns2.inwx.de",
@@ -59,24 +59,24 @@ resource "inwx_domain" "twomartens_eu" {
 
 // NS records
 resource "inwx_nameserver_record" "twomartens_de_ns1" {
-  domain  = "2martens.de"
+  domain  = local.domain
-  name    = "2martens.de"
+  name    = local.domain
   content = "ns.inwx.de"
   type    = "NS"
   ttl     = 86400
 }
 
 resource "inwx_nameserver_record" "twomartens_de_ns2" {
-  domain  = "2martens.de"
+  domain  = local.domain
-  name    = "2martens.de"
+  name    = local.domain
   content = "ns2.inwx.de"
   type    = "NS"
   ttl     = 86400
 }
 
 resource "inwx_nameserver_record" "twomartens_de_ns3" {
-  domain  = "2martens.de"
+  domain  = local.domain
-  name    = "2martens.de"
+  name    = local.domain
   content = "ns3.inwx.eu"
   type    = "NS"
   ttl     = 86400
@@ -109,11 +109,15 @@ resource "inwx_nameserver_record" "twomartens_eu_ns3" {
 // SOAs
 
 resource "inwx_nameserver_record" "twomartens_de_soa" {
-  domain  = "2martens.de"
+  domain  = local.domain
-  name    = "2martens.de"
+  name    = local.domain
   content = "ns.inwx.de hostmaster.inwx.de 2023103008 10800 3600 604800 3600"
   type    = "SOA"
   ttl     = 86400
+
+  lifecycle {
+    ignore_changes = [content]
+  }
 }
 
 resource "inwx_nameserver_record" "twomartens_eu_soa" {
@@ -122,6 +126,10 @@ resource "inwx_nameserver_record" "twomartens_eu_soa" {
   content = "ns.inwx.de hostmaster.inwx.de 2023103001 10800 3600 604800 3600"
   type    = "SOA"
   ttl     = 86400
+
+  lifecycle {
+    ignore_changes = [content]
+  }
 }
 
 // redirect 2martens.eu to 2martens.de

terraform/hetzner.tf

@@ -1,25 +1,5 @@
 // data sources
 
-data "hcloud_location" "falkenstein" {
-  name = "fsn1"
-}
-
-data "hcloud_location" "nuremberg" {
-  name = "nbg1"
-}
-
-data "hcloud_datacenter" "falkenstein" {
-  name = "fsn1-dc14"
-}
-
-data "hcloud_datacenter" "nuremberg" {
-  name = "nbg1-dc3"
-}
-
-data "hcloud_ssh_key" "macos" {
-  name = "2martens@Jims-Air.fritz.box"
-}
-
 data "hcloud_placement_group" "default" {
   name = "default"
 }
@@ -198,9 +178,16 @@ resource "hcloud_server" "server_k8s_test" {
   }
   firewall_ids = [hcloud_firewall.basic-firewall.id, hcloud_firewall.k8s-firewall.id]
   ssh_keys     = [data.hcloud_ssh_key.macos.id]
-  user_data = templatefile("${path.module}/templates/cloud-init-k8s.tftpl", {
+  user_data = templatefile("${path.module}/modules/k8s-cluster/templates/cloud-init-k8s.tftpl", {
     node_ip : "10.0.0.2",
-    cluster_token : "DkZaYY5JUAptUtq4HbXVb2x4HHD3HP7U"
+    admin_public_ssh_key : format("%s %s", data.hcloud_ssh_key.macos.public_key, data.hcloud_ssh_key.macos.name),
+    terraform_public_ssh_key : var.terraform_public_ssh_key,
+    microk8s_channel : "1.28-strict/stable",
+    admin_user : "2martensAdmin",
+    "microk8s_config" : templatefile("${path.module}/modules/k8s-cluster/templates/microk8s-config.tftpl", {
+      node_ip : "10.0.0.2",
+      api_server_domain : ""
+    })
   })
   shutdown_before_deletion = true
 
@@ -219,13 +206,13 @@ resource "hcloud_server" "server_k8s_test" {
 
 module "drone_domain" {
   source    = "./modules/domain"
-  domain    = "2martens.de"
+  domain    = local.domain
   subdomain = "ci"
   ipv4      = "49.12.69.146"
   ipv6      = "2a01:4f8:c012:1fef::1"
 }
 resource "inwx_nameserver_record" "twomartens_de_ci-ownercheck_txt" {
-  domain  = "2martens.de"
+  domain  = local.domain
   name    = "ownercheck.ci.2martens.de"
   content = "e1eb600b"
   type    = "TXT"

terraform/ingress-apps.tf

@@ -1,6 +1,6 @@
 module "argocd_domain" {
   source    = "./modules/domain"
-  domain    = "2martens.de"
+  domain    = local.domain
   subdomain = "argocd"
   ipv4      = "49.13.77.70"
   ipv6      = "2a01:4f8:c012:4d25::1"
@@ -8,7 +8,7 @@ module "argocd_domain" {
 
 module "api_domain" {
   source    = "./modules/domain"
-  domain    = "2martens.de"
+  domain    = local.domain
   subdomain = "api"
   ipv4      = "49.13.77.70"
   ipv6      = "2a01:4f8:c012:4d25::1"
@@ -16,7 +16,7 @@ module "api_domain" {
 
 module "prometheus_k8s_test_domain" {
   source    = "./modules/domain"
-  domain    = "2martens.de"
+  domain    = local.domain
   subdomain = "prometheus.k8s.test"
   ipv4      = "49.13.77.70"
   ipv6      = "2a01:4f8:c012:4d25::1"

terraform/main.tf

@@ -1,19 +1,55 @@
-terraform {
+data "hcloud_location" "falkenstein" {
-  cloud {
+  name = "fsn1"
-    organization = "2martens"
+}
-
+
-    workspaces {
+data "hcloud_location" "nuremberg" {
-      name = "cloud-configuration"
+  name = "nbg1"
-    }
+}
-  }
+
-  required_providers {
+data "hcloud_datacenter" "falkenstein" {
-    inwx = {
+  name = "fsn1-dc14"
-      source  = "inwx/inwx"
+}
-      version = "1.3.2"
+
-    }
+data "hcloud_datacenter" "nuremberg" {
-    hcloud = {
+  name = "nbg1-dc3"
-      source  = "hetznercloud/hcloud"
+}
-      version = "1.44.1"
+
-    }
+data "hcloud_ssh_key" "macos" {
-  }
+  name = "2martens@Jims-Air.fritz.box"
+}
+
+locals {
+  domain = "2martens.de"
+}
+
+module "test_cluster" {
+  source = "./modules/k8s-cluster"
+
+  admin_ssh_key             = data.hcloud_ssh_key.macos
+  admin_user                = "2martensAdmin"
+  terraform_public_ssh_key  = var.terraform_public_ssh_key
+  terraform_private_ssh_key = var.terraform_private_ssh_key
+  basic_firewall_id         = hcloud_firewall.basic-firewall.id
+  k8s_firewall_id           = hcloud_firewall.k8s-firewall.id
+  locations = [{
+    id              = data.hcloud_location.falkenstein.id,
+    name            = data.hcloud_location.falkenstein.name,
+    datacenter_name = data.hcloud_datacenter.falkenstein.name,
+    network_zone    = data.hcloud_location.falkenstein.network_zone
+    }, {
+    id              = data.hcloud_location.nuremberg.id,
+    name            = data.hcloud_location.nuremberg.name,
+    datacenter_name = data.hcloud_datacenter.nuremberg.name,
+    network_zone    = data.hcloud_location.nuremberg.network_zone
+  }]
+  domain              = local.domain
+  network_id          = hcloud_network.kubernetes-network.id
+  private_node_ips    = ["10.0.0.4", "10.0.0.5", "10.0.0.6"]
+  server_subnet_id    = hcloud_network_subnet.k8s-network-subnet.id
+  cluster_role        = "test"
+  number_nodes        = 1
+  server_type         = "cax21"
+  image_name          = "ubuntu-22.04"
+  create_loadbalancer = false
+  loadbalancer_ip     = "10.0.0.3"
 }

terraform/modules/domain/main.tf

@@ -1,12 +1,3 @@
-terraform {
-  required_providers {
-    inwx = {
-      source  = "inwx/inwx"
-      version = "1.3.2"
-    }
-  }
-}
-
 resource "inwx_nameserver_record" "a" {
   domain  = var.domain
   name    = var.subdomain != "" ? format("%s.%s", var.subdomain, var.domain) : var.domain

terraform/modules/domain/versions.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    inwx = {
+      source  = "inwx/inwx"
+      version = "1.3.2"
+    }
+  }
+}

terraform/modules/k8s-cluster/helm.tf

@@ -0,0 +1,18 @@
+resource "null_resource" "install_argocd" {
+  connection {
+    host        = hcloud_server_network.manager_private[0].ip
+    user        = "root"
+    type        = "ssh"
+    private_key = var.terraform_private_ssh_key
+    timeout     = "2m"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "helm repo add 2martens https://repo.2martens.de/charts/",
+      "helm install 2martens/argocd",
+    ]
+  }
+
+  depends_on = [null_resource.join_nodes, hcloud_server.manager]
+}

terraform/modules/k8s-cluster/inwx.tf

@@ -0,0 +1,14 @@
+resource "inwx_nameserver_record" "kube_api_server_a" {
+  domain  = var.domain
+  name    = local.kube_api_server_domain
+  content = var.create_loadbalancer ? hcloud_load_balancer.kubernetes[0].ipv4 : hcloud_primary_ip.ipv4_manager_address[0].ip_address
+  type    = "A"
+  ttl     = 3600
+}
+resource "inwx_nameserver_record" "kube_api_server_aaaa" {
+  domain  = var.domain
+  name    = local.kube_api_server_domain
+  content = var.create_loadbalancer ? hcloud_load_balancer.kubernetes[0].ipv6 : hcloud_primary_ip.ipv6_manager_address[0].ip_address
+  type    = "AAAA"
+  ttl     = 3600
+}

terraform/modules/k8s-cluster/main.tf

@@ -0,0 +1,37 @@
+data "hcloud_placement_group" "default" {
+  name = "default"
+}
+
+locals {
+  kube_api_server_domain = format("%s.%s.%s", "k8s", var.cluster_role, var.domain)
+}
+
+resource "hcloud_load_balancer" "kubernetes" {
+  count = var.create_loadbalancer ? 1 : 0
+
+  load_balancer_type = var.loadbalancer_type
+  name               = format("%s_%s", "k8s", var.cluster_role)
+  location           = var.locations[count.index].name
+  network_zone       = var.locations[count.index].network_zone
+  algorithm {
+    type = "round_robin"
+  }
+}
+
+resource "hcloud_load_balancer_network" "private" {
+  count = var.create_loadbalancer ? 1 : 0
+
+  load_balancer_id = hcloud_load_balancer.kubernetes[count.index].id
+  subnet_id        = var.server_subnet_id
+  ip               = var.loadbalancer_ip
+}
+
+resource "hcloud_load_balancer_target" "nodes" {
+  count = var.create_loadbalancer ? var.number_nodes : 0
+
+  type             = "server"
+  load_balancer_id = hcloud_load_balancer.kubernetes[0].id
+  server_id        = hcloud_server.manager[count.index].id
+  use_private_ip   = true
+}
+

terraform/modules/k8s-cluster/manager-node.tf

@@ -0,0 +1,160 @@
+resource "hcloud_primary_ip" "ipv4_manager_address" {
+  count = var.create_loadbalancer ? 0 : var.number_nodes
+
+  name          = format("%s_%s_%s_%d", "k8s", var.cluster_role, "ipv4", count.index)
+  datacenter    = element(var.locations.*.datacenter_name, count.index)
+  type          = "ipv4"
+  assignee_type = "server"
+  auto_delete   = false
+}
+
+resource "hcloud_primary_ip" "ipv6_manager_address" {
+  count = var.create_loadbalancer ? 0 : var.number_nodes
+
+  name          = format("%s_%s_%s_%d", "k8s", var.cluster_role, "ipv6", count.index)
+  datacenter    = element(var.locations.*.datacenter_name, count.index)
+  type          = "ipv6"
+  assignee_type = "server"
+  auto_delete   = false
+}
+
+resource "hcloud_server_network" "manager_private" {
+  count = var.number_nodes
+
+  server_id = hcloud_server.manager[count.index].id
+  subnet_id = var.server_subnet_id
+  ip        = var.private_node_ips[count.index]
+}
+
+resource "hcloud_server" "manager" {
+  count = var.number_nodes
+
+  name                    = format("%s_%s_%s_%d", "k8s", var.cluster_role, "node", count.index)
+  image                   = var.image_name
+  allow_deprecated_images = false
+  server_type             = var.server_type
+  location                = element(var.locations.*.name, count.index)
+  public_net {
+    ipv4_enabled = var.create_loadbalancer ? false : true
+    ipv4         = var.create_loadbalancer ? null : hcloud_primary_ip.ipv4_manager_address[count.index].id
+    ipv6_enabled = var.create_loadbalancer ? false : true
+    ipv6         = var.create_loadbalancer ? null : hcloud_primary_ip.ipv6_manager_address[count.index].id
+  }
+  ignore_remote_firewall_ids = false
+  keep_disk                  = false
+  placement_group_id         = data.hcloud_placement_group.default.id
+  labels = {
+    "kubernetes" : "yes",
+    "manager" : "yes"
+  }
+  firewall_ids = [var.basic_firewall_id, var.k8s_firewall_id]
+  ssh_keys     = [var.admin_ssh_key.id]
+  user_data = templatefile("${path.module}/templates/cloud-init-k8s.tftpl", {
+    node_ip : var.private_node_ips[count.index],
+    admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name),
+    terraform_public_ssh_key : var.terraform_public_ssh_key,
+    microk8s_channel : var.microk8s_channel,
+    admin_user : var.admin_user,
+    microk8s_config : templatefile("${path.module}/templates/microk8s-config.tftpl", {
+      node_ip : var.private_node_ips[count.index],
+      api_server_domain : local.kube_api_server_domain
+    })
+  })
+  shutdown_before_deletion = true
+
+  network {
+    network_id = var.network_id
+    ip         = var.private_node_ips[count.index]
+    alias_ips  = []
+  }
+
+  lifecycle {
+    ignore_changes = [ssh_keys, user_data]
+  }
+}
+
+resource "random_id" "cluster_token" {
+  byte_length = 16
+}
+
+resource "null_resource" "setup_tokens" {
+  count = var.number_nodes > 1 ? 1 : 0
+
+  triggers = {
+    rerun = random_id.cluster_token.hex
+  }
+
+  connection {
+    host        = hcloud_server_network.manager_private[0].ip
+    user        = "root"
+    type        = "ssh"
+    private_key = var.terraform_private_ssh_key
+    timeout     = "2m"
+  }
+
+  provisioner "local-exec" {
+    interpreter = ["bash", "-c"]
+    command     = <<EOT
+        echo "1" > /tmp/current_joining_node.txt
+        echo "0" > /tmp/current_joining_worker_node.txt
+        EOT
+  }
+
+  provisioner "file" {
+    content = templatefile("${path.module}/templates/add-node.sh",
+      {
+        cluster_token             = random_id.cluster_token.hex
+        cluster_token_ttl_seconds = var.cluster_token_ttl_seconds
+    })
+    destination = "/usr/local/bin/add-node.sh"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "sh /usr/local/bin/add-node.sh",
+    ]
+  }
+}
+
+
+resource "null_resource" "join_nodes" {
+  count = var.number_nodes - 1 < 1 ? 0 : var.number_nodes - 1
+
+  triggers = {
+    rerun = random_id.cluster_token.hex
+  }
+  connection {
+    host        = element(hcloud_server_network.manager_private.*.ip, count.index + 1)
+    user        = "root"
+    type        = "ssh"
+    private_key = var.terraform_private_ssh_key
+    timeout     = "20m"
+  }
+
+  provisioner "local-exec" {
+    interpreter = ["bash", "-c"]
+    command     = "while [[ $(cat /tmp/current_joining_node.txt) != \"${count.index + 1}\" ]]; do echo \"${count.index + 1} is waiting...\";sleep 5;done"
+  }
+
+  provisioner "file" {
+    content = templatefile("${path.module}/templates/join.sh",
+      {
+        cluster_token = random_id.cluster_token.hex
+        main_node_ip  = hcloud_server_network.manager_private[0].ip
+    })
+    destination = "/usr/local/bin/join.sh"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "sh /usr/local/bin/join.sh"
+    ]
+  }
+
+  provisioner "local-exec" {
+    interpreter = ["bash", "-c"]
+    command     = "echo \"${count.index + 2}\" > /tmp/current_joining_node.txt"
+  }
+
+  depends_on = [null_resource.setup_tokens]
+}

terraform/modules/k8s-cluster/templates/add-node.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+until microk8s.status --wait-ready;
+  do sleep 5; echo "waiting for worker status..";
+done
+
+sleep 10
+microk8s add-node --token ${cluster_token} --token-ttl ${cluster_token_ttl_seconds}

terraform/modules/k8s-cluster/templates/cloud-init-k8s.tftpl

@@ -1,16 +1,24 @@
 #cloud-config
 users:
-  - name: 2martensAdmin
+  - name: ${admin_user}
     groups: users, admin
     sudo: ALL=(ALL) NOPASSWD:ALL
     shell: /bin/bash
     ssh_authorized_keys:
-      - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBjYkz73HbsFtRAOfS5QB9xDbOajU9C4gpDp+OjQkLir 2martens@Jims-Air.fritz.box"
+      - ${admin_public_ssh_key}
+  - name: terraform
+    groups: users, admin
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    ssh_authorized_keys:
+      - ${terraform_public_ssh_key}
 packages:
   - fail2ban
   - ufw
 package_update: true
 package_upgrade: true
+snap:
+  commands:
+  - snap install kubectl --classic
 runcmd:
   - printf "[sshd]\nenabled = true\nbanaction = iptables-multiport" > /etc/fail2ban/jail.local
   - systemctl enable fail2ban
@@ -30,12 +38,14 @@ runcmd:
   - sed -i -e '/^\(#\|\)AllowTcpForwarding/s/^.*$/AllowTcpForwarding no/' /etc/ssh/sshd_config
   - sed -i -e '/^\(#\|\)AllowAgentForwarding/s/^.*$/AllowAgentForwarding no/' /etc/ssh/sshd_config
   - sed -i -e '/^\(#\|\)AuthorizedKeysFile/s/^.*$/AuthorizedKeysFile .ssh\/authorized_keys/' /etc/ssh/sshd_config
-  - sed -i '$a AllowUsers 2martensAdmin' /etc/ssh/sshd_config
+  - sed -i '$a AllowUsers ${admin_user} terraform' /etc/ssh/sshd_config
   - curl https://packages.hetzner.com/hcloud/deb/hc-utils_0.0.4-1_all.deb -o /tmp/hc-utils_0.0.4-1_all.deb -s
   - apt install /tmp/hc-utils_0.0.4-1_all.deb
   - mkdir -p /var/snap/microk8s/common/
-  - curl https://repo.2martens.de/files/microk8s-config.yaml -o /var/snap/microk8s/common/.microk8s.yaml
+  - echo ${microk8s_config} > /var/snap/microk8s/common/.microk8s.yaml
-  - sed -i -e '/\$NODE_IP/s//${node_ip}/' /var/snap/microk8s/common/.microk8s.yaml
+  - snap install microk8s --channel ${microk8s_channel}
-  - sed -i -e '/\$CLUSTER_TOKEN/s//${cluster_token}/' /var/snap/microk8s/common/.microk8s.yaml
+  - mkdir -p /home/terraform/.kube
-  - snap install microk8s --channel 1.28-strict/stable
+  - microk8s kubectl config view --raw > /home/terraform/.kube/config
-  - reboot
+  - chown -R terraform:terraform /home/terraform/.kube
+  - chmod go-r /home/terraform/.kube/config
+  - microk8s.status --wait-ready

terraform/modules/k8s-cluster/templates/join-worker.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+until microk8s.status --wait-ready;
+  do sleep 3; echo "waiting for worker status..";
+done
+
+
+if microk8s status | grep "datastore master nodes: 127.0.0.1:19001" > /dev/null 2>&1; then
+  microk8s join ${main_node_ip}:25000/${cluster_token} --worker
+else
+  echo "Join process already done. Nothing to do"
+fi

terraform/modules/k8s-cluster/templates/join.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+until microk8s.status --wait-ready;
+  do sleep 3; echo "waiting for worker status..";
+done
+
+
+if microk8s status | grep "datastore master nodes: 127.0.0.1:19001" > /dev/null 2>&1; then
+  microk8s join "${main_node_ip}:25000/${cluster_token}"
+else
+  echo "Join process already done. Nothing to do"
+fi

terraform/modules/k8s-cluster/templates/microk8s-config.tftpl

@@ -0,0 +1,20 @@
+---
+version: 0.1.0
+extraKubeAPIServerArgs:
+  --advertise-address=${node_ip}
+  --authorization-mode=RBAC,Node
+extraKubeletArgs:
+  --node-ip=${node_ip}
+  --cluster-dns=10.152.183.10
+  --cluster-domain=cluster.local
+extraSANs:
+  - ${api_server_domain}
+addons:
+  - name: dns
+  - name: cis-hardening
+  - name: community
+  - name: helm3
+  - name: ingress
+  - name: ha-cluster
+  - name: rbac
+  - name: cert-manager

terraform/modules/k8s-cluster/variables.tf

@@ -0,0 +1,130 @@
+variable "admin_ssh_key" {
+  description = "SSH key to provide to each server for the admin"
+  type = object({
+    id : number,
+    name : string
+    public_key : string
+  })
+}
+
+variable "admin_user" {
+  description = "Linux username for the admin user on each server"
+  type        = string
+}
+
+variable "terraform_private_ssh_key" {
+  description = "Private SSH key to be used by Terraform to connect with remote machines"
+  type        = string
+  sensitive   = true
+}
+
+variable "terraform_public_ssh_key" {
+  description = "Public SSH key for the private key used by Terraform to connect with remote machines"
+  type        = string
+}
+
+variable "locations" {
+  description = "Locations in which the resources should be created. Singular resources will always be created in the first. Must be all in the same network zone."
+  type = list(object({
+    id : number,
+    name : string,
+    datacenter_name : string,
+    network_zone : string
+  }))
+}
+
+variable "domain" {
+  description = "The registered domain that should be used as the base for all subdomains."
+  type        = string
+}
+
+variable "network_id" {
+  description = "ID of the private network used to connect servers with each other and the loadbalancer"
+  type        = number
+}
+
+variable "server_subnet_id" {
+  description = "ID of the subnet of the private network that is supposed to house the servers and loadbalancer"
+  type        = string
+}
+
+variable "private_node_ips" {
+  description = "One IP in the private network subnet for each manager node created."
+  type        = list(string)
+}
+
+variable "private_worker_node_ips" {
+  description = "One IP in the private network subnet for each worker node created."
+  type        = list(string)
+  default     = []
+}
+
+variable "basic_firewall_id" {
+  description = "ID of the basic firewall for each server"
+  type        = number
+}
+
+variable "k8s_firewall_id" {
+  description = "ID of the K8s specific firewall"
+  type        = number
+}
+
+variable "cluster_role" {
+  description = "The role the cluster serves. For example test, qs, prod. Will be used for names and subdomains and must be unique within the Hetzner project."
+  type        = string
+  default     = "test"
+}
+
+variable "number_nodes" {
+  description = "Number of manager nodes in the cluster."
+  type        = number
+  default     = 1
+}
+
+variable "number_worker_nodes" {
+  description = "Number of worker nodes in the cluster."
+  type        = number
+  default     = 0
+}
+
+variable "image_name" {
+  description = "Name of the image to use for the servers."
+  type        = string
+  default     = "ubuntu-22.04"
+}
+
+variable "server_type" {
+  description = "Hetzner server type like cax21 (default)"
+  type        = string
+  default     = "cax21"
+}
+
+variable "create_loadbalancer" {
+  description = "Specifies if a loadbalancer is created as gateway in front of the k8s servers. If false, the first server acts as gateway."
+  type        = bool
+  default     = false
+}
+
+variable "loadbalancer_type" {
+  description = "Type of the loadbalancer used. By default lb11 is used."
+  type        = string
+  default     = "lb11"
+}
+
+variable "loadbalancer_ip" {
+  description = "The private IP of the loadbalancer. Variable contains valid IP but must be overriden if a loadbalancer is used."
+  type        = string
+  default     = "0.0.0.0"
+}
+
+variable "cluster_token_ttl_seconds" {
+  description = "Number of seconds the join token is valid"
+  type        = number
+  default     = 3600
+}
+
+variable "microk8s_channel" {
+  description = "The snap channel used for the microk8s installation"
+  type        = string
+  default     = "1.28-strict/stable"
+}

terraform/modules/k8s-cluster/versions.tf

@@ -0,0 +1,17 @@
+terraform {
+  required_providers {
+    helm = {
+      source  = "hashicorp/helm"
+      version = "2.11.0"
+    }
+    inwx = {
+      source  = "inwx/inwx"
+      version = "1.3.2"
+    }
+    hcloud = {
+      source  = "hetznercloud/hcloud"
+      version = "1.44.1"
+    }
+  }
+}
+

terraform/modules/k8s-cluster/worker-node.tf

@@ -0,0 +1,94 @@
+resource "hcloud_server_network" "worker_private" {
+  count = var.number_worker_nodes
+
+  server_id = hcloud_server.worker[count.index].id
+  subnet_id = var.server_subnet_id
+  ip        = var.private_worker_node_ips[count.index]
+}
+
+resource "hcloud_server" "worker" {
+  count = var.number_worker_nodes
+
+  name                    = format("%s_%s_%s_%d", "k8s", var.cluster_role, "node", count.index)
+  image                   = var.image_name
+  allow_deprecated_images = false
+  server_type             = var.server_type
+  location                = element(var.locations.*.name, count.index)
+  public_net {
+    ipv4_enabled = false
+    ipv6_enabled = false
+  }
+  ignore_remote_firewall_ids = false
+  keep_disk                  = false
+  placement_group_id         = data.hcloud_placement_group.default.id
+  labels = {
+    "kubernetes" : "yes",
+    "worker" : "yes"
+  }
+  firewall_ids = [var.basic_firewall_id, var.k8s_firewall_id]
+  ssh_keys     = [var.admin_ssh_key.id]
+  user_data = templatefile("${path.module}/templates/cloud-init-k8s.tftpl", {
+    node_ip : var.private_worker_node_ips[count.index],
+    admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name),
+    terraform_public_ssh_key : var.terraform_public_ssh_key,
+    microk8s_channel : var.microk8s_channel,
+    admin_user : var.admin_user,
+    microk8s_config : templatefile("${path.module}/templates/microk8s-config.tftpl", {
+      node_ip : var.private_worker_node_ips[count.index],
+      api_server_domain : local.kube_api_server_domain
+    })
+  })
+  shutdown_before_deletion = true
+
+  network {
+    network_id = var.network_id
+    ip         = var.private_worker_node_ips[count.index]
+    alias_ips  = []
+  }
+
+  lifecycle {
+    ignore_changes = [ssh_keys, user_data]
+  }
+}
+
+resource "null_resource" "join_workers" {
+  count = var.number_worker_nodes
+
+  triggers = {
+    rerun = random_id.cluster_token.hex
+  }
+  connection {
+    host        = element(hcloud_server_network.worker_private.*.ip, count.index)
+    user        = "root"
+    type        = "ssh"
+    private_key = var.terraform_private_ssh_key
+    timeout     = "20m"
+  }
+
+  provisioner "local-exec" {
+    interpreter = ["bash", "-c"]
+    command     = "while [[ $(cat /tmp/current_joining_worker_node.txt) != \"${count.index}\" ]]; do echo \"${count.index} is waiting...\";sleep 5;done"
+  }
+
+  provisioner "file" {
+    content = templatefile("${path.module}/templates/join-worker.sh",
+      {
+        cluster_token = random_id.cluster_token.hex
+        main_node_ip  = hcloud_server_network.manager_private[0].ip
+    })
+    destination = "/usr/local/bin/join-worker.sh"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "sh /usr/local/bin/join-worker.sh"
+    ]
+  }
+
+  provisioner "local-exec" {
+    interpreter = ["bash", "-c"]
+    command     = "echo \"${count.index + 1}\" > /tmp/current_joining_worker_node.txt"
+  }
+
+  depends_on = [null_resource.setup_tokens, null_resource.join_nodes]
+}

terraform/nameserver_records.tf

@@ -1,6 +1,6 @@
 // A records
 resource "inwx_nameserver_record" "twomartens_de_allris_a" {
-  domain  = "2martens.de"
+  domain  = local.domain
   name    = "allris.2martens.de"
   content = "185.26.156.105"
   type    = "A"
@@ -8,7 +8,7 @@ resource "inwx_nameserver_record" "twomartens_de_allris_a" {
 }
 
 resource "inwx_nameserver_record" "twomartens_de_autoconfig_a" {
-  domain  = "2martens.de"
+  domain  = local.domain
   name    = "autoconfig.2martens.de"
   content = "185.26.156.22"
   type    = "A"
@@ -16,7 +16,7 @@ resource "inwx_nameserver_record" "twomartens_de_autoconfig_a" {
 }
 
 resource "inwx_nameserver_record" "twomartens_de_autodiscover_a" {
-  domain  = "2martens.de"
+  domain  = local.domain
   name    = "autodiscover.2martens.de"
   content = "185.26.156.22"
   type    = "A"
@@ -24,7 +24,7 @@ resource "inwx_nameserver_record" "twomartens_de_autodiscover_a" {
 }
 
 resource "inwx_nameserver_record" "twomartens_de_jim_a" {
-  domain  = "2martens.de"
+  domain  = local.domain
   name    = "jim.2martens.de"
   content = "185.26.156.65"
   type    = "A"
@@ -32,7 +32,7 @@ resource "inwx_nameserver_record" "twomartens_de_jim_a" {
 }
 
 resource "inwx_nameserver_record" "twomartens_de_pad_a" {
-  domain  = "2martens.de"
+  domain  = local.domain
   name    = "pad.2martens.de"
   content = "185.26.156.132"
   type    = "A"
@@ -40,7 +40,7 @@ resource "inwx_nameserver_record" "twomartens_de_pad_a" {
 }
 
 resource "inwx_nameserver_record" "twomartens_de_short_a" {
-  domain  = "2martens.de"
+  domain  = local.domain
   name    = "short.2martens.de"
   content = "95.143.172.56"
   type    = "A"
@@ -50,7 +50,7 @@ resource "inwx_nameserver_record" "twomartens_de_short_a" {
 // AAAA records
 
 resource "inwx_nameserver_record" "twomartens_de_allris_aaaa" {
-  domain  = "2martens.de"
+  domain  = local.domain
   name    = "allris.2martens.de"
   content = "2a00:d0c0:200:0:b9:1a:9c:61"
   type    = "AAAA"
@@ -58,7 +58,7 @@ resource "inwx_nameserver_record" "twomartens_de_allris_aaaa" {
 }
 
 resource "inwx_nameserver_record" "twomartens_de_autoconfig_aaaa" {
-  domain  = "2martens.de"
+  domain  = local.domain
   name    = "autoconfig.2martens.de"
   content = "2a00:d0c0:200:0:b9:1a:9c16:5d"
   type    = "AAAA"
@@ -66,7 +66,7 @@ resource "inwx_nameserver_record" "twomartens_de_autoconfig_aaaa" {
 }
 
 resource "inwx_nameserver_record" "twomartens_de_autodiscover_aaaa" {
-  domain  = "2martens.de"
+  domain  = local.domain
   name    = "autodiscover.2martens.de"
   content = "2a00:d0c0:200:0:b9:1a:9c16:5d"
   type    = "AAAA"
@@ -74,7 +74,7 @@ resource "inwx_nameserver_record" "twomartens_de_autodiscover_aaaa" {
 }
 
 resource "inwx_nameserver_record" "twomartens_de_jim_aaaa" {
-  domain  = "2martens.de"
+  domain  = local.domain
   name    = "jim.2martens.de"
   content = "2a00:d0c0:200:0:b9:1a:9c:40"
   type    = "AAAA"
@@ -82,7 +82,7 @@ resource "inwx_nameserver_record" "twomartens_de_jim_aaaa" {
 }
 
 resource "inwx_nameserver_record" "twomartens_de_pad_aaaa" {
-  domain  = "2martens.de"
+  domain  = local.domain
   name    = "pad.2martens.de"
   content = "2a00:d0c0:200:0:c0bf:e0ff:fec1:af72"
   type    = "AAAA"
@@ -90,7 +90,7 @@ resource "inwx_nameserver_record" "twomartens_de_pad_aaaa" {
 }
 
 resource "inwx_nameserver_record" "twomartens_de_short_aaaa" {
-  domain  = "2martens.de"
+  domain  = local.domain
   name    = "short.2martens.de"
   content = "2001:1a50:11:0:3854:d4ff:fe62:e0d8"
   type    = "AAAA"
@@ -100,7 +100,7 @@ resource "inwx_nameserver_record" "twomartens_de_short_aaaa" {
 // CNAME records
 
 resource "inwx_nameserver_record" "twomartens_de_cdn_cname" {
-  domain  = "2martens.de"
+  domain  = local.domain
   name    = "cdn.2martens.de"
   content = "d1fvxyvcoii67h.cloudfront.net"
   type    = "CNAME"
@@ -108,7 +108,7 @@ resource "inwx_nameserver_record" "twomartens_de_cdn_cname" {
 }
 
 resource "inwx_nameserver_record" "twomartens_de_aws-validations-cdn_cname" {
-  domain  = "2martens.de"
+  domain  = local.domain
   name    = "_4741e2d9c8a605950eebfa048029e4ef.cdn.2martens.de"
   content = "_cf71e85bb40cc12717f2718e4590ad23.acm-validations.aws"
   type    = "CNAME"

terraform/uberspace.tf

@@ -1,7 +1,7 @@
 // Identity management
 module "keycloak_domain" {
   source      = "./modules/domain"
-  domain      = "2martens.de"
+  domain      = local.domain
   subdomain   = "id"
   ipv4        = "95.143.172.184"
   ipv6        = "2001:1a50:11:0:98b3:92ff:fe3e:bba1"
@@ -14,7 +14,7 @@ module "keycloak_domain" {
 // Angular frontends
 module "wahlrecht_domain" {
   source    = "./modules/domain"
-  domain    = "2martens.de"
+  domain    = local.domain
   subdomain = "wahlrecht"
   ipv4      = "185.26.156.43"
   ipv6      = "2a00:d0c0:200:0:470:9fff:fe83:16bc"
@@ -23,7 +23,7 @@ module "wahlrecht_domain" {
 // CI/CD
 module "gitea_domain" {
   source    = "./modules/domain"
-  domain    = "2martens.de"
+  domain    = local.domain
   subdomain = "git"
   ipv4      = "185.26.156.91"
   ipv6      = "2a00:d0c0:200:0:b9:1a:9c:5a"
@@ -31,7 +31,7 @@ module "gitea_domain" {
 
 module "helm-repo_domain" {
   source    = "./modules/domain"
-  domain    = "2martens.de"
+  domain    = local.domain
   subdomain = "repo"
   ipv4      = "185.26.156.49"
   ipv6      = "2a00:d0c0:200:0:2ca6:bff:fe78:832f"
@@ -40,7 +40,7 @@ module "helm-repo_domain" {
 // Monitoring
 module "statping_domain" {
   source    = "./modules/domain"
-  domain    = "2martens.de"
+  domain    = local.domain
   subdomain = "status"
   ipv4      = "185.26.156.33"
   ipv6      = "2a00:d0c0:200:0:f8e2:6fff:fec2:7a92"
@@ -49,7 +49,7 @@ module "statping_domain" {
 // Personal Website
 module "personal_website_domain" {
   source      = "./modules/domain"
-  domain      = "2martens.de"
+  domain      = local.domain
   subdomain   = ""
   ipv4        = "185.26.156.65"
   ipv6        = "2a00:d0c0:200:0:b9:1a:9c:40"
@@ -59,8 +59,8 @@ module "personal_website_domain" {
   mxSpf       = "v=spf1 include:spf.uberspace.de"
 }
 resource "inwx_nameserver_record" "twomartens_de_google-verification_txt" {
-  domain  = "2martens.de"
+  domain  = local.domain
-  name    = "2martens.de"
+  name    = local.domain
   content = "google-site-verification=nUFiHQFxBpBMdX96ELH3TcfyIfXf2ZlwMFYtXjVq5lo"
   type    = "TXT"
   ttl     = 3600
@@ -69,7 +69,7 @@ resource "inwx_nameserver_record" "twomartens_de_google-verification_txt" {
 // Nextcloud
 module "nextcloud_domain" {
   source      = "./modules/domain"
-  domain      = "2martens.de"
+  domain      = local.domain
   subdomain   = "cloud"
   ipv4        = "185.26.156.194"
   ipv6        = "2a00:d0c0:200:0:b9:1a:9c:95"
@@ -81,7 +81,7 @@ module "nextcloud_domain" {
 
 // domain key
 resource "inwx_nameserver_record" "twomartens_de_uberspace-domainkey_txt" {
-  domain  = "2martens.de"
+  domain  = local.domain
   name    = "uberspace._domainkey.id.2martens.de"
   content = "v=DKIM1;t=s;n=core;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqs3bFL8W77z7VJHCFbsJ3o68Y1uBbVsbIS1y35CMfRv6wp7si7aIG2ZeKqzdh2dTMsvtvSMaYVq1gT/EtRmTbU/BYC21ff8sqcVqn/ll5nNsk5jKWXTYAlTQp4LBQN7icw94ZNGr5/SDYcnv2nsBYFf2GUviObWXGHX4RaBFNj9NVUNWNin/HicvW+LsbfYq37QtlhjmUn9K96VCwKcTV1mx+Ek0osErYefcOVNawqWIlVRQDLkHZhk1StLsOpRqV+qEjhzTk4n4ZiNtLJG1D9CpHl24d5DKsQDyVFdEfqHimFTSNgiitlkuXg+i+NMiRA9G3gNJtvw8uvfN8f+stQIDAQAB"
   type    = "TXT"

terraform/variables.tf

@@ -37,3 +37,14 @@ variable "hcloud_token" {
   type      = string
   sensitive = true
 }
+
+variable "terraform_private_ssh_key" {
+  description = "Private SSH key to be used by Terraform to connect with remote machines"
+  type        = string
+  sensitive   = true
+}
+
+variable "terraform_public_ssh_key" {
+  description = "Public SSH key for the private key used by Terraform to connect with remote machines"
+  type        = string
+}

terraform/versions.tf

@@ -0,0 +1,19 @@
+terraform {
+  cloud {
+    organization = "2martens"
+
+    workspaces {
+      name = "cloud-configuration"
+    }
+  }
+  required_providers {
+    inwx = {
+      source  = "inwx/inwx"
+      version = "1.3.2"
+    }
+    hcloud = {
+      source  = "hetznercloud/hcloud"
+      version = "1.44.1"
+    }
+  }
+}