feat: Add module for docker-network (WIP)

Adds a first draft of a module for a docker network. Outstanding: Nginx configuration.

terraform/modules/docker-network/main.tf

@@ -0,0 +1,23 @@
+data "hcloud_placement_group" "default" {
+  name = "default"
+}
+
+locals {
+  docker_network_domain = format("%s.%s.%s", "docker", var.stage_name, var.domain)
+}
+
+# TODO: use Hetzner DNS
+# resource "inwx_nameserver_record" "kube_api_server_a" {
+#   domain  = var.domain
+#   name    = local.docker_network_domain
+#   content = hcloud_server.nginx.ipv4_address
+#   type    = "A"
+#   ttl     = 3600
+# }
+# resource "inwx_nameserver_record" "kube_api_server_aaaa" {
+#   domain  = var.domain
+#   name    = local.docker_network_domain
+#   content = hcloud_server.nginx.ipv6_address
+#   type    = "AAAA"
+#   ttl     = 3600
+# }

terraform/modules/docker-network/nginx-vm.tf

@@ -0,0 +1,96 @@
+# TODO: Use Hetzner DNS
+# resource "inwx_nameserver_record" "manager_aaaa" {
+#   domain  = var.domain
+#   name = format("%s.%s", "nginx", local.docker_network_domain)
+#   content = hcloud_server.nginx.ipv6_address
+#   type    = "AAAA"
+#   ttl     = 3600
+# }
+
+resource "hcloud_primary_ip" "ipv4_nginx_address" {
+  name = format("%s_%s_%s", "docker", var.stage_name, "ipv4")
+  datacenter = element(var.locations.*.datacenter_name, 0)
+  type          = "ipv4"
+  assignee_type = "server"
+  auto_delete   = false
+}
+
+resource "hcloud_rdns" "ipv4_nginx" {
+  primary_ip_id = hcloud_primary_ip.ipv4_nginx_address.id
+  ip_address    = hcloud_primary_ip.ipv4_nginx_address.ip_address
+  # TODO: use Hetzner DNS
+  # dns_ptr       = inwx_nameserver_record.manager_aaaa.name
+}
+
+resource "hcloud_primary_ip" "ipv6_nginx_address" {
+  name = format("%s_%s_%s", "k8s", var.stage_name, "ipv6")
+  datacenter = element(var.locations.*.datacenter_name, 0)
+  type          = "ipv6"
+  assignee_type = "server"
+  auto_delete   = false
+}
+
+resource "hcloud_rdns" "ipv6_nginx" {
+  primary_ip_id = hcloud_primary_ip.ipv6_nginx_address.id
+  ip_address    = hcloud_server.nginx.ipv6_address
+  # TODO: use Hetzner DNS
+  # dns_ptr       = inwx_nameserver_record.manager_aaaa.name
+}
+
+resource "hcloud_server_network" "nginx_private" {
+  server_id = hcloud_server.nginx.id
+  subnet_id = var.server_subnet_id
+  ip        = var.private_nginx_vm_ip
+}
+
+resource "hcloud_server" "nginx" {
+  name = format("%s-%s-%s", "docker", var.stage_name, "nginx")
+  image                   = var.image_name
+  allow_deprecated_images = false
+  server_type             = var.server_type
+  location = element(var.locations.*.name, 0)
+
+  public_net {
+    ipv4_enabled = true
+    ipv4         = hcloud_primary_ip.ipv4_nginx_address.id
+    ipv6_enabled = true
+    ipv6         = hcloud_primary_ip.ipv6_nginx_address.id
+  }
+
+  ignore_remote_firewall_ids = false
+  keep_disk                  = false
+  placement_group_id         = data.hcloud_placement_group.default.id
+  firewall_ids = [var.basic_firewall_id, var.docker_firewall_id]
+  ssh_keys = [var.admin_ssh_key.id]
+  shutdown_before_deletion   = true
+
+  labels = {
+    "docker" : "yes",
+    "nginx" : "yes"
+  }
+
+  user_data = templatefile("${path.module}/templates/cloud-init-docker-with-nginx.yaml.tftpl", {
+    admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name)
+    admin_user : var.admin_user
+    terraform_public_ssh_key : var.terraform_public_ssh_key
+    packages_setup : base64encode(file("${path.module}/templates/scripts/install-packages.sh"))
+    firewall_setup : base64encode(templatefile("${path.module}/templates/scripts/firewall-setup.sh", {
+      node_ip : var.private_nginx_vm_ip
+    }))
+    ssh_setup : base64encode(templatefile("${path.module}/templates/scripts/ssh-setup.sh", {
+      admin_user : var.admin_user
+    }))
+    sysctl_setup : base64encode(file("${path.module}/templates/scripts/sysctl-setup.sh"))
+    nginx_setup : base64encode(file("${path.module}/templates/scripts/nginx-setup.sh"))
+  })
+
+  network {
+    network_id = var.network_id
+    ip         = var.private_nginx_vm_ip
+    alias_ips = []
+  }
+
+  lifecycle {
+    ignore_changes = [ssh_keys, user_data]
+  }
+}

terraform/modules/docker-network/outputs.tf

@@ -0,0 +1,7 @@
+output "nginx_server_ipv4_address" {
+  value = hcloud_server.nginx.ipv4_address
+}
+
+output "nginx_server_ipv6_address" {
+  value = hcloud_server.nginx.ipv6_address
+}

terraform/modules/docker-network/templates/cloud-init-docker-no-nginx.yaml.tftpl

@@ -0,0 +1,43 @@
+#cloud-config
+users:
+  - name: ${admin_user}
+    groups: users, admin, microk8s
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    shell: /bin/bash
+    ssh_authorized_keys:
+      - ${admin_public_ssh_key}
+  - name: terraform
+    groups: users, admin
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    ssh_authorized_keys:
+      - ${terraform_public_ssh_key}
+package_update: true
+package_upgrade: true
+package_reboot_if_required: true
+write_files:
+  - owner: root:root
+    content: "${packages_setup}"
+    path: /run/tmpfiles.d/install-packages.sh
+    encoding: base64
+    defer: true
+  - owner: root:root
+    content: "${firewall_setup}"
+    path: /run/tmpfiles.d/firewall-setup.sh
+    encoding: base64
+    defer: true
+  - owner: root:root
+    content: "${ssh_setup}"
+    path: /run/tmpfiles.d/ssh-setup.sh
+    encoding: base64
+    defer: true
+  - owner: root:root
+    content: "${sysctl_setup}"
+    path: /run/tmpfiles.d/sysctl-setup.sh
+    encoding: base64
+    defer: true
+runcmd:
+  - install /run/tmpfiles.d/*.sh /usr/local/bin
+  - /usr/local/bin/install-packages.sh
+  - /usr/local/bin/firewall-setup.sh
+  - /usr/local/bin/ssh-setup.sh
+  - /usr/local/bin/sysctl-setup.sh

terraform/modules/docker-network/templates/cloud-init-docker-with-nginx.yaml.tftpl

@@ -11,22 +11,10 @@ users:
     sudo: ALL=(ALL) NOPASSWD:ALL
     ssh_authorized_keys:
       - ${terraform_public_ssh_key}
-packages:
-  - linux-generic-hwe-22.04
 package_update: true
 package_upgrade: true
 package_reboot_if_required: true
 write_files:
-  - owner: root:root
-    content: "${microk8s_config}"
-    path: /run/tmpfiles.d/microk8s.yaml
-    encoding: base64
-    defer: true
-  - owner: root:root
-    content: "${snapcraft}"
-    path: /run/tmpfiles.d/snapcraft.yaml
-    encoding: base64
-    defer: true
   - owner: root:root
     content: "${packages_setup}"
     path: /run/tmpfiles.d/install-packages.sh
@@ -48,8 +36,8 @@ write_files:
     encoding: base64
     defer: true
   - owner: root:root
-    content: "${microk8s_setup}"
+    content: "${nginx_setup}"
-    path: /run/tmpfiles.d/microk8s-setup.sh
+    path: /run/tmpfiles.d/nginx-setup.sh
     encoding: base64
     defer: true
 runcmd:
@@ -58,5 +46,4 @@ runcmd:
   - /usr/local/bin/firewall-setup.sh
   - /usr/local/bin/ssh-setup.sh
   - /usr/local/bin/sysctl-setup.sh
-  - /usr/local/bin/microk8s-setup.sh
+  - /usr/local/bin/nginx-setup.sh
-

terraform/modules/docker-network/templates/scripts/firewall-setup.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+printf "[sshd]\nenabled = true\nbanaction = iptables-multiport" > /etc/fail2ban/jail.local
+systemctl enable fail2ban
+
+ufw allow OpenSSH
+ufw allow 80 comment HTTP
+ufw allow 443 comment HTTPS
+ufw enable

terraform/modules/docker-network/templates/scripts/install-packages.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+# Add Docker's official GPG key:
+apt-get update
+apt-get -y install ca-certificates curl
+install -m 0755 -d /etc/apt/keyrings
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+chmod a+r /etc/apt/keyrings/docker.asc
+
+# Add the repository to Apt sources:
+echo \
+  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+  $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
+  tee /etc/apt/sources.list.d/docker.list > /dev/null
+apt-get update
+
+apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+
+apt-get -y install fail2ban
+apt-get -y install ufw
+
+curl https://packages.hetzner.com/hcloud/deb/hc-utils_0.0.4-1_all.deb -o /run/tmpfiles.d/hc-utils_0.0.4-1_all.deb -s
+apt-get -y install /run/tmpfiles.d/hc-utils_0.0.4-1_all.deb

terraform/modules/docker-network/templates/scripts/nginx-setup.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+apt-get -y install gnupg2 ubuntu-keyring
+
+curl https://nginx.org/keys/nginx_signing.key | gpg --dearmor \
+    | tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null
+
+echo \
+  "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] http://nginx.org/packages/mainline/ubuntu \
+  $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") nginx" | \
+  tee /etc/apt/sources.list.d/nginx.list
+
+echo -e "Package: *\nPin: origin nginx.org\nPin: release o=nginx\nPin-Priority: 900\n" \
+    | tee /etc/apt/preferences.d/99nginx
+
+apt-get update
+apt-get -y install nginx
+
+# Remove the default Nginx configuration file
+rm /etc/nginx/sites-enabled/default
+
+# Enable the Nginx site configuration
+ln -s /etc/nginx/sites-available/default /etc/nginx/sites-enabled/default
+
+nginx

terraform/modules/docker-network/templates/scripts/ssh-setup.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+sed -i -e '/^\(#\|\)PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
+sed -i -e '/^\(#\|\)PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
+sed -i -e '/^\(#\|\)X11Forwarding/s/^.*$/X11Forwarding no/' /etc/ssh/sshd_config
+sed -i -e '/^\(#\|\)MaxAuthTries/s/^.*$/MaxAuthTries 2/' /etc/ssh/sshd_config
+sed -i -e '/^\(#\|\)AllowTcpForwarding/s/^.*$/AllowTcpForwarding no/' /etc/ssh/sshd_config
+sed -i -e '/^\(#\|\)AllowAgentForwarding/s/^.*$/AllowAgentForwarding no/' /etc/ssh/sshd_config
+sed -i -e '/^\(#\|\)AuthorizedKeysFile/s/^.*$/AuthorizedKeysFile .ssh\/authorized_keys/' /etc/ssh/sshd_config
+sed -i '$a AllowUsers ${admin_user} terraform' /etc/ssh/sshd_config

terraform/modules/docker-network/templates/scripts/sysctl-setup.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+printf "vm.overcommit_memory = 1\nkernel.panic = 10\nkernel.panic_on_oops = 1" > /etc/sysctl.d/90-kubelet.conf

terraform/modules/docker-network/variables.tf

@@ -0,0 +1,94 @@
+variable "admin_ssh_key" {
+  description = "SSH key to provide to each server for the admin"
+  type = object({
+    id : number,
+    name : string
+    public_key : string
+  })
+}
+
+variable "admin_user" {
+  description = "Linux username for the admin user on each server"
+  type        = string
+}
+
+variable "terraform_private_ssh_key" {
+  description = "Private SSH key to be used by Terraform to connect with remote machines"
+  type        = string
+  sensitive   = true
+}
+
+variable "terraform_public_ssh_key" {
+  description = "Public SSH key for the private key used by Terraform to connect with remote machines"
+  type        = string
+}
+
+variable "locations" {
+  description = "Locations in which the resources should be created. Singular resources will always be created in the first. Must be all in the same network zone."
+  type = list(object({
+    id : number,
+    name : string,
+    datacenter_name : string,
+    network_zone : string
+  }))
+}
+
+variable "domain" {
+  description = "The registered domain that should be used as the base for all subdomains."
+  type        = string
+}
+
+variable "network_id" {
+  description = "ID of the private network used to connect servers with each other"
+  type        = number
+}
+
+variable "server_subnet_id" {
+  description = "ID of the subnet of the private network that is supposed to house the servers"
+  type        = string
+}
+
+variable "private_nginx_vm_ip" {
+  description = "IP in the private network subnet for the VM containing Nginx"
+  type        = string
+}
+
+variable "private_worker_vm_ips" {
+  description = "One IP in the private network subnet for each VM without Nginx created."
+  type = list(string)
+}
+
+variable "basic_firewall_id" {
+  description = "ID of the basic firewall for each server"
+  type        = number
+}
+
+variable "docker_firewall_id" {
+  description = "ID of the docker specific firewall"
+  type        = number
+}
+
+variable "stage_name" {
+  description = "The name of the stage. For example test, qs, prod. Will be used for names and subdomains and must be unique within the Hetzner project (identified by API token)."
+  type        = string
+  default     = "test"
+}
+
+variable "number_worker_vms" {
+  description = "Number of worker VMs in the docker network."
+  type        = number
+  default     = 0
+}
+
+variable "image_name" {
+  description = "Name of the image to use for the servers."
+  type        = string
+  default     = "ubuntu-24.04"
+}
+
+variable "server_type" {
+  description = "Hetzner server type like cax21 (default)"
+  type        = string
+  default     = "cax21"
+}
+

terraform/modules/docker-network/versions.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_providers {
+    hcloud = {
+      source  = "hetznercloud/hcloud"
+      version = "1.44.1"
+    }
+  }
+}
+

terraform/modules/docker-network/worker-vm.tf

@@ -0,0 +1,86 @@
+# TODO: Use Hetzner DNS
+# resource "inwx_nameserver_record" "worker_aaaa" {
+#   count = var.number_worker_vms
+#
+#   domain  = var.domain
+#   name    = format("%s-%d.%s", "node", count.index, local.docker_network_domain)
+#   content = hcloud_server.worker[count.index].ipv6_address
+#   type    = "AAAA"
+#   ttl     = 3600
+# }
+
+resource "hcloud_primary_ip" "ipv6_worker_address" {
+  count = var.number_worker_vms
+
+  name          = format("%s_%s_%s_%d", "docker", var.stage_name, "ipv6", count.index)
+  datacenter    = element(var.locations.*.datacenter_name, count.index)
+  type          = "ipv6"
+  assignee_type = "server"
+  auto_delete   = false
+}
+
+resource "hcloud_rdns" "ipv6_worker" {
+  count = var.number_worker_vms
+
+  primary_ip_id = hcloud_primary_ip.ipv6_worker_address[count.index].id
+  ip_address    = hcloud_server.worker[count.index].ipv6_address
+  # TODO: Use Hetzner DNS
+  # dns_ptr       = inwx_nameserver_record.worker_aaaa[count.index].name
+}
+
+resource "hcloud_server_network" "worker_private" {
+  count = var.number_worker_vms
+
+  server_id = hcloud_server.worker[count.index].id
+  subnet_id = var.server_subnet_id
+  ip        = var.private_worker_vm_ips[count.index]
+}
+
+resource "hcloud_server" "worker" {
+  count = var.number_worker_vms
+
+  name                    = format("%s-%s-%s-%d", "docker", var.stage_name, "node", count.index)
+  image                   = var.image_name
+  allow_deprecated_images = false
+  server_type             = var.server_type
+  location                = element(var.locations.*.name, count.index)
+  public_net {
+    ipv4_enabled = false
+    ipv6_enabled = true
+    ipv6         = hcloud_primary_ip.ipv6_worker_address[count.index].id
+  }
+  ignore_remote_firewall_ids = false
+  keep_disk                  = false
+  placement_group_id         = data.hcloud_placement_group.default.id
+  firewall_ids               = [var.basic_firewall_id, var.docker_firewall_id]
+  ssh_keys                   = [var.admin_ssh_key.id]
+  shutdown_before_deletion   = true
+
+  labels = {
+    "docker" : "yes"
+  }
+
+  user_data = templatefile("${path.module}/templates/cloud-init-docker-no-nginx.yaml.tftpl", {
+    admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name)
+    admin_user : var.admin_user
+    terraform_public_ssh_key : var.terraform_public_ssh_key
+    packages_setup : base64encode(file("${path.module}/templates/scripts/install-packages.sh"))
+    firewall_setup : base64encode(templatefile("${path.module}/templates/scripts/firewall-setup.sh", {
+      node_ip : var.private_worker_vm_ips[count.index]
+    }))
+    ssh_setup : base64encode(templatefile("${path.module}/templates/scripts/ssh-setup.sh", {
+      admin_user : var.admin_user
+    }))
+    sysctl_setup : base64encode(file("${path.module}/templates/scripts/sysctl-setup.sh"))
+  })
+
+  network {
+    network_id = var.network_id
+    ip         = var.private_worker_vm_ips[count.index]
+    alias_ips  = []
+  }
+
+  lifecycle {
+    ignore_changes = [ssh_keys, user_data]
+  }
+}

terraform/modules/k8s-cluster/templates/cloud-init-k8s-manager.tftpl

@@ -1,91 +0,0 @@
-#cloud-config
-users:
-  - name: ${admin_user}
-    groups: users, admin, microk8s
-    sudo: ALL=(ALL) NOPASSWD:ALL
-    shell: /bin/bash
-    ssh_authorized_keys:
-      - ${admin_public_ssh_key}
-  - name: terraform
-    groups: users, admin
-    sudo: ALL=(ALL) NOPASSWD:ALL
-    ssh_authorized_keys:
-      - ${terraform_public_ssh_key}
-packages:
-  - linux-generic-hwe-22.04
-package_update: true
-package_upgrade: true
-package_reboot_if_required: true
-write_files:
-  - owner: root:root
-    content: "${microk8s_config}"
-    path: /run/tmpfiles.d/microk8s.yaml
-    encoding: base64
-    defer: true
-  - owner: root:root
-    content: "${snapcraft}"
-    path: /run/tmpfiles.d/snapcraft.yaml
-    encoding: base64
-    defer: true
-  - owner: root:root
-    content: "${packages_setup}"
-    path: /run/tmpfiles.d/install-packages.sh
-    encoding: base64
-    defer: true
-  - owner: root:root
-    content: "${firewall_setup}"
-    path: /run/tmpfiles.d/firewall-setup.sh
-    encoding: base64
-    defer: true
-  - owner: root:root
-    content: "${ssh_setup}"
-    path: /run/tmpfiles.d/ssh-setup.sh
-    encoding: base64
-    defer: true
-  - owner: root:root
-    content: "${sysctl_setup}"
-    path: /run/tmpfiles.d/sysctl-setup.sh
-    encoding: base64
-    defer: true
-  - owner: root:root
-    content: "${microk8s_setup}"
-    path: /run/tmpfiles.d/microk8s-setup.sh
-    encoding: base64
-    defer: true
-%{ if main_node ~}
-  - owner: root:root
-    content: "${cluster_setup}"
-    path: /run/tmpfiles.d/cluster-setup.sh
-    encoding: base64
-    defer: true
-  - owner: root:root
-    content: "${cluster_setup_values}"
-    path: /run/tmpfiles.d/cluster-setup-values.yaml
-    encoding: base64
-    defer: true
-  - owner: root:root
-    content: "${argocd_ha_values}"
-    path: /run/tmpfiles.d/argocd-values-ha.yaml
-    encoding: base64
-    defer: true
-  - owner: root:root
-    content: "${argocd_values}"
-    path: /run/tmpfiles.d/argocd-values.yaml
-    encoding: base64
-    defer: true
-  - owner: root:root
-    content: "${cilium_values}"
-    path: /run/tmpfiles.d/cilium-values.yaml
-    encoding: base64
-    defer: true
-%{ endif ~}
-runcmd:
-  - install /run/tmpfiles.d/*.sh /usr/local/bin
-  - /usr/local/bin/install-packages.sh
-  - /usr/local/bin/firewall-setup.sh
-  - /usr/local/bin/ssh-setup.sh
-  - /usr/local/bin/sysctl-setup.sh
-  - /usr/local/bin/microk8s-setup.sh
-%{ if main_node ~}
-  - sudo /usr/local/bin/cluster-setup.sh
-%{ endif ~}