change: Add support for manager and worker nodes in Docker swarm

change: Extract port 80/443 to separate firewalls

terraform/hetzner.tf

@@ -32,6 +32,20 @@ resource "hcloud_firewall" "basic-firewall" {
     ]
   }
 
+  rule {
+    direction   = "in"
+    protocol    = "icmp"
+    description = "allow ICMP"
+    source_ips = [
+      "0.0.0.0/0",
+      "::/0",
+    ]
+  }
+}
+
+resource "hcloud_firewall" "http-firewall" {
+  name = "http-firewall"
+
   rule {
     direction   = "in"
     protocol    = "tcp"
@@ -53,16 +67,6 @@ resource "hcloud_firewall" "basic-firewall" {
       "::/0",
     ]
   }
-
-  rule {
-    direction   = "in"
-    protocol    = "icmp"
-    description = "allow ICMP"
-    source_ips = [
-      "0.0.0.0/0",
-      "::/0",
-    ]
-  }
 }
 
 resource "hcloud_firewall" "k8s-firewall" {

terraform/main.tf

@@ -33,6 +33,7 @@ module "test_cluster" {
   terraform_public_ssh_key  = var.terraform_public_ssh_key
   terraform_private_ssh_key = var.terraform_private_ssh_key
   basic_firewall_id         = hcloud_firewall.basic-firewall.id
+  http_firewall_id          = hcloud_firewall.http-firewall.id
   k8s_firewall_id           = hcloud_firewall.k8s-firewall.id
   locations = [
     {
@@ -40,7 +41,7 @@ module "test_cluster" {
       name            = data.hcloud_location.falkenstein.name,
       datacenter_name = data.hcloud_datacenter.falkenstein.name,
       network_zone    = data.hcloud_location.falkenstein.network_zone
-    }, {
+      }, {
       id              = data.hcloud_location.nuremberg.id,
       name            = data.hcloud_location.nuremberg.name,
       datacenter_name = data.hcloud_datacenter.nuremberg.name,
@@ -61,12 +62,12 @@ module "test_cluster" {
   server_type         = "cax21"
   image_name          = "ubuntu-22.04"
   create_loadbalancer = false
-  loadbalancer_ip = cidrhost(hcloud_network_subnet.k8s-network-subnet.ip_range, 3)
+  loadbalancer_ip     = cidrhost(hcloud_network_subnet.k8s-network-subnet.ip_range, 3)
   vault_service_principal = {
     client_id : var.vault_client_id
     client_secret : var.vault_client_secret
   }
-  vault_allowed_namespaces = ["wahlrecht", "timetable", "hcloud", "routing"]
+  vault_allowed_namespaces      = ["wahlrecht", "timetable", "hcloud", "routing"]
   hcloud_token_enabled          = false
   hcloud_token                  = var.hcloud_token
   thanos_enabled                = true
@@ -86,6 +87,7 @@ module "monitoring_cluster" {
   terraform_public_ssh_key  = var.terraform_public_ssh_key
   terraform_private_ssh_key = var.terraform_private_ssh_key
   basic_firewall_id         = hcloud_firewall.basic-firewall.id
+  http_firewall_id          = hcloud_firewall.http-firewall.id
   k8s_firewall_id           = hcloud_firewall.k8s-firewall.id
   locations = [
     {
@@ -93,7 +95,7 @@ module "monitoring_cluster" {
       name            = data.hcloud_location.falkenstein.name,
       datacenter_name = data.hcloud_datacenter.falkenstein.name,
       network_zone    = data.hcloud_location.falkenstein.network_zone
-    }, {
+      }, {
       id              = data.hcloud_location.nuremberg.id,
       name            = data.hcloud_location.nuremberg.name,
       datacenter_name = data.hcloud_datacenter.nuremberg.name,
@@ -112,12 +114,12 @@ module "monitoring_cluster" {
   server_type         = "cax21"
   image_name          = "ubuntu-22.04"
   create_loadbalancer = false
-  loadbalancer_ip = cidrhost(hcloud_network_subnet.k8s-network-subnet.ip_range, 3)
+  loadbalancer_ip     = cidrhost(hcloud_network_subnet.k8s-network-subnet.ip_range, 3)
   vault_service_principal = {
     client_id : var.vault_client_id
     client_secret : var.vault_client_secret
   }
-  vault_allowed_namespaces = ["hcloud"]
+  vault_allowed_namespaces      = ["hcloud"]
   hcloud_token_enabled          = false
   hcloud_token                  = var.hcloud_token
   thanos_enabled                = true
@@ -131,20 +133,20 @@ module "monitoring_cluster" {
 module "test_swarm_cluster" {
   source = "./modules/docker-network"
 
-  admin_ssh_key             = data.hcloud_ssh_key.macos
-  admin_user                = "2martensAdmin"
-  terraform_public_ssh_key  = var.terraform_public_ssh_key
-  terraform_private_ssh_key = var.terraform_private_ssh_key
-  github_public_ssh_key     = var.github_public_ssh_key
-  basic_firewall_id         = hcloud_firewall.basic-firewall.id
-  domain                    = local.domain
+  admin_ssh_key            = data.hcloud_ssh_key.macos
+  admin_user               = "2martensAdmin"
+  terraform_public_ssh_key = var.terraform_public_ssh_key
+  github_public_ssh_key    = var.github_public_ssh_key
+  basic_firewall_id        = hcloud_firewall.basic-firewall.id
+  http_firewall_id         = hcloud_firewall.http-firewall.id
+  domain                   = local.domain
   locations = [
     {
       id              = data.hcloud_location.falkenstein.id,
       name            = data.hcloud_location.falkenstein.name,
       datacenter_name = data.hcloud_datacenter.falkenstein.name,
       network_zone    = data.hcloud_location.falkenstein.network_zone
-    }, {
+      }, {
       id              = data.hcloud_location.nuremberg.id,
       name            = data.hcloud_location.nuremberg.name,
       datacenter_name = data.hcloud_datacenter.nuremberg.name,
@@ -159,5 +161,5 @@ module "test_swarm_cluster" {
   ]
   server_subnet_id = hcloud_network_subnet.k8s-network-subnet.id
   stage_name       = local.test_cluster
-  number_nodes     = 2
+  number_nodes     = 3
 }

terraform/modules/docker-network/main.tf

@@ -5,3 +5,32 @@ data "hcloud_placement_group" "default" {
 locals {
   docker_network_domain = format("%s.%s.%s", "docker", var.stage_name, var.domain)
 }
+
+resource "hcloud_load_balancer" "docker" {
+  count = var.number_nodes > 1 ? 1 : 0
+
+  load_balancer_type = var.loadbalancer_type
+  name               = format("%s_%s", "docker", var.stage_name)
+  location           = var.locations[count.index].name
+  network_zone       = var.locations[count.index].network_zone
+  algorithm {
+    type = "round_robin"
+  }
+}
+
+resource "hcloud_load_balancer_network" "private" {
+  count = var.number_nodes > 1 ? 1 : 0
+
+  load_balancer_id = hcloud_load_balancer.docker[count.index].id
+  subnet_id        = var.server_subnet_id
+  ip               = var.loadbalancer_ip
+}
+
+resource "hcloud_load_balancer_target" "nodes" {
+  count = var.number_nodes > 1 ? var.number_nodes : 0
+
+  type             = "server"
+  load_balancer_id = hcloud_load_balancer.docker[0].id
+  server_id        = hcloud_server.node[count.index].id
+  use_private_ip   = true
+}

terraform/modules/docker-network/swarm-manager-node.tf

@@ -61,16 +61,17 @@ resource "hcloud_server" "node" {
   ignore_remote_firewall_ids = false
   keep_disk                  = false
   placement_group_id         = data.hcloud_placement_group.default.id
-  firewall_ids = [var.basic_firewall_id]
+  firewall_ids = [var.basic_firewall_id, var.http_firewall_id]
   ssh_keys = [var.admin_ssh_key.id]
   shutdown_before_deletion   = true
 
   labels = {
     "docker" : "yes",
-    "swarm" : "yes"
+    "swarm" : "yes",
+    "manager" : "yes"
   }
 
-  user_data = templatefile("${path.module}/templates/cloud-init-docker.yaml.tftpl", {
+  user_data = templatefile("${path.module}/templates/cloud-init-manager-node.yaml.tftpl", {
     admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name)
     admin_user : var.admin_user
     terraform_public_ssh_key : var.terraform_public_ssh_key
@@ -79,16 +80,18 @@ resource "hcloud_server" "node" {
     packages_setup : base64encode(file("${path.module}/templates/scripts/install-packages.sh"))
     firewall_setup : base64encode(templatefile("${path.module}/templates/scripts/firewall-setup.sh.tftpl", {
       node_ip : var.private_node_ips[count.index]
-      main_node : count.index == 0
+      manager_node : true
     }))
     ssh_setup : base64encode(templatefile("${path.module}/templates/scripts/ssh-setup.sh.tftpl", {
       admin_user : var.admin_user
+      manager_node : true
     }))
     sysctl_setup : base64encode(file("${path.module}/templates/scripts/sysctl-setup.sh"))
     swarm_setup : base64encode(templatefile("${path.module}/templates/scripts/swarm-setup.sh.tftpl", {
       main_node : count.index == 0
       main_node_private_ip : var.private_node_ips[0]
     }))
+    certbot_setup : base64encode(file("${path.module}/templates/scripts/certbot-setup.sh"))
   })
 
   network {

terraform/modules/docker-network/swarm-worker-node.tf

@@ -0,0 +1,99 @@
+resource "hcloud_primary_ip" "ipv4_worker_node_address" {
+  count = var.number_worker_nodes
+
+  name = format("%s_%s_%s_%s_%d", "docker", "worker", var.stage_name, "ipv4", count.index)
+  datacenter    = var.locations[count.index].datacenter_name
+  type          = "ipv4"
+  assignee_type = "server"
+  auto_delete   = false
+}
+
+resource "hcloud_rdns" "ipv4_worker_node" {
+  count = var.number_worker_nodes
+
+  primary_ip_id = hcloud_primary_ip.ipv4_worker_node_address[count.index].id
+  ip_address    = hcloud_primary_ip.ipv4_worker_node_address[count.index].ip_address
+  dns_ptr = format("%s-%s-%d.%s", "worker", "node", count.index, local.docker_network_domain)
+}
+
+resource "hcloud_primary_ip" "ipv6_worker_node_address" {
+  count = var.number_worker_nodes
+
+  name = format("%s_%s_%s_%s_%d", "docker", "worker", var.stage_name, "ipv6", count.index)
+  datacenter    = var.locations[count.index].datacenter_name
+  type          = "ipv6"
+  assignee_type = "server"
+  auto_delete   = false
+}
+
+resource "hcloud_rdns" "ipv6_node" {
+  count = var.number_worker_nodes
+
+  primary_ip_id = hcloud_primary_ip.ipv6_worker_node_address[count.index].id
+  ip_address    = hcloud_server.worker_node[count.index].ipv6_address
+  dns_ptr = format("%s-%s-%d.%s", "worker", "node", count.index, local.docker_network_domain)
+}
+
+resource "hcloud_server_network" "worker_network_private" {
+  count = var.number_worker_nodes
+
+  server_id = hcloud_server.worker_node[count.index].id
+  subnet_id = var.server_subnet_id
+  ip        = var.private_worker_node_ips[count.index]
+}
+
+resource "hcloud_server" "worker_node" {
+  count = var.number_worker_nodes
+
+  name = format("%s-%s-%s-%s-%d", "docker", var.stage_name, "worker", "node", count.index)
+  image                   = var.image_name
+  allow_deprecated_images = false
+  server_type             = var.server_type
+  location                = var.locations[count.index].name
+
+  public_net {
+    ipv4_enabled = true
+    ipv4         = hcloud_primary_ip.ipv4_worker_node_address[count.index].id
+    ipv6_enabled = true
+    ipv6         = hcloud_primary_ip.ipv6_worker_node_address[count.index].id
+  }
+
+  ignore_remote_firewall_ids = false
+  keep_disk                  = false
+  placement_group_id         = data.hcloud_placement_group.default.id
+  firewall_ids = [var.basic_firewall_id]
+  ssh_keys = [var.admin_ssh_key.id]
+  shutdown_before_deletion   = true
+
+  labels = {
+    "docker" : "yes",
+    "swarm" : "yes",
+    "worker" : "yes"
+  }
+
+  user_data = templatefile("${path.module}/templates/cloud-init-worker-node.yaml.tftpl", {
+    admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name)
+    admin_user : var.admin_user
+    terraform_public_ssh_key : var.terraform_public_ssh_key
+    packages_setup : base64encode(file("${path.module}/templates/scripts/install-packages.sh"))
+    firewall_setup : base64encode(templatefile("${path.module}/templates/scripts/firewall-setup.sh.tftpl", {
+      node_ip : var.private_node_ips[count.index]
+      manager_node : false
+    }))
+    ssh_setup : base64encode(templatefile("${path.module}/templates/scripts/ssh-setup.sh.tftpl", {
+      admin_user : var.admin_user
+      manager_node : false
+    }))
+    sysctl_setup : base64encode(file("${path.module}/templates/scripts/sysctl-setup.sh"))
+  })
+
+  network {
+    network_id = var.network_id
+    ip         = var.private_worker_node_ips[count.index]
+    alias_ips = []
+  }
+
+  lifecycle {
+    ignore_changes = [ssh_keys, user_data]
+  }
+}

terraform/modules/docker-network/templates/cloud-init-manager-node.yaml.tftpl

@@ -41,6 +41,11 @@ write_files:
     path: /run/tmpfiles.d/sysctl-setup.sh
     encoding: base64
     defer: true
+  - owner: root:root
+    content: "${certbot_setup}"
+    path: /run/tmpfiles.d/certbot-setup.sh
+    encoding: base64
+    defer: true
 %{ if main_node ~}
   - owner: root:root
     content: "${swarm_setup}"
@@ -54,6 +59,7 @@ runcmd:
   - /usr/local/bin/firewall-setup.sh
   - /usr/local/bin/ssh-setup.sh
   - /usr/local/bin/sysctl-setup.sh
+  - /usr/local/bin/certbot-setup.sh
 %{ if main_node ~}
   - /usr/local/bin/swarm-setup.sh
 %{ endif ~}

terraform/modules/docker-network/templates/cloud-init-worker-node.yaml.tftpl

@@ -0,0 +1,45 @@
+#cloud-config
+groups:
+- docker
+users:
+  - name: ${admin_user}
+    groups: users, admin, docker
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    shell: /bin/bash
+    ssh_authorized_keys:
+      - ${admin_public_ssh_key}
+  - name: terraform
+    groups: users, admin
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    ssh_authorized_keys:
+      - ${terraform_public_ssh_key}
+package_update: true
+package_upgrade: true
+package_reboot_if_required: true
+write_files:
+  - owner: root:root
+    content: "${packages_setup}"
+    path: /run/tmpfiles.d/install-packages.sh
+    encoding: base64
+    defer: true
+  - owner: root:root
+    content: "${firewall_setup}"
+    path: /run/tmpfiles.d/firewall-setup.sh
+    encoding: base64
+    defer: true
+  - owner: root:root
+    content: "${ssh_setup}"
+    path: /run/tmpfiles.d/ssh-setup.sh
+    encoding: base64
+    defer: true
+  - owner: root:root
+    content: "${sysctl_setup}"
+    path: /run/tmpfiles.d/sysctl-setup.sh
+    encoding: base64
+    defer: true
+runcmd:
+  - install /run/tmpfiles.d/*.sh /usr/local/bin
+  - /usr/local/bin/install-packages.sh
+  - /usr/local/bin/firewall-setup.sh
+  - /usr/local/bin/ssh-setup.sh
+  - /usr/local/bin/sysctl-setup.sh

terraform/modules/docker-network/templates/scripts/certbot-setup.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+mkdir -p /var/www/certbot

terraform/modules/docker-network/templates/scripts/firewall-setup.sh.tftpl

@@ -4,10 +4,10 @@ printf "[sshd]\nenabled = true\nbanaction = iptables-multiport" > /etc/fail2ban/
 systemctl enable fail2ban
 
 ufw allow OpenSSH
+%{if manager_node ~}
 ufw allow 80 comment HTTP
 ufw allow 443 comment HTTPS
 
-%{if main_node ~}
 ufw allow to ${node_ip} port 2377 proto tcp comment "Swarm join port and communication between manager nodes"
 %{endif ~}
 ufw allow to ${node_ip} port 7946 comment "overlay network node discovery"

terraform/modules/docker-network/templates/scripts/ssh-setup.sh.tftpl

@@ -7,4 +7,8 @@ sed -i -e '/^\(#\|\)MaxAuthTries/s/^.*$/MaxAuthTries 2/' /etc/ssh/sshd_config
 sed -i -e '/^\(#\|\)AllowTcpForwarding/s/^.*$/AllowTcpForwarding no/' /etc/ssh/sshd_config
 sed -i -e '/^\(#\|\)AllowAgentForwarding/s/^.*$/AllowAgentForwarding no/' /etc/ssh/sshd_config
 sed -i -e '/^\(#\|\)AuthorizedKeysFile/s/^.*$/AuthorizedKeysFile .ssh\/authorized_keys/' /etc/ssh/sshd_config
-sed -i '$a AllowUsers ${admin_user} terraform github' /etc/ssh/sshd_config
+%{if manager_node ~}
+sed -i '$a AllowUsers ${admin_user} terraform github' /etc/ssh/sshd_config
+%{else ~}
+sed -i '$a AllowUsers ${admin_user} terraform' /etc/ssh/sshd_config
+%{endif ~}

terraform/modules/docker-network/variables.tf

@@ -12,12 +12,6 @@ variable "admin_user" {
   type        = string
 }
 
-variable "terraform_private_ssh_key" {
-  description = "Private SSH key to be used by Terraform to connect with remote machines"
-  type        = string
-  sensitive   = true
-}
-
 variable "terraform_public_ssh_key" {
   description = "Public SSH key for the private key used by Terraform to connect with remote machines"
   type        = string
@@ -53,16 +47,39 @@ variable "server_subnet_id" {
   type        = string
 }
 
+variable "loadbalancer_type" {
+  description = "Type of the loadbalancer used. By default lb11 is used."
+  type        = string
+  default     = "lb11"
+}
+
+variable "loadbalancer_ip" {
+  description = "The private IP of the loadbalancer. Variable contains valid IP but must be overridden if more than one manager node is created."
+  type        = string
+  default     = "0.0.0.0"
+}
+
 variable "private_node_ips" {
-  description = "One IP in the private network subnet for each Docker Swarm node created."
+  description = "One IP in the private network subnet for each Docker Swarm manager node created."
   type = list(string)
 }
 
+variable "private_worker_node_ips" {
+  description = "One IP in the private network subnet for each Docker Swarm worker node created."
+  type = list(string)
+  default = []
+}
+
 variable "basic_firewall_id" {
   description = "ID of the basic firewall for each server"
   type        = number
 }
 
+variable "http_firewall_id" {
+  description = "ID of the http firewall for each server"
+  type        = number
+}
+
 variable "stage_name" {
   description = "The name of the stage. For example test, qs, prod. Will be used for names and subdomains and must be unique within the Hetzner project (identified by API token)."
   type        = string
@@ -70,7 +87,13 @@ variable "stage_name" {
 }
 
 variable "number_nodes" {
-  description = "Number of nodes in the docker swarm."
+  description = "Number of manager nodes in the docker swarm."
+  type        = number
+  default     = 3
+}
+
+variable "number_worker_nodes" {
+  description = "Number of worker nodes in the docker swarm."
   type        = number
   default     = 0
 }

terraform/modules/k8s-cluster/manager-node.tf

@@ -71,7 +71,7 @@ resource "hcloud_server" "manager" {
   ignore_remote_firewall_ids = false
   keep_disk                  = false
   placement_group_id         = data.hcloud_placement_group.default.id
-  firewall_ids = [var.basic_firewall_id, var.k8s_firewall_id]
+  firewall_ids = [var.basic_firewall_id, var.http_firewall_id, var.k8s_firewall_id]
   ssh_keys = [var.admin_ssh_key.id]
   shutdown_before_deletion   = true
 

terraform/modules/k8s-cluster/variables.tf

@@ -69,6 +69,11 @@ variable "basic_firewall_id" {
   type        = number
 }
 
+variable "http_firewall_id" {
+  description = "ID of the http firewall for each server"
+  type        = number
+}
+
 variable "k8s_firewall_id" {
   description = "ID of the K8s specific firewall"
   type        = number

terraform/modules/k8s-cluster/worker-node.tf

@@ -50,7 +50,7 @@ resource "hcloud_server" "worker" {
   ignore_remote_firewall_ids = false
   keep_disk                  = false
   placement_group_id         = data.hcloud_placement_group.default.id
-  firewall_ids               = [var.basic_firewall_id, var.k8s_firewall_id]
+  firewall_ids               = [var.basic_firewall_id, var.http_firewall_id, var.k8s_firewall_id]
   ssh_keys                   = [var.admin_ssh_key.id]
   shutdown_before_deletion   = true