From eba8edf610b4f1bcde70b899162cef3397bd5992 Mon Sep 17 00:00:00 2001 From: Jim Martens Date: Sun, 3 Aug 2025 15:05:35 +0200 Subject: [PATCH] change: Add support for manager and worker nodes in Docker swarm change: Extract port 80/443 to separate firewalls --- terraform/hetzner.tf | 24 +++-- terraform/main.tf | 32 +++--- terraform/modules/docker-network/main.tf | 29 ++++++ .../{swarm-node.tf => swarm-manager-node.tf} | 11 ++- .../docker-network/swarm-worker-node.tf | 99 +++++++++++++++++++ ...tpl => cloud-init-manager-node.yaml.tftpl} | 6 ++ .../cloud-init-worker-node.yaml.tftpl | 45 +++++++++ .../templates/scripts/certbot-setup.sh | 3 + .../templates/scripts/firewall-setup.sh.tftpl | 2 +- .../templates/scripts/ssh-setup.sh.tftpl | 6 +- terraform/modules/docker-network/variables.tf | 39 ++++++-- terraform/modules/k8s-cluster/manager-node.tf | 2 +- terraform/modules/k8s-cluster/variables.tf | 5 + terraform/modules/k8s-cluster/worker-node.tf | 2 +- 14 files changed, 264 insertions(+), 41 deletions(-) rename terraform/modules/docker-network/{swarm-node.tf => swarm-manager-node.tf} (92%) create mode 100644 terraform/modules/docker-network/swarm-worker-node.tf rename terraform/modules/docker-network/templates/{cloud-init-docker.yaml.tftpl => cloud-init-manager-node.yaml.tftpl} (89%) create mode 100644 terraform/modules/docker-network/templates/cloud-init-worker-node.yaml.tftpl create mode 100644 terraform/modules/docker-network/templates/scripts/certbot-setup.sh diff --git a/terraform/hetzner.tf b/terraform/hetzner.tf index 3b17147..de33a93 100644 --- a/terraform/hetzner.tf +++ b/terraform/hetzner.tf @@ -32,6 +32,20 @@ resource "hcloud_firewall" "basic-firewall" { ] } + rule { + direction = "in" + protocol = "icmp" + description = "allow ICMP" + source_ips = [ + "0.0.0.0/0", + "::/0", + ] + } +} + +resource "hcloud_firewall" "http-firewall" { + name = "http-firewall" + rule { direction = "in" protocol = "tcp" @@ -53,16 +67,6 @@ resource "hcloud_firewall" "basic-firewall" { "::/0", ] } - - rule { - direction = "in" - protocol = "icmp" - description = "allow ICMP" - source_ips = [ - "0.0.0.0/0", - "::/0", - ] - } } resource "hcloud_firewall" "k8s-firewall" { diff --git a/terraform/main.tf b/terraform/main.tf index 4c277dc..ae17518 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -33,6 +33,7 @@ module "test_cluster" { terraform_public_ssh_key = var.terraform_public_ssh_key terraform_private_ssh_key = var.terraform_private_ssh_key basic_firewall_id = hcloud_firewall.basic-firewall.id + http_firewall_id = hcloud_firewall.http-firewall.id k8s_firewall_id = hcloud_firewall.k8s-firewall.id locations = [ { @@ -40,7 +41,7 @@ module "test_cluster" { name = data.hcloud_location.falkenstein.name, datacenter_name = data.hcloud_datacenter.falkenstein.name, network_zone = data.hcloud_location.falkenstein.network_zone - }, { + }, { id = data.hcloud_location.nuremberg.id, name = data.hcloud_location.nuremberg.name, datacenter_name = data.hcloud_datacenter.nuremberg.name, @@ -61,12 +62,12 @@ module "test_cluster" { server_type = "cax21" image_name = "ubuntu-22.04" create_loadbalancer = false - loadbalancer_ip = cidrhost(hcloud_network_subnet.k8s-network-subnet.ip_range, 3) + loadbalancer_ip = cidrhost(hcloud_network_subnet.k8s-network-subnet.ip_range, 3) vault_service_principal = { client_id : var.vault_client_id client_secret : var.vault_client_secret } - vault_allowed_namespaces = ["wahlrecht", "timetable", "hcloud", "routing"] + vault_allowed_namespaces = ["wahlrecht", "timetable", "hcloud", "routing"] hcloud_token_enabled = false hcloud_token = var.hcloud_token thanos_enabled = true @@ -86,6 +87,7 @@ module "monitoring_cluster" { terraform_public_ssh_key = var.terraform_public_ssh_key terraform_private_ssh_key = var.terraform_private_ssh_key basic_firewall_id = hcloud_firewall.basic-firewall.id + http_firewall_id = hcloud_firewall.http-firewall.id k8s_firewall_id = hcloud_firewall.k8s-firewall.id locations = [ { @@ -93,7 +95,7 @@ module "monitoring_cluster" { name = data.hcloud_location.falkenstein.name, datacenter_name = data.hcloud_datacenter.falkenstein.name, network_zone = data.hcloud_location.falkenstein.network_zone - }, { + }, { id = data.hcloud_location.nuremberg.id, name = data.hcloud_location.nuremberg.name, datacenter_name = data.hcloud_datacenter.nuremberg.name, @@ -112,12 +114,12 @@ module "monitoring_cluster" { server_type = "cax21" image_name = "ubuntu-22.04" create_loadbalancer = false - loadbalancer_ip = cidrhost(hcloud_network_subnet.k8s-network-subnet.ip_range, 3) + loadbalancer_ip = cidrhost(hcloud_network_subnet.k8s-network-subnet.ip_range, 3) vault_service_principal = { client_id : var.vault_client_id client_secret : var.vault_client_secret } - vault_allowed_namespaces = ["hcloud"] + vault_allowed_namespaces = ["hcloud"] hcloud_token_enabled = false hcloud_token = var.hcloud_token thanos_enabled = true @@ -131,20 +133,20 @@ module "monitoring_cluster" { module "test_swarm_cluster" { source = "./modules/docker-network" - admin_ssh_key = data.hcloud_ssh_key.macos - admin_user = "2martensAdmin" - terraform_public_ssh_key = var.terraform_public_ssh_key - terraform_private_ssh_key = var.terraform_private_ssh_key - github_public_ssh_key = var.github_public_ssh_key - basic_firewall_id = hcloud_firewall.basic-firewall.id - domain = local.domain + admin_ssh_key = data.hcloud_ssh_key.macos + admin_user = "2martensAdmin" + terraform_public_ssh_key = var.terraform_public_ssh_key + github_public_ssh_key = var.github_public_ssh_key + basic_firewall_id = hcloud_firewall.basic-firewall.id + http_firewall_id = hcloud_firewall.http-firewall.id + domain = local.domain locations = [ { id = data.hcloud_location.falkenstein.id, name = data.hcloud_location.falkenstein.name, datacenter_name = data.hcloud_datacenter.falkenstein.name, network_zone = data.hcloud_location.falkenstein.network_zone - }, { + }, { id = data.hcloud_location.nuremberg.id, name = data.hcloud_location.nuremberg.name, datacenter_name = data.hcloud_datacenter.nuremberg.name, @@ -159,5 +161,5 @@ module "test_swarm_cluster" { ] server_subnet_id = hcloud_network_subnet.k8s-network-subnet.id stage_name = local.test_cluster - number_nodes = 2 + number_nodes = 3 } diff --git a/terraform/modules/docker-network/main.tf b/terraform/modules/docker-network/main.tf index 389cb7a..55d9a57 100644 --- a/terraform/modules/docker-network/main.tf +++ b/terraform/modules/docker-network/main.tf @@ -5,3 +5,32 @@ data "hcloud_placement_group" "default" { locals { docker_network_domain = format("%s.%s.%s", "docker", var.stage_name, var.domain) } + +resource "hcloud_load_balancer" "docker" { + count = var.number_nodes > 1 ? 1 : 0 + + load_balancer_type = var.loadbalancer_type + name = format("%s_%s", "docker", var.stage_name) + location = var.locations[count.index].name + network_zone = var.locations[count.index].network_zone + algorithm { + type = "round_robin" + } +} + +resource "hcloud_load_balancer_network" "private" { + count = var.number_nodes > 1 ? 1 : 0 + + load_balancer_id = hcloud_load_balancer.docker[count.index].id + subnet_id = var.server_subnet_id + ip = var.loadbalancer_ip +} + +resource "hcloud_load_balancer_target" "nodes" { + count = var.number_nodes > 1 ? var.number_nodes : 0 + + type = "server" + load_balancer_id = hcloud_load_balancer.docker[0].id + server_id = hcloud_server.node[count.index].id + use_private_ip = true +} diff --git a/terraform/modules/docker-network/swarm-node.tf b/terraform/modules/docker-network/swarm-manager-node.tf similarity index 92% rename from terraform/modules/docker-network/swarm-node.tf rename to terraform/modules/docker-network/swarm-manager-node.tf index c73c303..43e3aa5 100644 --- a/terraform/modules/docker-network/swarm-node.tf +++ b/terraform/modules/docker-network/swarm-manager-node.tf @@ -61,16 +61,17 @@ resource "hcloud_server" "node" { ignore_remote_firewall_ids = false keep_disk = false placement_group_id = data.hcloud_placement_group.default.id - firewall_ids = [var.basic_firewall_id] + firewall_ids = [var.basic_firewall_id, var.http_firewall_id] ssh_keys = [var.admin_ssh_key.id] shutdown_before_deletion = true labels = { "docker" : "yes", - "swarm" : "yes" + "swarm" : "yes", + "manager" : "yes" } - user_data = templatefile("${path.module}/templates/cloud-init-docker.yaml.tftpl", { + user_data = templatefile("${path.module}/templates/cloud-init-manager-node.yaml.tftpl", { admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name) admin_user : var.admin_user terraform_public_ssh_key : var.terraform_public_ssh_key @@ -79,16 +80,18 @@ resource "hcloud_server" "node" { packages_setup : base64encode(file("${path.module}/templates/scripts/install-packages.sh")) firewall_setup : base64encode(templatefile("${path.module}/templates/scripts/firewall-setup.sh.tftpl", { node_ip : var.private_node_ips[count.index] - main_node : count.index == 0 + manager_node : true })) ssh_setup : base64encode(templatefile("${path.module}/templates/scripts/ssh-setup.sh.tftpl", { admin_user : var.admin_user + manager_node : true })) sysctl_setup : base64encode(file("${path.module}/templates/scripts/sysctl-setup.sh")) swarm_setup : base64encode(templatefile("${path.module}/templates/scripts/swarm-setup.sh.tftpl", { main_node : count.index == 0 main_node_private_ip : var.private_node_ips[0] })) + certbot_setup : base64encode(file("${path.module}/templates/scripts/certbot-setup.sh")) }) network { diff --git a/terraform/modules/docker-network/swarm-worker-node.tf b/terraform/modules/docker-network/swarm-worker-node.tf new file mode 100644 index 0000000..fdea5c0 --- /dev/null +++ b/terraform/modules/docker-network/swarm-worker-node.tf @@ -0,0 +1,99 @@ +resource "hcloud_primary_ip" "ipv4_worker_node_address" { + count = var.number_worker_nodes + + name = format("%s_%s_%s_%s_%d", "docker", "worker", var.stage_name, "ipv4", count.index) + datacenter = var.locations[count.index].datacenter_name + type = "ipv4" + assignee_type = "server" + auto_delete = false +} + +resource "hcloud_rdns" "ipv4_worker_node" { + count = var.number_worker_nodes + + primary_ip_id = hcloud_primary_ip.ipv4_worker_node_address[count.index].id + ip_address = hcloud_primary_ip.ipv4_worker_node_address[count.index].ip_address + dns_ptr = format("%s-%s-%d.%s", "worker", "node", count.index, local.docker_network_domain) +} + +resource "hcloud_primary_ip" "ipv6_worker_node_address" { + count = var.number_worker_nodes + + name = format("%s_%s_%s_%s_%d", "docker", "worker", var.stage_name, "ipv6", count.index) + datacenter = var.locations[count.index].datacenter_name + type = "ipv6" + assignee_type = "server" + auto_delete = false +} + +resource "hcloud_rdns" "ipv6_node" { + count = var.number_worker_nodes + + primary_ip_id = hcloud_primary_ip.ipv6_worker_node_address[count.index].id + ip_address = hcloud_server.worker_node[count.index].ipv6_address + dns_ptr = format("%s-%s-%d.%s", "worker", "node", count.index, local.docker_network_domain) +} + +resource "hcloud_server_network" "worker_network_private" { + count = var.number_worker_nodes + + server_id = hcloud_server.worker_node[count.index].id + subnet_id = var.server_subnet_id + ip = var.private_worker_node_ips[count.index] +} + +resource "hcloud_server" "worker_node" { + count = var.number_worker_nodes + + name = format("%s-%s-%s-%s-%d", "docker", var.stage_name, "worker", "node", count.index) + image = var.image_name + allow_deprecated_images = false + server_type = var.server_type + location = var.locations[count.index].name + + public_net { + ipv4_enabled = true + ipv4 = hcloud_primary_ip.ipv4_worker_node_address[count.index].id + ipv6_enabled = true + ipv6 = hcloud_primary_ip.ipv6_worker_node_address[count.index].id + } + + ignore_remote_firewall_ids = false + keep_disk = false + placement_group_id = data.hcloud_placement_group.default.id + firewall_ids = [var.basic_firewall_id] + ssh_keys = [var.admin_ssh_key.id] + shutdown_before_deletion = true + + labels = { + "docker" : "yes", + "swarm" : "yes", + "worker" : "yes" + } + + user_data = templatefile("${path.module}/templates/cloud-init-worker-node.yaml.tftpl", { + admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name) + admin_user : var.admin_user + terraform_public_ssh_key : var.terraform_public_ssh_key + packages_setup : base64encode(file("${path.module}/templates/scripts/install-packages.sh")) + firewall_setup : base64encode(templatefile("${path.module}/templates/scripts/firewall-setup.sh.tftpl", { + node_ip : var.private_node_ips[count.index] + manager_node : false + })) + ssh_setup : base64encode(templatefile("${path.module}/templates/scripts/ssh-setup.sh.tftpl", { + admin_user : var.admin_user + manager_node : false + })) + sysctl_setup : base64encode(file("${path.module}/templates/scripts/sysctl-setup.sh")) + }) + + network { + network_id = var.network_id + ip = var.private_worker_node_ips[count.index] + alias_ips = [] + } + + lifecycle { + ignore_changes = [ssh_keys, user_data] + } +} diff --git a/terraform/modules/docker-network/templates/cloud-init-docker.yaml.tftpl b/terraform/modules/docker-network/templates/cloud-init-manager-node.yaml.tftpl similarity index 89% rename from terraform/modules/docker-network/templates/cloud-init-docker.yaml.tftpl rename to terraform/modules/docker-network/templates/cloud-init-manager-node.yaml.tftpl index ed7a120..bf97101 100644 --- a/terraform/modules/docker-network/templates/cloud-init-docker.yaml.tftpl +++ b/terraform/modules/docker-network/templates/cloud-init-manager-node.yaml.tftpl @@ -41,6 +41,11 @@ write_files: path: /run/tmpfiles.d/sysctl-setup.sh encoding: base64 defer: true + - owner: root:root + content: "${certbot_setup}" + path: /run/tmpfiles.d/certbot-setup.sh + encoding: base64 + defer: true %{ if main_node ~} - owner: root:root content: "${swarm_setup}" @@ -54,6 +59,7 @@ runcmd: - /usr/local/bin/firewall-setup.sh - /usr/local/bin/ssh-setup.sh - /usr/local/bin/sysctl-setup.sh + - /usr/local/bin/certbot-setup.sh %{ if main_node ~} - /usr/local/bin/swarm-setup.sh %{ endif ~} diff --git a/terraform/modules/docker-network/templates/cloud-init-worker-node.yaml.tftpl b/terraform/modules/docker-network/templates/cloud-init-worker-node.yaml.tftpl new file mode 100644 index 0000000..5d26f41 --- /dev/null +++ b/terraform/modules/docker-network/templates/cloud-init-worker-node.yaml.tftpl @@ -0,0 +1,45 @@ +#cloud-config +groups: +- docker +users: + - name: ${admin_user} + groups: users, admin, docker + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + ssh_authorized_keys: + - ${admin_public_ssh_key} + - name: terraform + groups: users, admin + sudo: ALL=(ALL) NOPASSWD:ALL + ssh_authorized_keys: + - ${terraform_public_ssh_key} +package_update: true +package_upgrade: true +package_reboot_if_required: true +write_files: + - owner: root:root + content: "${packages_setup}" + path: /run/tmpfiles.d/install-packages.sh + encoding: base64 + defer: true + - owner: root:root + content: "${firewall_setup}" + path: /run/tmpfiles.d/firewall-setup.sh + encoding: base64 + defer: true + - owner: root:root + content: "${ssh_setup}" + path: /run/tmpfiles.d/ssh-setup.sh + encoding: base64 + defer: true + - owner: root:root + content: "${sysctl_setup}" + path: /run/tmpfiles.d/sysctl-setup.sh + encoding: base64 + defer: true +runcmd: + - install /run/tmpfiles.d/*.sh /usr/local/bin + - /usr/local/bin/install-packages.sh + - /usr/local/bin/firewall-setup.sh + - /usr/local/bin/ssh-setup.sh + - /usr/local/bin/sysctl-setup.sh diff --git a/terraform/modules/docker-network/templates/scripts/certbot-setup.sh b/terraform/modules/docker-network/templates/scripts/certbot-setup.sh new file mode 100644 index 0000000..978a4f8 --- /dev/null +++ b/terraform/modules/docker-network/templates/scripts/certbot-setup.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +mkdir -p /var/www/certbot diff --git a/terraform/modules/docker-network/templates/scripts/firewall-setup.sh.tftpl b/terraform/modules/docker-network/templates/scripts/firewall-setup.sh.tftpl index e411315..559ffb6 100644 --- a/terraform/modules/docker-network/templates/scripts/firewall-setup.sh.tftpl +++ b/terraform/modules/docker-network/templates/scripts/firewall-setup.sh.tftpl @@ -4,10 +4,10 @@ printf "[sshd]\nenabled = true\nbanaction = iptables-multiport" > /etc/fail2ban/ systemctl enable fail2ban ufw allow OpenSSH +%{if manager_node ~} ufw allow 80 comment HTTP ufw allow 443 comment HTTPS -%{if main_node ~} ufw allow to ${node_ip} port 2377 proto tcp comment "Swarm join port and communication between manager nodes" %{endif ~} ufw allow to ${node_ip} port 7946 comment "overlay network node discovery" diff --git a/terraform/modules/docker-network/templates/scripts/ssh-setup.sh.tftpl b/terraform/modules/docker-network/templates/scripts/ssh-setup.sh.tftpl index 372b366..3bed65c 100644 --- a/terraform/modules/docker-network/templates/scripts/ssh-setup.sh.tftpl +++ b/terraform/modules/docker-network/templates/scripts/ssh-setup.sh.tftpl @@ -7,4 +7,8 @@ sed -i -e '/^\(#\|\)MaxAuthTries/s/^.*$/MaxAuthTries 2/' /etc/ssh/sshd_config sed -i -e '/^\(#\|\)AllowTcpForwarding/s/^.*$/AllowTcpForwarding no/' /etc/ssh/sshd_config sed -i -e '/^\(#\|\)AllowAgentForwarding/s/^.*$/AllowAgentForwarding no/' /etc/ssh/sshd_config sed -i -e '/^\(#\|\)AuthorizedKeysFile/s/^.*$/AuthorizedKeysFile .ssh\/authorized_keys/' /etc/ssh/sshd_config -sed -i '$a AllowUsers ${admin_user} terraform github' /etc/ssh/sshd_config \ No newline at end of file +%{if manager_node ~} +sed -i '$a AllowUsers ${admin_user} terraform github' /etc/ssh/sshd_config +%{else ~} +sed -i '$a AllowUsers ${admin_user} terraform' /etc/ssh/sshd_config +%{endif ~} diff --git a/terraform/modules/docker-network/variables.tf b/terraform/modules/docker-network/variables.tf index c6b6fc5..c2ef4b7 100644 --- a/terraform/modules/docker-network/variables.tf +++ b/terraform/modules/docker-network/variables.tf @@ -12,12 +12,6 @@ variable "admin_user" { type = string } -variable "terraform_private_ssh_key" { - description = "Private SSH key to be used by Terraform to connect with remote machines" - type = string - sensitive = true -} - variable "terraform_public_ssh_key" { description = "Public SSH key for the private key used by Terraform to connect with remote machines" type = string @@ -53,16 +47,39 @@ variable "server_subnet_id" { type = string } +variable "loadbalancer_type" { + description = "Type of the loadbalancer used. By default lb11 is used." + type = string + default = "lb11" +} + +variable "loadbalancer_ip" { + description = "The private IP of the loadbalancer. Variable contains valid IP but must be overridden if more than one manager node is created." + type = string + default = "0.0.0.0" +} + variable "private_node_ips" { - description = "One IP in the private network subnet for each Docker Swarm node created." + description = "One IP in the private network subnet for each Docker Swarm manager node created." type = list(string) } +variable "private_worker_node_ips" { + description = "One IP in the private network subnet for each Docker Swarm worker node created." + type = list(string) + default = [] +} + variable "basic_firewall_id" { description = "ID of the basic firewall for each server" type = number } +variable "http_firewall_id" { + description = "ID of the http firewall for each server" + type = number +} + variable "stage_name" { description = "The name of the stage. For example test, qs, prod. Will be used for names and subdomains and must be unique within the Hetzner project (identified by API token)." type = string @@ -70,7 +87,13 @@ variable "stage_name" { } variable "number_nodes" { - description = "Number of nodes in the docker swarm." + description = "Number of manager nodes in the docker swarm." + type = number + default = 3 +} + +variable "number_worker_nodes" { + description = "Number of worker nodes in the docker swarm." type = number default = 0 } diff --git a/terraform/modules/k8s-cluster/manager-node.tf b/terraform/modules/k8s-cluster/manager-node.tf index da4dc09..630a7ae 100644 --- a/terraform/modules/k8s-cluster/manager-node.tf +++ b/terraform/modules/k8s-cluster/manager-node.tf @@ -71,7 +71,7 @@ resource "hcloud_server" "manager" { ignore_remote_firewall_ids = false keep_disk = false placement_group_id = data.hcloud_placement_group.default.id - firewall_ids = [var.basic_firewall_id, var.k8s_firewall_id] + firewall_ids = [var.basic_firewall_id, var.http_firewall_id, var.k8s_firewall_id] ssh_keys = [var.admin_ssh_key.id] shutdown_before_deletion = true diff --git a/terraform/modules/k8s-cluster/variables.tf b/terraform/modules/k8s-cluster/variables.tf index 7945300..883a721 100644 --- a/terraform/modules/k8s-cluster/variables.tf +++ b/terraform/modules/k8s-cluster/variables.tf @@ -69,6 +69,11 @@ variable "basic_firewall_id" { type = number } +variable "http_firewall_id" { + description = "ID of the http firewall for each server" + type = number +} + variable "k8s_firewall_id" { description = "ID of the K8s specific firewall" type = number diff --git a/terraform/modules/k8s-cluster/worker-node.tf b/terraform/modules/k8s-cluster/worker-node.tf index 4af0e5b..4290295 100644 --- a/terraform/modules/k8s-cluster/worker-node.tf +++ b/terraform/modules/k8s-cluster/worker-node.tf @@ -50,7 +50,7 @@ resource "hcloud_server" "worker" { ignore_remote_firewall_ids = false keep_disk = false placement_group_id = data.hcloud_placement_group.default.id - firewall_ids = [var.basic_firewall_id, var.k8s_firewall_id] + firewall_ids = [var.basic_firewall_id, var.http_firewall_id, var.k8s_firewall_id] ssh_keys = [var.admin_ssh_key.id] shutdown_before_deletion = true