From a2256dbdb0bab71cb504329ec7b84e311650b47f Mon Sep 17 00:00:00 2001 From: Jim Martens Date: Sat, 2 Aug 2025 19:48:07 +0200 Subject: [PATCH] Finish docker network module --- terraform/modules/docker-network/main.tf | 16 --- terraform/modules/docker-network/nginx-vm.tf | 96 ----------------- terraform/modules/docker-network/outputs.tf | 18 ++-- .../modules/docker-network/swarm-node.tf | 101 ++++++++++++++++++ .../cloud-init-docker-no-nginx.yaml.tftpl | 43 -------- ...aml.tftpl => cloud-init-docker.yaml.tftpl} | 10 +- ...ewall-setup.sh => firewall-setup.sh.tftpl} | 2 +- .../templates/scripts/nginx-setup.sh | 25 ----- .../{ssh-setup.sh => ssh-setup.sh.tftpl} | 0 .../templates/scripts/swarm-setup.sh.tftpl | 6 ++ terraform/modules/docker-network/variables.tf | 13 +-- terraform/modules/docker-network/worker-vm.tf | 86 --------------- 12 files changed, 130 insertions(+), 286 deletions(-) delete mode 100644 terraform/modules/docker-network/nginx-vm.tf create mode 100644 terraform/modules/docker-network/swarm-node.tf delete mode 100644 terraform/modules/docker-network/templates/cloud-init-docker-no-nginx.yaml.tftpl rename terraform/modules/docker-network/templates/{cloud-init-docker-with-nginx.yaml.tftpl => cloud-init-docker.yaml.tftpl} (87%) rename terraform/modules/docker-network/templates/scripts/{firewall-setup.sh => firewall-setup.sh.tftpl} (94%) delete mode 100644 terraform/modules/docker-network/templates/scripts/nginx-setup.sh rename terraform/modules/docker-network/templates/scripts/{ssh-setup.sh => ssh-setup.sh.tftpl} (100%) create mode 100644 terraform/modules/docker-network/templates/scripts/swarm-setup.sh.tftpl delete mode 100644 terraform/modules/docker-network/worker-vm.tf diff --git a/terraform/modules/docker-network/main.tf b/terraform/modules/docker-network/main.tf index 588fbb5..389cb7a 100644 --- a/terraform/modules/docker-network/main.tf +++ b/terraform/modules/docker-network/main.tf @@ -5,19 +5,3 @@ data "hcloud_placement_group" "default" { locals { docker_network_domain = format("%s.%s.%s", "docker", var.stage_name, var.domain) } - -# TODO: use Hetzner DNS -# resource "inwx_nameserver_record" "kube_api_server_a" { -# domain = var.domain -# name = local.docker_network_domain -# content = hcloud_server.nginx.ipv4_address -# type = "A" -# ttl = 3600 -# } -# resource "inwx_nameserver_record" "kube_api_server_aaaa" { -# domain = var.domain -# name = local.docker_network_domain -# content = hcloud_server.nginx.ipv6_address -# type = "AAAA" -# ttl = 3600 -# } \ No newline at end of file diff --git a/terraform/modules/docker-network/nginx-vm.tf b/terraform/modules/docker-network/nginx-vm.tf deleted file mode 100644 index 6e476c2..0000000 --- a/terraform/modules/docker-network/nginx-vm.tf +++ /dev/null @@ -1,96 +0,0 @@ -# TODO: Use Hetzner DNS -# resource "inwx_nameserver_record" "manager_aaaa" { -# domain = var.domain -# name = format("%s.%s", "nginx", local.docker_network_domain) -# content = hcloud_server.nginx.ipv6_address -# type = "AAAA" -# ttl = 3600 -# } - -resource "hcloud_primary_ip" "ipv4_nginx_address" { - name = format("%s_%s_%s", "docker", var.stage_name, "ipv4") - datacenter = element(var.locations.*.datacenter_name, 0) - type = "ipv4" - assignee_type = "server" - auto_delete = false -} - -resource "hcloud_rdns" "ipv4_nginx" { - primary_ip_id = hcloud_primary_ip.ipv4_nginx_address.id - ip_address = hcloud_primary_ip.ipv4_nginx_address.ip_address - # TODO: use Hetzner DNS - # dns_ptr = inwx_nameserver_record.manager_aaaa.name -} - -resource "hcloud_primary_ip" "ipv6_nginx_address" { - name = format("%s_%s_%s", "k8s", var.stage_name, "ipv6") - datacenter = element(var.locations.*.datacenter_name, 0) - type = "ipv6" - assignee_type = "server" - auto_delete = false -} - -resource "hcloud_rdns" "ipv6_nginx" { - primary_ip_id = hcloud_primary_ip.ipv6_nginx_address.id - ip_address = hcloud_server.nginx.ipv6_address - # TODO: use Hetzner DNS - # dns_ptr = inwx_nameserver_record.manager_aaaa.name -} - -resource "hcloud_server_network" "nginx_private" { - server_id = hcloud_server.nginx.id - subnet_id = var.server_subnet_id - ip = var.private_nginx_vm_ip -} - -resource "hcloud_server" "nginx" { - name = format("%s-%s-%s", "docker", var.stage_name, "nginx") - image = var.image_name - allow_deprecated_images = false - server_type = var.server_type - location = element(var.locations.*.name, 0) - - public_net { - ipv4_enabled = true - ipv4 = hcloud_primary_ip.ipv4_nginx_address.id - ipv6_enabled = true - ipv6 = hcloud_primary_ip.ipv6_nginx_address.id - } - - ignore_remote_firewall_ids = false - keep_disk = false - placement_group_id = data.hcloud_placement_group.default.id - firewall_ids = [var.basic_firewall_id, var.docker_firewall_id] - ssh_keys = [var.admin_ssh_key.id] - shutdown_before_deletion = true - - labels = { - "docker" : "yes", - "nginx" : "yes" - } - - user_data = templatefile("${path.module}/templates/cloud-init-docker-with-nginx.yaml.tftpl", { - admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name) - admin_user : var.admin_user - terraform_public_ssh_key : var.terraform_public_ssh_key - packages_setup : base64encode(file("${path.module}/templates/scripts/install-packages.sh")) - firewall_setup : base64encode(templatefile("${path.module}/templates/scripts/firewall-setup.sh", { - node_ip : var.private_nginx_vm_ip - })) - ssh_setup : base64encode(templatefile("${path.module}/templates/scripts/ssh-setup.sh", { - admin_user : var.admin_user - })) - sysctl_setup : base64encode(file("${path.module}/templates/scripts/sysctl-setup.sh")) - nginx_setup : base64encode(file("${path.module}/templates/scripts/nginx-setup.sh")) - }) - - network { - network_id = var.network_id - ip = var.private_nginx_vm_ip - alias_ips = [] - } - - lifecycle { - ignore_changes = [ssh_keys, user_data] - } -} \ No newline at end of file diff --git a/terraform/modules/docker-network/outputs.tf b/terraform/modules/docker-network/outputs.tf index e7ce2c4..aac3ada 100644 --- a/terraform/modules/docker-network/outputs.tf +++ b/terraform/modules/docker-network/outputs.tf @@ -1,7 +1,11 @@ -output "nginx_server_ipv4_address" { - value = hcloud_server.nginx.ipv4_address -} - -output "nginx_server_ipv6_address" { - value = hcloud_server.nginx.ipv6_address -} +output "node_details" { + value = { + for idx, instance in hcloud_server.node : + "node-${idx}" => { + id = instance.id + name = instance.name + ipv4 = instance.ipv4_address + ipv6 = instance.ipv6_address + } + } +} \ No newline at end of file diff --git a/terraform/modules/docker-network/swarm-node.tf b/terraform/modules/docker-network/swarm-node.tf new file mode 100644 index 0000000..2553f2e --- /dev/null +++ b/terraform/modules/docker-network/swarm-node.tf @@ -0,0 +1,101 @@ +resource "hcloud_primary_ip" "ipv4_node_address" { + count = var.number_nodes + + name = format("%s_%s_%s_%d", "docker", var.stage_name, "ipv4", count.index) + datacenter = var.locations[count.index].datacenter_name + type = "ipv4" + assignee_type = "server" + auto_delete = false +} + +resource "hcloud_rdns" "ipv4_node" { + count = var.number_nodes + + primary_ip_id = hcloud_primary_ip.ipv4_node_address[count.index].id + ip_address = hcloud_primary_ip.ipv4_node_address[count.index].ip_address + dns_ptr = format("%s-%d.%s", "node", count.index, local.docker_network_domain) +} + +resource "hcloud_primary_ip" "ipv6_node_address" { + count = var.number_nodes + + name = format("%s_%s_%s_%d", "docker", var.stage_name, "ipv6", count.index) + datacenter = var.locations[count.index].datacenter_name + type = "ipv6" + assignee_type = "server" + auto_delete = false +} + +resource "hcloud_rdns" "ipv6_node" { + count = var.number_nodes + + primary_ip_id = hcloud_primary_ip.ipv6_node_address[count.index].id + ip_address = hcloud_server.node[count.index].ipv6_address + dns_ptr = format("%s-%d.%s", "node", count.index, local.docker_network_domain) +} + +resource "hcloud_server_network" "nginx_private" { + count = var.number_nodes + + server_id = hcloud_server.node[count.index].id + subnet_id = var.server_subnet_id + ip = var.private_node_ips[count.index] +} + +resource "hcloud_server" "node" { + count = var.number_nodes + + name = format("%s-%s-%s-%d", "docker", var.stage_name, "node", count.index) + image = var.image_name + allow_deprecated_images = false + server_type = var.server_type + location = var.locations[count.index].name + + public_net { + ipv4_enabled = true + ipv4 = hcloud_primary_ip.ipv4_node_address[count.index].id + ipv6_enabled = true + ipv6 = hcloud_primary_ip.ipv6_node_address[count.index].id + } + + ignore_remote_firewall_ids = false + keep_disk = false + placement_group_id = data.hcloud_placement_group.default.id + firewall_ids = [var.basic_firewall_id, var.docker_firewall_id] + ssh_keys = [var.admin_ssh_key.id] + shutdown_before_deletion = true + + labels = { + "docker" : "yes", + "swarm" : "yes" + } + + user_data = templatefile("cloud-init-docker.yaml.tftpl", { + admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name) + admin_user : var.admin_user + terraform_public_ssh_key : var.terraform_public_ssh_key + main_node : count.index == 0 + packages_setup : base64encode(file("${path.module}/templates/scripts/install-packages.sh")) + firewall_setup : base64encode(templatefile("${path.module}/templates/scripts/firewall-setup.sh.tftpl", { + node_ip : var.private_node_ips[count.index] + })) + ssh_setup : base64encode(templatefile("${path.module}/templates/scripts/ssh-setup.sh.tftpl", { + admin_user : var.admin_user + })) + sysctl_setup : base64encode(file("${path.module}/templates/scripts/sysctl-setup.sh")) + swarm_setup : base64encode(templatefile("${path.module}/templates/scripts/swarm-setup.sh.tftpl", { + main_node : count.index == 0 + main_node_private_ip : var.private_node_ips[0] + })) + }) + + network { + network_id = var.network_id + ip = var.private_node_ips[count.index] + alias_ips = [] + } + + lifecycle { + ignore_changes = [ssh_keys, user_data] + } +} diff --git a/terraform/modules/docker-network/templates/cloud-init-docker-no-nginx.yaml.tftpl b/terraform/modules/docker-network/templates/cloud-init-docker-no-nginx.yaml.tftpl deleted file mode 100644 index 826c303..0000000 --- a/terraform/modules/docker-network/templates/cloud-init-docker-no-nginx.yaml.tftpl +++ /dev/null @@ -1,43 +0,0 @@ -#cloud-config -users: - - name: ${admin_user} - groups: users, admin, microk8s - sudo: ALL=(ALL) NOPASSWD:ALL - shell: /bin/bash - ssh_authorized_keys: - - ${admin_public_ssh_key} - - name: terraform - groups: users, admin - sudo: ALL=(ALL) NOPASSWD:ALL - ssh_authorized_keys: - - ${terraform_public_ssh_key} -package_update: true -package_upgrade: true -package_reboot_if_required: true -write_files: - - owner: root:root - content: "${packages_setup}" - path: /run/tmpfiles.d/install-packages.sh - encoding: base64 - defer: true - - owner: root:root - content: "${firewall_setup}" - path: /run/tmpfiles.d/firewall-setup.sh - encoding: base64 - defer: true - - owner: root:root - content: "${ssh_setup}" - path: /run/tmpfiles.d/ssh-setup.sh - encoding: base64 - defer: true - - owner: root:root - content: "${sysctl_setup}" - path: /run/tmpfiles.d/sysctl-setup.sh - encoding: base64 - defer: true -runcmd: - - install /run/tmpfiles.d/*.sh /usr/local/bin - - /usr/local/bin/install-packages.sh - - /usr/local/bin/firewall-setup.sh - - /usr/local/bin/ssh-setup.sh - - /usr/local/bin/sysctl-setup.sh diff --git a/terraform/modules/docker-network/templates/cloud-init-docker-with-nginx.yaml.tftpl b/terraform/modules/docker-network/templates/cloud-init-docker.yaml.tftpl similarity index 87% rename from terraform/modules/docker-network/templates/cloud-init-docker-with-nginx.yaml.tftpl rename to terraform/modules/docker-network/templates/cloud-init-docker.yaml.tftpl index 4ffb3b2..cda3d3a 100644 --- a/terraform/modules/docker-network/templates/cloud-init-docker-with-nginx.yaml.tftpl +++ b/terraform/modules/docker-network/templates/cloud-init-docker.yaml.tftpl @@ -35,15 +35,19 @@ write_files: path: /run/tmpfiles.d/sysctl-setup.sh encoding: base64 defer: true +%{ if main_node ~} - owner: root:root - content: "${nginx_setup}" - path: /run/tmpfiles.d/nginx-setup.sh + content: "${swarm_setup}" + path: /run/tmpfiles.d/swarm-setup.sh encoding: base64 defer: true +%{ endif ~} runcmd: - install /run/tmpfiles.d/*.sh /usr/local/bin - /usr/local/bin/install-packages.sh - /usr/local/bin/firewall-setup.sh - /usr/local/bin/ssh-setup.sh - /usr/local/bin/sysctl-setup.sh - - /usr/local/bin/nginx-setup.sh +%{ if main_node ~} + - /usr/local/bin/swarm-setup.sh +%{ endif ~} diff --git a/terraform/modules/docker-network/templates/scripts/firewall-setup.sh b/terraform/modules/docker-network/templates/scripts/firewall-setup.sh.tftpl similarity index 94% rename from terraform/modules/docker-network/templates/scripts/firewall-setup.sh rename to terraform/modules/docker-network/templates/scripts/firewall-setup.sh.tftpl index c060740..702d9a2 100644 --- a/terraform/modules/docker-network/templates/scripts/firewall-setup.sh +++ b/terraform/modules/docker-network/templates/scripts/firewall-setup.sh.tftpl @@ -6,4 +6,4 @@ systemctl enable fail2ban ufw allow OpenSSH ufw allow 80 comment HTTP ufw allow 443 comment HTTPS -ufw enable \ No newline at end of file +ufw enable diff --git a/terraform/modules/docker-network/templates/scripts/nginx-setup.sh b/terraform/modules/docker-network/templates/scripts/nginx-setup.sh deleted file mode 100644 index 59b082b..0000000 --- a/terraform/modules/docker-network/templates/scripts/nginx-setup.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -apt-get -y install gnupg2 ubuntu-keyring - -curl https://nginx.org/keys/nginx_signing.key | gpg --dearmor \ - | tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null - -echo \ - "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] http://nginx.org/packages/mainline/ubuntu \ - $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") nginx" | \ - tee /etc/apt/sources.list.d/nginx.list - -echo -e "Package: *\nPin: origin nginx.org\nPin: release o=nginx\nPin-Priority: 900\n" \ - | tee /etc/apt/preferences.d/99nginx - -apt-get update -apt-get -y install nginx - -# Remove the default Nginx configuration file -rm /etc/nginx/sites-enabled/default - -# Enable the Nginx site configuration -ln -s /etc/nginx/sites-available/default /etc/nginx/sites-enabled/default - -nginx \ No newline at end of file diff --git a/terraform/modules/docker-network/templates/scripts/ssh-setup.sh b/terraform/modules/docker-network/templates/scripts/ssh-setup.sh.tftpl similarity index 100% rename from terraform/modules/docker-network/templates/scripts/ssh-setup.sh rename to terraform/modules/docker-network/templates/scripts/ssh-setup.sh.tftpl diff --git a/terraform/modules/docker-network/templates/scripts/swarm-setup.sh.tftpl b/terraform/modules/docker-network/templates/scripts/swarm-setup.sh.tftpl new file mode 100644 index 0000000..1f378ab --- /dev/null +++ b/terraform/modules/docker-network/templates/scripts/swarm-setup.sh.tftpl @@ -0,0 +1,6 @@ +#!/bin/bash + +%{ if main_node ~} +docker swarm init --advertise-addr ${main_node_private_ip} +docker network create --driver overlay --attachable internal_net +%{ endif ~} diff --git a/terraform/modules/docker-network/variables.tf b/terraform/modules/docker-network/variables.tf index aff4e53..ba62478 100644 --- a/terraform/modules/docker-network/variables.tf +++ b/terraform/modules/docker-network/variables.tf @@ -48,13 +48,8 @@ variable "server_subnet_id" { type = string } -variable "private_nginx_vm_ip" { - description = "IP in the private network subnet for the VM containing Nginx" - type = string -} - -variable "private_worker_vm_ips" { - description = "One IP in the private network subnet for each VM without Nginx created." +variable "private_node_ips" { + description = "One IP in the private network subnet for each Docker Swarm node created." type = list(string) } @@ -74,8 +69,8 @@ variable "stage_name" { default = "test" } -variable "number_worker_vms" { - description = "Number of worker VMs in the docker network." +variable "number_nodes" { + description = "Number of nodes in the docker swarm." type = number default = 0 } diff --git a/terraform/modules/docker-network/worker-vm.tf b/terraform/modules/docker-network/worker-vm.tf deleted file mode 100644 index 3d0b5c4..0000000 --- a/terraform/modules/docker-network/worker-vm.tf +++ /dev/null @@ -1,86 +0,0 @@ -# TODO: Use Hetzner DNS -# resource "inwx_nameserver_record" "worker_aaaa" { -# count = var.number_worker_vms -# -# domain = var.domain -# name = format("%s-%d.%s", "node", count.index, local.docker_network_domain) -# content = hcloud_server.worker[count.index].ipv6_address -# type = "AAAA" -# ttl = 3600 -# } - -resource "hcloud_primary_ip" "ipv6_worker_address" { - count = var.number_worker_vms - - name = format("%s_%s_%s_%d", "docker", var.stage_name, "ipv6", count.index) - datacenter = element(var.locations.*.datacenter_name, count.index) - type = "ipv6" - assignee_type = "server" - auto_delete = false -} - -resource "hcloud_rdns" "ipv6_worker" { - count = var.number_worker_vms - - primary_ip_id = hcloud_primary_ip.ipv6_worker_address[count.index].id - ip_address = hcloud_server.worker[count.index].ipv6_address - # TODO: Use Hetzner DNS - # dns_ptr = inwx_nameserver_record.worker_aaaa[count.index].name -} - -resource "hcloud_server_network" "worker_private" { - count = var.number_worker_vms - - server_id = hcloud_server.worker[count.index].id - subnet_id = var.server_subnet_id - ip = var.private_worker_vm_ips[count.index] -} - -resource "hcloud_server" "worker" { - count = var.number_worker_vms - - name = format("%s-%s-%s-%d", "docker", var.stage_name, "node", count.index) - image = var.image_name - allow_deprecated_images = false - server_type = var.server_type - location = element(var.locations.*.name, count.index) - public_net { - ipv4_enabled = false - ipv6_enabled = true - ipv6 = hcloud_primary_ip.ipv6_worker_address[count.index].id - } - ignore_remote_firewall_ids = false - keep_disk = false - placement_group_id = data.hcloud_placement_group.default.id - firewall_ids = [var.basic_firewall_id, var.docker_firewall_id] - ssh_keys = [var.admin_ssh_key.id] - shutdown_before_deletion = true - - labels = { - "docker" : "yes" - } - - user_data = templatefile("${path.module}/templates/cloud-init-docker-no-nginx.yaml.tftpl", { - admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name) - admin_user : var.admin_user - terraform_public_ssh_key : var.terraform_public_ssh_key - packages_setup : base64encode(file("${path.module}/templates/scripts/install-packages.sh")) - firewall_setup : base64encode(templatefile("${path.module}/templates/scripts/firewall-setup.sh", { - node_ip : var.private_worker_vm_ips[count.index] - })) - ssh_setup : base64encode(templatefile("${path.module}/templates/scripts/ssh-setup.sh", { - admin_user : var.admin_user - })) - sysctl_setup : base64encode(file("${path.module}/templates/scripts/sysctl-setup.sh")) - }) - - network { - network_id = var.network_id - ip = var.private_worker_vm_ips[count.index] - alias_ips = [] - } - - lifecycle { - ignore_changes = [ssh_keys, user_data] - } -} \ No newline at end of file