From 3a6b841724a5e1123b46d84c38141e8fdd0b2876 Mon Sep 17 00:00:00 2001
From: Jim Martens <github@2martens.de>
Date: Mon, 4 Aug 2025 21:50:44 +0200
Subject: [PATCH] change: Set up certsync on renewal

change: Remove volume
---
 terraform/main.tf                             |  2 ++
 .../docker-network/swarm-manager-node.tf      | 29 +++++--------------
 .../cloud-init-manager-node.yaml.tftpl        | 16 ++++++++++
 .../scripts/certificate-setup.sh.tftpl        |  9 ++++--
 .../templates/scripts/certsync-setup.sh.tftpl |  4 +++
 .../templates/scripts/certsync.sh.tftpl       |  7 +++++
 .../templates/scripts/ssh-setup.sh.tftpl      |  2 +-
 terraform/modules/docker-network/variables.tf | 11 +++++++
 terraform/variables.tf                        | 11 +++++++
 9 files changed, 67 insertions(+), 24 deletions(-)
 create mode 100644 terraform/modules/docker-network/templates/scripts/certsync-setup.sh.tftpl
 create mode 100644 terraform/modules/docker-network/templates/scripts/certsync.sh.tftpl

diff --git a/terraform/main.tf b/terraform/main.tf
index add0a99..59896bd 100644
--- a/terraform/main.tf
+++ b/terraform/main.tf
@@ -137,6 +137,8 @@ module "test_swarm_cluster" {
   admin_user               = "2martensAdmin"
   terraform_public_ssh_key = var.terraform_public_ssh_key
   github_public_ssh_key    = var.github_public_ssh_key
+  certsync_public_ssh_key  = var.certsync_public_ssh_key
+  certsync_private_ssh_key = var.certsync_private_ssh_key
   basic_firewall_id        = hcloud_firewall.basic-firewall.id
   http_firewall_id         = hcloud_firewall.http-firewall.id
   domain                   = local.domain
diff --git a/terraform/modules/docker-network/swarm-manager-node.tf b/terraform/modules/docker-network/swarm-manager-node.tf
index dd517bd..4957fca 100644
--- a/terraform/modules/docker-network/swarm-manager-node.tf
+++ b/terraform/modules/docker-network/swarm-manager-node.tf
@@ -42,26 +42,6 @@ resource "hcloud_server_network" "nginx_private" {
   ip        = var.private_node_ips[count.index]
 }
 
-resource "hcloud_volume" "certificates" {
-  name = format("%s_%s_%s", "docker", var.stage_name, "certificates")
-  location = element(var.locations.*.name, 0)
-  size = 10
-  labels = {
-    "docker" : "yes",
-    "swarm" : "yes",
-    "certificates" : "yes"
-  }
-  format = "ext4"
-}
-
-resource "hcloud_volume_attachment" "certificates" {
-  count = var.number_nodes
-
-  server_id = hcloud_server.node[count.index].id
-  volume_id = hcloud_volume.certificates.id
-  automount = true
-}
-
 resource "hcloud_server" "node" {
   count = var.number_nodes
 
@@ -96,6 +76,7 @@ resource "hcloud_server" "node" {
     admin_user : var.admin_user
     terraform_public_ssh_key : var.terraform_public_ssh_key
     github_public_ssh_key : var.github_public_ssh_key
+    certsync_public_ssh_key : var.certsync_public_ssh_key
     main_node : count.index == 0
     packages_setup : base64encode(templatefile("${path.module}/templates/scripts/install-packages.sh.tftpl", {
       main_node : count.index == 0
@@ -114,7 +95,13 @@ resource "hcloud_server" "node" {
       main_node_private_ip : var.private_node_ips[0]
     }))
     certificate_setup : base64encode(templatefile("${path.module}/templates/scripts/certificate-setup.sh.tftpl", {
-      volume_id : hcloud_volume.certificates.id
+      certsync_private_ssh_key : var.certsync_private_ssh_key
+    }))
+    certsync_setup : base64encode(templatefile("${path.module}/templates/scripts/certsync-setup.sh.tftpl", {
+
+    }))
+    certsync : base64encode(templatefile("${path.module}/templates/scripts/certsync.sh.tftpl", {
+      manager_nodes: slice(var.private_node_ips, 1, var.number_nodes)
     }))
   })
 
diff --git a/terraform/modules/docker-network/templates/cloud-init-manager-node.yaml.tftpl b/terraform/modules/docker-network/templates/cloud-init-manager-node.yaml.tftpl
index ba39565..599cd5e 100644
--- a/terraform/modules/docker-network/templates/cloud-init-manager-node.yaml.tftpl
+++ b/terraform/modules/docker-network/templates/cloud-init-manager-node.yaml.tftpl
@@ -17,6 +17,11 @@ users:
     groups: users, docker
     ssh_authorized_keys:
       - ${github_public_ssh_key}
+  - name: certsync
+    groups: users
+    sudo: ALL=(ALL) NOPASSWD:/usr/bin/rsync
+    ssh_authorized_keys:
+      - ${certsync_public_ssh_key}
 package_update: true
 package_upgrade: true
 package_reboot_if_required: true
@@ -52,6 +57,16 @@ write_files:
     path: /run/tmpfiles.d/swarm-setup.sh
     encoding: base64
     defer: true
+  - owner: root:root
+    content: "${certsync_setup}"
+    path: /run/tmpfiles.d/certsync-setup.sh
+    encoding: base64
+    defer: true
+  - owner: root:root
+    content: "${certsync}"
+    path: /run/tmpfiles.d/certsync.sh
+    encoding: base64
+    defer: true
 %{ endif ~}
 runcmd:
   - install /run/tmpfiles.d/*.sh /usr/local/bin
@@ -62,4 +77,5 @@ runcmd:
   - /usr/local/bin/sysctl-setup.sh
 %{ if main_node ~}
   - /usr/local/bin/swarm-setup.sh
+  - /usr/local/bin/certsync-setup.sh
 %{ endif ~}
diff --git a/terraform/modules/docker-network/templates/scripts/certificate-setup.sh.tftpl b/terraform/modules/docker-network/templates/scripts/certificate-setup.sh.tftpl
index 0008004..cf368ea 100644
--- a/terraform/modules/docker-network/templates/scripts/certificate-setup.sh.tftpl
+++ b/terraform/modules/docker-network/templates/scripts/certificate-setup.sh.tftpl
@@ -1,4 +1,9 @@
-#!/bin/bash
+#!/bin/sh
 
 mkdir -p /var/www/certbot
-ln -s /mnt/HC_Volume_${volume_id} /etc/letsencrypt
+mkdir -p /etc/letsencrypt/live
+chmod 700 /etc/letsencrypt/live
+
+echo ${certsync_private_ssh_key} > /home/certsync/.ssh/id_ed25519
+chmod 600 /home/certsync/.ssh/id_ed25519
+chown -R certsync:certsync /home/certsync/.ssh
diff --git a/terraform/modules/docker-network/templates/scripts/certsync-setup.sh.tftpl b/terraform/modules/docker-network/templates/scripts/certsync-setup.sh.tftpl
new file mode 100644
index 0000000..80acc21
--- /dev/null
+++ b/terraform/modules/docker-network/templates/scripts/certsync-setup.sh.tftpl
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+ln /usr/local/bin/certsync.sh /etc/letsencrypt/renewal-hooks/deploy/certsync.sh
+chmod +x /etc/letsencrypt/renewal-hooks/deploy/certsync.sh
diff --git a/terraform/modules/docker-network/templates/scripts/certsync.sh.tftpl b/terraform/modules/docker-network/templates/scripts/certsync.sh.tftpl
new file mode 100644
index 0000000..c01acc1
--- /dev/null
+++ b/terraform/modules/docker-network/templates/scripts/certsync.sh.tftpl
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+%{for manager_node_ip in manager_nodes ~}
+rsync -azr -e "ssh -i /home/certsync/.ssh/id_ed25519" /etc/letsencrypt/live certsync@${manager_node_ip}:/home/certsync/
+ssh -i /home/certsync/.ssh/id_ed25519 certsync@${manager_node_ip} sudo rsync -azr /home/certsync/live /etc/letsencrypt/
+%{endfor ~}
+docker service update --force nginx_nginx-proxy
diff --git a/terraform/modules/docker-network/templates/scripts/ssh-setup.sh.tftpl b/terraform/modules/docker-network/templates/scripts/ssh-setup.sh.tftpl
index 3bed65c..fa8db96 100644
--- a/terraform/modules/docker-network/templates/scripts/ssh-setup.sh.tftpl
+++ b/terraform/modules/docker-network/templates/scripts/ssh-setup.sh.tftpl
@@ -8,7 +8,7 @@ sed -i -e '/^\(#\|\)AllowTcpForwarding/s/^.*$/AllowTcpForwarding no/' /etc/ssh/s
 sed -i -e '/^\(#\|\)AllowAgentForwarding/s/^.*$/AllowAgentForwarding no/' /etc/ssh/sshd_config
 sed -i -e '/^\(#\|\)AuthorizedKeysFile/s/^.*$/AuthorizedKeysFile .ssh\/authorized_keys/' /etc/ssh/sshd_config
 %{if manager_node ~}
-sed -i '$a AllowUsers ${admin_user} terraform github' /etc/ssh/sshd_config
+sed -i '$a AllowUsers ${admin_user} terraform github certsync' /etc/ssh/sshd_config
 %{else ~}
 sed -i '$a AllowUsers ${admin_user} terraform' /etc/ssh/sshd_config
 %{endif ~}
diff --git a/terraform/modules/docker-network/variables.tf b/terraform/modules/docker-network/variables.tf
index 0ac3a1a..7c53389 100644
--- a/terraform/modules/docker-network/variables.tf
+++ b/terraform/modules/docker-network/variables.tf
@@ -22,6 +22,17 @@ variable "github_public_ssh_key" {
   type        = string
 }
 
+variable "certsync_public_ssh_key" {
+  description = "Public SSH key for the private key used by the main manager node to synchronize the certificates"
+  type        = string
+}
+
+variable "certsync_private_ssh_key" {
+  description = "Public SSH key for the private key used by the main manager node to synchronize the certificates"
+  type        = string
+  sensitive   = true
+}
+
 variable "locations" {
   description = "Locations in which the resources should be created. Singular resources will always be created in the first. Must be all in the same network zone."
   type = list(object({
diff --git a/terraform/variables.tf b/terraform/variables.tf
index 6ca2dc9..8d6b2c4 100644
--- a/terraform/variables.tf
+++ b/terraform/variables.tf
@@ -54,6 +54,17 @@ variable "github_public_ssh_key" {
   type        = string
 }
 
+variable "certsync_public_ssh_key" {
+  description = "Public SSH key for the private key used by the main manager node to synchronize the certificates"
+  type        = string
+}
+
+variable "certsync_private_ssh_key" {
+  description = "Private SSH key for the private key used by the main manager node to synchronize the certificates"
+  type        = string
+  sensitive   = true
+}
+
 variable "vault_client_id" {
   description = "Client ID of the service principal used to access Hashicorp Cloud"
   type        = string