change: Set up certsync on renewal

change: Remove volume

terraform/main.tf

@@ -137,6 +137,8 @@ module "test_swarm_cluster" {
   admin_user               = "2martensAdmin"
   terraform_public_ssh_key = var.terraform_public_ssh_key
   github_public_ssh_key    = var.github_public_ssh_key
+  certsync_public_ssh_key  = var.certsync_public_ssh_key
+  certsync_private_ssh_key = var.certsync_private_ssh_key
   basic_firewall_id        = hcloud_firewall.basic-firewall.id
   http_firewall_id         = hcloud_firewall.http-firewall.id
   domain                   = local.domain

terraform/modules/docker-network/swarm-manager-node.tf

@@ -42,26 +42,6 @@ resource "hcloud_server_network" "nginx_private" {
   ip        = var.private_node_ips[count.index]
 }
 
-resource "hcloud_volume" "certificates" {
-  name = format("%s_%s_%s", "docker", var.stage_name, "certificates")
-  location = element(var.locations.*.name, 0)
-  size = 10
-  labels = {
-    "docker" : "yes",
-    "swarm" : "yes",
-    "certificates" : "yes"
-  }
-  format = "ext4"
-}
-
-resource "hcloud_volume_attachment" "certificates" {
-  count = var.number_nodes
-
-  server_id = hcloud_server.node[count.index].id
-  volume_id = hcloud_volume.certificates.id
-  automount = true
-}
-
 resource "hcloud_server" "node" {
   count = var.number_nodes
 
@@ -96,6 +76,7 @@ resource "hcloud_server" "node" {
     admin_user : var.admin_user
     terraform_public_ssh_key : var.terraform_public_ssh_key
     github_public_ssh_key : var.github_public_ssh_key
+    certsync_public_ssh_key : var.certsync_public_ssh_key
     main_node : count.index == 0
     packages_setup : base64encode(templatefile("${path.module}/templates/scripts/install-packages.sh.tftpl", {
       main_node : count.index == 0
@@ -114,7 +95,13 @@ resource "hcloud_server" "node" {
       main_node_private_ip : var.private_node_ips[0]
     }))
     certificate_setup : base64encode(templatefile("${path.module}/templates/scripts/certificate-setup.sh.tftpl", {
-      volume_id : hcloud_volume.certificates.id
+      certsync_private_ssh_key : var.certsync_private_ssh_key
+    }))
+    certsync_setup : base64encode(templatefile("${path.module}/templates/scripts/certsync-setup.sh.tftpl", {
+
+    }))
+    certsync : base64encode(templatefile("${path.module}/templates/scripts/certsync.sh.tftpl", {
+      manager_nodes: slice(var.private_node_ips, 1, var.number_nodes)
     }))
   })
 

terraform/modules/docker-network/templates/cloud-init-manager-node.yaml.tftpl

@@ -17,6 +17,11 @@ users:
     groups: users, docker
     ssh_authorized_keys:
       - ${github_public_ssh_key}
+  - name: certsync
+    groups: users
+    sudo: ALL=(ALL) NOPASSWD:/usr/bin/rsync
+    ssh_authorized_keys:
+      - ${certsync_public_ssh_key}
 package_update: true
 package_upgrade: true
 package_reboot_if_required: true
@@ -52,6 +57,16 @@ write_files:
     path: /run/tmpfiles.d/swarm-setup.sh
     encoding: base64
     defer: true
+  - owner: root:root
+    content: "${certsync_setup}"
+    path: /run/tmpfiles.d/certsync-setup.sh
+    encoding: base64
+    defer: true
+  - owner: root:root
+    content: "${certsync}"
+    path: /run/tmpfiles.d/certsync.sh
+    encoding: base64
+    defer: true
 %{ endif ~}
 runcmd:
   - install /run/tmpfiles.d/*.sh /usr/local/bin
@@ -62,4 +77,5 @@ runcmd:
   - /usr/local/bin/sysctl-setup.sh
 %{ if main_node ~}
   - /usr/local/bin/swarm-setup.sh
+  - /usr/local/bin/certsync-setup.sh
 %{ endif ~}

terraform/modules/docker-network/templates/scripts/certificate-setup.sh.tftpl

@@ -1,4 +1,9 @@
-#!/bin/bash
+#!/bin/sh
 
 mkdir -p /var/www/certbot
-ln -s /mnt/HC_Volume_${volume_id} /etc/letsencrypt
+mkdir -p /etc/letsencrypt/live
+chmod 700 /etc/letsencrypt/live
+
+echo ${certsync_private_ssh_key} > /home/certsync/.ssh/id_ed25519
+chmod 600 /home/certsync/.ssh/id_ed25519
+chown -R certsync:certsync /home/certsync/.ssh

terraform/modules/docker-network/templates/scripts/certsync-setup.sh.tftpl

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+ln /usr/local/bin/certsync.sh /etc/letsencrypt/renewal-hooks/deploy/certsync.sh
+chmod +x /etc/letsencrypt/renewal-hooks/deploy/certsync.sh

terraform/modules/docker-network/templates/scripts/certsync.sh.tftpl

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+%{for manager_node_ip in manager_nodes ~}
+rsync -azr -e "ssh -i /home/certsync/.ssh/id_ed25519" /etc/letsencrypt/live certsync@${manager_node_ip}:/home/certsync/
+ssh -i /home/certsync/.ssh/id_ed25519 certsync@${manager_node_ip} sudo rsync -azr /home/certsync/live /etc/letsencrypt/
+%{endfor ~}
+docker service update --force nginx_nginx-proxy

terraform/modules/docker-network/templates/scripts/ssh-setup.sh.tftpl

@@ -8,7 +8,7 @@ sed -i -e '/^\(#\|\)AllowTcpForwarding/s/^.*$/AllowTcpForwarding no/' /etc/ssh/s
 sed -i -e '/^\(#\|\)AllowAgentForwarding/s/^.*$/AllowAgentForwarding no/' /etc/ssh/sshd_config
 sed -i -e '/^\(#\|\)AuthorizedKeysFile/s/^.*$/AuthorizedKeysFile .ssh\/authorized_keys/' /etc/ssh/sshd_config
 %{if manager_node ~}
-sed -i '$a AllowUsers ${admin_user} terraform github' /etc/ssh/sshd_config
+sed -i '$a AllowUsers ${admin_user} terraform github certsync' /etc/ssh/sshd_config
 %{else ~}
 sed -i '$a AllowUsers ${admin_user} terraform' /etc/ssh/sshd_config
 %{endif ~}

terraform/modules/docker-network/variables.tf

@@ -22,6 +22,17 @@ variable "github_public_ssh_key" {
   type        = string
 }
 
+variable "certsync_public_ssh_key" {
+  description = "Public SSH key for the private key used by the main manager node to synchronize the certificates"
+  type        = string
+}
+
+variable "certsync_private_ssh_key" {
+  description = "Public SSH key for the private key used by the main manager node to synchronize the certificates"
+  type        = string
+  sensitive   = true
+}
+
 variable "locations" {
   description = "Locations in which the resources should be created. Singular resources will always be created in the first. Must be all in the same network zone."
   type = list(object({

terraform/variables.tf

@@ -54,6 +54,17 @@ variable "github_public_ssh_key" {
   type        = string
 }
 
+variable "certsync_public_ssh_key" {
+  description = "Public SSH key for the private key used by the main manager node to synchronize the certificates"
+  type        = string
+}
+
+variable "certsync_private_ssh_key" {
+  description = "Private SSH key for the private key used by the main manager node to synchronize the certificates"
+  type        = string
+  sensitive   = true
+}
+
 variable "vault_client_id" {
   description = "Client ID of the service principal used to access Hashicorp Cloud"
   type        = string