From a701a707756828e5b8ee84d1ec9f372e66f6f802 Mon Sep 17 00:00:00 2001 From: Jim Martens Date: Sat, 4 Nov 2023 21:25:20 +0100 Subject: [PATCH] Use content snap --- terraform/hetzner.tf | 17 +++++++------- terraform/modules/k8s-cluster/manager-node.tf | 17 +++++++------- .../templates/cloud-init-k8s-manager.tftpl | 5 ++++ .../templates/cloud-init-k8s-worker.tftpl | 5 ++++ .../k8s-cluster/templates/cluster-setup.sh | 9 -------- .../{ => helm}/argocd-values-ha.yaml | 0 .../{ => helm}/cluster-setup-values.yaml | 0 .../k8s-cluster/templates/microk8s-setup.sh | 7 ------ .../templates/{ => scripts}/add-node.sh | 0 .../templates/scripts/cluster-setup.sh | 9 ++++++++ .../templates/{ => scripts}/firewall-setup.sh | 0 .../{ => scripts}/install-packages.sh | 0 .../templates/{ => scripts}/join-worker.sh | 0 .../templates/{ => scripts}/join.sh | 0 .../templates/scripts/microk8s-setup.sh | 13 +++++++++++ .../templates/{ => scripts}/ssh-setup.sh | 0 .../{ => snap}/microk8s-config-manager.tftpl | 0 .../{ => snap}/microk8s-config-worker.tftpl | 0 .../k8s-cluster/templates/snap/snapcraft.yaml | 23 +++++++++++++++++++ terraform/modules/k8s-cluster/worker-node.tf | 17 +++++++------- 20 files changed, 82 insertions(+), 40 deletions(-) delete mode 100644 terraform/modules/k8s-cluster/templates/cluster-setup.sh rename terraform/modules/k8s-cluster/templates/{ => helm}/argocd-values-ha.yaml (100%) rename terraform/modules/k8s-cluster/templates/{ => helm}/cluster-setup-values.yaml (100%) delete mode 100644 terraform/modules/k8s-cluster/templates/microk8s-setup.sh rename terraform/modules/k8s-cluster/templates/{ => scripts}/add-node.sh (100%) create mode 100644 terraform/modules/k8s-cluster/templates/scripts/cluster-setup.sh rename terraform/modules/k8s-cluster/templates/{ => scripts}/firewall-setup.sh (100%) rename terraform/modules/k8s-cluster/templates/{ => scripts}/install-packages.sh (100%) rename terraform/modules/k8s-cluster/templates/{ => scripts}/join-worker.sh (100%) rename terraform/modules/k8s-cluster/templates/{ => scripts}/join.sh (100%) create mode 100644 terraform/modules/k8s-cluster/templates/scripts/microk8s-setup.sh rename terraform/modules/k8s-cluster/templates/{ => scripts}/ssh-setup.sh (100%) rename terraform/modules/k8s-cluster/templates/{ => snap}/microk8s-config-manager.tftpl (100%) rename terraform/modules/k8s-cluster/templates/{ => snap}/microk8s-config-worker.tftpl (100%) create mode 100644 terraform/modules/k8s-cluster/templates/snap/snapcraft.yaml diff --git a/terraform/hetzner.tf b/terraform/hetzner.tf index f8e8e72..9f4daa2 100644 --- a/terraform/hetzner.tf +++ b/terraform/hetzner.tf @@ -183,7 +183,7 @@ resource "hcloud_server" "server_k8s_test" { admin_user : "2martensAdmin" terraform_public_ssh_key : var.terraform_public_ssh_key main_node : true - microk8s_config : base64encode(templatefile("${path.module}/modules/k8s-cluster/templates/microk8s-config-manager.tftpl", { + microk8s_config : base64encode(templatefile("${path.module}/modules/k8s-cluster/templates/snap/microk8s-config-manager.tftpl", { node_ip : "10.0.0.2" api_server_domain : "" api_server_ip : hcloud_primary_ip.primary_ipv4_k8s_test.ip_address @@ -191,26 +191,27 @@ resource "hcloud_server" "server_k8s_test" { main_node_ip : "10.0.0.2" cluster_token : "" })) - packages_setup : base64encode(templatefile("${path.module}/modules/k8s-cluster/templates/install-packages.sh", { + snapcraft : base64encode(file("${path.module}/modules/k8s-cluster/templates/snap/snapcraft.yaml")) + packages_setup : base64encode(templatefile("${path.module}/modules/k8s-cluster/templates/scripts/install-packages.sh", { refresh_day : "fri" refresh_hour : format("%02d", 0) })) - firewall_setup : base64encode(templatefile("${path.module}/modules/k8s-cluster/templates/firewall-setup.sh", { + firewall_setup : base64encode(templatefile("${path.module}/modules/k8s-cluster/templates/scripts/firewall-setup.sh", { node_ip : "10.0.0.2" })) - ssh_setup : base64encode(templatefile("${path.module}/modules/k8s-cluster/templates/ssh-setup.sh", { + ssh_setup : base64encode(templatefile("${path.module}/modules/k8s-cluster/templates/scripts/ssh-setup.sh", { admin_user : "2martensAdmin" })) - microk8s_setup : base64encode(templatefile("${path.module}/modules/k8s-cluster/templates/microk8s-setup.sh", { + microk8s_setup : base64encode(templatefile("${path.module}/modules/k8s-cluster/templates/scripts/microk8s-setup.sh", { microk8s_channel : "1.28-strict/stable" main_node : true })) - cluster_setup_values : base64encode(templatefile("${path.module}/modules/k8s-cluster/templates/cluster-setup-values.yaml", { + cluster_setup_values : base64encode(templatefile("${path.module}/modules/k8s-cluster/templates/helm/cluster-setup-values.yaml", { client_id : var.vault_client_id client_secret : var.vault_client_secret })) - argocd_ha_values : base64encode(file("${path.module}/modules/k8s-cluster/templates/argocd-values-ha.yaml")) - cluster_setup : base64encode(templatefile("${path.module}/modules/k8s-cluster/templates/cluster-setup.sh", { + argocd_ha_values : base64encode(file("${path.module}/modules/k8s-cluster/templates/helm/argocd-values-ha.yaml")) + cluster_setup : base64encode(templatefile("${path.module}/modules/k8s-cluster/templates/scripts/cluster-setup.sh", { argocd_environment : "test" high_availability : false })) diff --git a/terraform/modules/k8s-cluster/manager-node.tf b/terraform/modules/k8s-cluster/manager-node.tf index 5e11887..3ec827c 100644 --- a/terraform/modules/k8s-cluster/manager-node.tf +++ b/terraform/modules/k8s-cluster/manager-node.tf @@ -85,7 +85,7 @@ resource "hcloud_server" "manager" { admin_user : var.admin_user terraform_public_ssh_key : var.terraform_public_ssh_key main_node : count.index == 0 - microk8s_config : base64encode(templatefile("${path.module}/templates/microk8s-config-manager.tftpl", { + microk8s_config : base64encode(templatefile("${path.module}/templates/snap/microk8s-config-manager.tftpl", { node_ip : var.private_node_ips[count.index] api_server_domain : local.kube_api_server_domain main_node : count.index == 0 @@ -93,26 +93,27 @@ resource "hcloud_server" "manager" { api_server_ip : var.create_loadbalancer ? hcloud_load_balancer.kubernetes[0].ipv4 : hcloud_primary_ip.ipv4_manager_address[0].ip_address cluster_token : random_id.cluster_token.hex })) - packages_setup : base64encode(templatefile("${path.module}/templates/install-packages.sh", { + snapcraft : base64encode(file("${path.module}/templates/snap/snapcraft.yaml")) + packages_setup : base64encode(templatefile("${path.module}/templates/scripts/install-packages.sh", { refresh_day : "fri" refresh_hour : format("%02d", count.index) })) - firewall_setup : base64encode(templatefile("${path.module}/templates/firewall-setup.sh", { + firewall_setup : base64encode(templatefile("${path.module}/templates/scripts/firewall-setup.sh", { node_ip : var.private_node_ips[count.index] })) - ssh_setup : base64encode(templatefile("${path.module}/templates/ssh-setup.sh", { + ssh_setup : base64encode(templatefile("${path.module}/templates/scripts/ssh-setup.sh", { admin_user : var.admin_user })) - microk8s_setup : base64encode(templatefile("${path.module}/templates/microk8s-setup.sh", { + microk8s_setup : base64encode(templatefile("${path.module}/templates/scripts/microk8s-setup.sh", { microk8s_channel : var.microk8s_channel main_node = count.index == 0 })) - cluster_setup_values : base64encode(templatefile("${path.module}/templates/cluster-setup-values.yaml", { + cluster_setup_values : base64encode(templatefile("${path.module}/templates/helm/cluster-setup-values.yaml", { client_id : var.vault_service_principal.client_id client_secret : var.vault_service_principal.client_secret })) - argocd_ha_values : base64encode(file("${path.module}/templates/argocd-values-ha.yaml")) - cluster_setup : base64encode(templatefile("${path.module}/templates/cluster-setup.sh", { + argocd_ha_values : base64encode(file("${path.module}/templates/helm/argocd-values-ha.yaml")) + cluster_setup : base64encode(templatefile("${path.module}/templates/scripts/cluster-setup.sh", { argocd_environment : var.argocd_environment high_availability : var.number_nodes > 2 })) diff --git a/terraform/modules/k8s-cluster/templates/cloud-init-k8s-manager.tftpl b/terraform/modules/k8s-cluster/templates/cloud-init-k8s-manager.tftpl index c804113..6a7aadc 100644 --- a/terraform/modules/k8s-cluster/templates/cloud-init-k8s-manager.tftpl +++ b/terraform/modules/k8s-cluster/templates/cloud-init-k8s-manager.tftpl @@ -20,6 +20,11 @@ write_files: path: /run/tmpfiles.d/microk8s.yaml encoding: base64 defer: true + - owner: root:root + content: "${snapcraft}" + path: /run/tmpfiles.d/snapcraft.yaml + encoding: base64 + defer: true - owner: root:root content: "${packages_setup}" path: /run/tmpfiles.d/install-packages.sh diff --git a/terraform/modules/k8s-cluster/templates/cloud-init-k8s-worker.tftpl b/terraform/modules/k8s-cluster/templates/cloud-init-k8s-worker.tftpl index f9b9f00..e0939f8 100644 --- a/terraform/modules/k8s-cluster/templates/cloud-init-k8s-worker.tftpl +++ b/terraform/modules/k8s-cluster/templates/cloud-init-k8s-worker.tftpl @@ -20,6 +20,11 @@ write_files: path: /run/tmpfiles.d/microk8s.yaml encoding: base64 defer: true + - owner: root:root + content: "${snapcraft}" + path: /run/tmpfiles.d/snapcraft.yaml + encoding: base64 + defer: true - owner: root:root content: "${packages_setup}" path: /run/tmpfiles.d/install-packages.sh diff --git a/terraform/modules/k8s-cluster/templates/cluster-setup.sh b/terraform/modules/k8s-cluster/templates/cluster-setup.sh deleted file mode 100644 index 98aa709..0000000 --- a/terraform/modules/k8s-cluster/templates/cluster-setup.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -microk8s helm repo add 2martens https://repo.2martens.de/charts/ -microk8s helm install setup 2martens/cluster_setup --values /run/tmpfiles.d/cluster-setup-values.yaml -%{ if high_availability ~} -microk8s helm install argocd 2martens/argocd --set environment=${argocd_environment} --values /run/tmpfiles.d/argocd-values-ha.yaml -%{ else ~} -microk8s helm install argocd 2martens/argocd --set environment=${argocd_environment} -%{ endif ~} \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/templates/argocd-values-ha.yaml b/terraform/modules/k8s-cluster/templates/helm/argocd-values-ha.yaml similarity index 100% rename from terraform/modules/k8s-cluster/templates/argocd-values-ha.yaml rename to terraform/modules/k8s-cluster/templates/helm/argocd-values-ha.yaml diff --git a/terraform/modules/k8s-cluster/templates/cluster-setup-values.yaml b/terraform/modules/k8s-cluster/templates/helm/cluster-setup-values.yaml similarity index 100% rename from terraform/modules/k8s-cluster/templates/cluster-setup-values.yaml rename to terraform/modules/k8s-cluster/templates/helm/cluster-setup-values.yaml diff --git a/terraform/modules/k8s-cluster/templates/microk8s-setup.sh b/terraform/modules/k8s-cluster/templates/microk8s-setup.sh deleted file mode 100644 index 83d2fe8..0000000 --- a/terraform/modules/k8s-cluster/templates/microk8s-setup.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -cp /run/tmpfiles.d/microk8s.yaml /etc/microk8s.yaml -%{ if !main_node ~} -sleep 120 -%{ endif ~} -snap install microk8s --channel "${microk8s_channel}" diff --git a/terraform/modules/k8s-cluster/templates/add-node.sh b/terraform/modules/k8s-cluster/templates/scripts/add-node.sh similarity index 100% rename from terraform/modules/k8s-cluster/templates/add-node.sh rename to terraform/modules/k8s-cluster/templates/scripts/add-node.sh diff --git a/terraform/modules/k8s-cluster/templates/scripts/cluster-setup.sh b/terraform/modules/k8s-cluster/templates/scripts/cluster-setup.sh new file mode 100644 index 0000000..99b7a31 --- /dev/null +++ b/terraform/modules/k8s-cluster/templates/scripts/cluster-setup.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +/snap/bin/microk8s helm repo add 2martens https://repo.2martens.de/charts/ +/snap/bin/microk8s helm install setup 2martens/cluster_setup --values /run/tmpfiles.d/cluster-setup-values.yaml +%{ if high_availability ~} +/snap/bin/microk8s helm install argocd 2martens/argocd --set environment=${argocd_environment} --values /run/tmpfiles.d/argocd-values-ha.yaml +%{ else ~} +/snap/bin/microk8s helm install argocd 2martens/argocd --set environment=${argocd_environment} +%{ endif ~} \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/templates/firewall-setup.sh b/terraform/modules/k8s-cluster/templates/scripts/firewall-setup.sh similarity index 100% rename from terraform/modules/k8s-cluster/templates/firewall-setup.sh rename to terraform/modules/k8s-cluster/templates/scripts/firewall-setup.sh diff --git a/terraform/modules/k8s-cluster/templates/install-packages.sh b/terraform/modules/k8s-cluster/templates/scripts/install-packages.sh similarity index 100% rename from terraform/modules/k8s-cluster/templates/install-packages.sh rename to terraform/modules/k8s-cluster/templates/scripts/install-packages.sh diff --git a/terraform/modules/k8s-cluster/templates/join-worker.sh b/terraform/modules/k8s-cluster/templates/scripts/join-worker.sh similarity index 100% rename from terraform/modules/k8s-cluster/templates/join-worker.sh rename to terraform/modules/k8s-cluster/templates/scripts/join-worker.sh diff --git a/terraform/modules/k8s-cluster/templates/join.sh b/terraform/modules/k8s-cluster/templates/scripts/join.sh similarity index 100% rename from terraform/modules/k8s-cluster/templates/join.sh rename to terraform/modules/k8s-cluster/templates/scripts/join.sh diff --git a/terraform/modules/k8s-cluster/templates/scripts/microk8s-setup.sh b/terraform/modules/k8s-cluster/templates/scripts/microk8s-setup.sh new file mode 100644 index 0000000..511ef7e --- /dev/null +++ b/terraform/modules/k8s-cluster/templates/scripts/microk8s-setup.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +mkdir -p /run/snap/launcher/configuration +cp /run/tmpfiles.d/microk8s.yaml /run/snap/launcher/configuration/content.yaml +cp /run/tmpfiles.d/snapcraft.yaml /run/snap/snapcraft.yaml +cd /run/snap && snapcraft +snap install /run/snap/microk8s-cluster-config_*.snap --dangerous + +%{ if !main_node ~} +sleep 120 +%{ endif ~} +snap install microk8s --channel "${microk8s_channel}" +snap connect microk8s-cluster-config:configuration microk8s diff --git a/terraform/modules/k8s-cluster/templates/ssh-setup.sh b/terraform/modules/k8s-cluster/templates/scripts/ssh-setup.sh similarity index 100% rename from terraform/modules/k8s-cluster/templates/ssh-setup.sh rename to terraform/modules/k8s-cluster/templates/scripts/ssh-setup.sh diff --git a/terraform/modules/k8s-cluster/templates/microk8s-config-manager.tftpl b/terraform/modules/k8s-cluster/templates/snap/microk8s-config-manager.tftpl similarity index 100% rename from terraform/modules/k8s-cluster/templates/microk8s-config-manager.tftpl rename to terraform/modules/k8s-cluster/templates/snap/microk8s-config-manager.tftpl diff --git a/terraform/modules/k8s-cluster/templates/microk8s-config-worker.tftpl b/terraform/modules/k8s-cluster/templates/snap/microk8s-config-worker.tftpl similarity index 100% rename from terraform/modules/k8s-cluster/templates/microk8s-config-worker.tftpl rename to terraform/modules/k8s-cluster/templates/snap/microk8s-config-worker.tftpl diff --git a/terraform/modules/k8s-cluster/templates/snap/snapcraft.yaml b/terraform/modules/k8s-cluster/templates/snap/snapcraft.yaml new file mode 100644 index 0000000..0876a86 --- /dev/null +++ b/terraform/modules/k8s-cluster/templates/snap/snapcraft.yaml @@ -0,0 +1,23 @@ +name: microk8s-cluster-config +version: 'latest' +summary: Adds a launch configuration to MicroK8s +description: Configuration yaml file that MicroK8s can apply. +base: core20 + +grade: stable +confinement: strict + +slots: + configuration: + interface: content + content: configuration + source: + read: + - $SNAP/launcher + +parts: + configuration: + plugin: dump + source: . + stage: + - launcher/* \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/worker-node.tf b/terraform/modules/k8s-cluster/worker-node.tf index 083b8c1..f01b880 100644 --- a/terraform/modules/k8s-cluster/worker-node.tf +++ b/terraform/modules/k8s-cluster/worker-node.tf @@ -64,7 +64,7 @@ resource "hcloud_server" "worker" { admin_user : var.admin_user terraform_public_ssh_key : var.terraform_public_ssh_key main_node : count.index == 0 - microk8s_config : base64encode(templatefile("${path.module}/templates/microk8s-config-worker.tftpl", { + microk8s_config : base64encode(templatefile("${path.module}/templates/snap/microk8s-config-worker.tftpl", { node_ip : var.private_node_ips[count.index] api_server_domain : local.kube_api_server_domain main_node : count.index == 0 @@ -72,26 +72,27 @@ resource "hcloud_server" "worker" { api_server_ip : var.create_loadbalancer ? hcloud_load_balancer.kubernetes[0].ipv4 : hcloud_server.manager[0].ipv4_address cluster_token : random_id.cluster_token.hex })) - packages_setup : base64encode(templatefile("${path.module}/templates/install-packages.sh", { + snapcraft : base64encode(file("${path.module}/templates/snap/snapcraft.yaml")) + packages_setup : base64encode(templatefile("${path.module}/templates/scripts/install-packages.sh", { refresh_day : "sat" refresh_hour : format("%02d", count.index) })) - firewall_setup : base64encode(templatefile("${path.module}/templates/firewall-setup.sh", { + firewall_setup : base64encode(templatefile("${path.module}/templates/scripts/firewall-setup.sh", { node_ip : var.private_node_ips[count.index] })) - ssh_setup : base64encode(templatefile("${path.module}/templates/ssh-setup.sh", { + ssh_setup : base64encode(templatefile("${path.module}/templates/scripts/ssh-setup.sh", { admin_user : var.admin_user })) - microk8s_setup : base64encode(templatefile("${path.module}/templates/microk8s-setup.sh", { + microk8s_setup : base64encode(templatefile("${path.module}/templates/scripts/microk8s-setup.sh", { microk8s_channel : var.microk8s_channel main_node : false })) - cluster_setup_values : base64encode(templatefile("${path.module}/templates/cluster-setup-values.yaml", { + cluster_setup_values : base64encode(templatefile("${path.module}/templates/helm/cluster-setup-values.yaml", { client_id : var.vault_service_principal.client_id client_secret : var.vault_service_principal.client_secret })) - argocd_ha_values : base64encode(file("${path.module}/templates/argocd-values-ha.yaml")) - cluster_setup : base64encode(templatefile("${path.module}/templates/cluster-setup.sh", { + argocd_ha_values : base64encode(file("${path.module}/templates/helm/argocd-values-ha.yaml")) + cluster_setup : base64encode(templatefile("${path.module}/templates/scripts/cluster-setup.sh", { argocd_environment : var.argocd_environment high_availability : var.number_nodes > 2 }))