diff --git a/terraform/hetzner.tf b/terraform/hetzner.tf index 16a8b12..9951217 100644 --- a/terraform/hetzner.tf +++ b/terraform/hetzner.tf @@ -179,15 +179,33 @@ resource "hcloud_server" "server_k8s_test" { firewall_ids = [hcloud_firewall.basic-firewall.id, hcloud_firewall.k8s-firewall.id] ssh_keys = [data.hcloud_ssh_key.macos.id] user_data = templatefile("${path.module}/modules/k8s-cluster/templates/cloud-init-k8s.tftpl", { - node_ip : "10.0.0.2", - admin_public_ssh_key : format("%s %s", data.hcloud_ssh_key.macos.public_key, data.hcloud_ssh_key.macos.name), - terraform_public_ssh_key : var.terraform_public_ssh_key, - microk8s_channel : "1.28-strict/stable", - admin_user : "2martensAdmin", - "microk8s_config" : templatefile("${path.module}/modules/k8s-cluster/templates/microk8s-config.tftpl", { - node_ip : "10.0.0.2", + admin_public_ssh_key : format("%s %s", data.hcloud_ssh_key.macos.public_key, data.hcloud_ssh_key.macos.name) + admin_user : "2martensAdmin" + terraform_public_ssh_key : var.terraform_public_ssh_key + main_node : true + microk8s_config : base64encode(templatefile("${path.module}/templates/microk8s-config.tftpl", { + node_ip : "10.0.0.2" api_server_domain : "" - }) + })) + packages_setup : base64encode(file("${path.module}/templates/install-packages.sh")) + firewall_setup : base64encode(templatefile("${path.module}/templates/firewall-setup.sh", { + node_ip : "10.0.0.2" + })) + ssh_setup : base64encode(templatefile("${path.module}/templates/ssh-setup.sh", { + admin_user : "2martensAdmin" + })) + microk8s_setup : base64encode(templatefile("${path.module}/templates/microk8s-setup.sh", { + microk8s_channel : "1.28-strict/stable" + })) + cluster_setup_values : base64encode(templatefile("${path.module}/templates/cluster-setup-values.yaml", { + client_id : var.vault_client_id + client_secret : var.vault_client_secret + })) + cluster_setup_values : base64encode(file("${path.module}/templates/argocd-values-ha.yaml")) + cluster_setup : base64encode(templatefile("${path.module}/templates/cluster-setup.sh", { + argocd_environment : "test" + high_availability : false + })) }) shutdown_before_deletion = true diff --git a/terraform/modules/k8s-cluster/ha.tf b/terraform/modules/k8s-cluster/ha.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/modules/k8s-cluster/helm.tf b/terraform/modules/k8s-cluster/helm.tf index 140bb28..245c4d4 100644 --- a/terraform/modules/k8s-cluster/helm.tf +++ b/terraform/modules/k8s-cluster/helm.tf @@ -1,36 +1,3 @@ -resource "null_resource" "install_setup" { - connection { - host = hcloud_server.manager[0].ipv6_address - user = "terraform" - type = "ssh" - private_key = var.terraform_private_ssh_key - timeout = "10m" - } - - provisioner "remote-exec" { - inline = [ - "while (test -z `command -v helm`); do echo \"Waiting for cloud init to finish...\";sleep 5;done" - ] - } - - provisioner "file" { - content = templatefile("${path.module}/templates/cluster-setup-values.yaml", { - client_id : var.vault_service_principal.client_id, - client_secret : var.vault_service_principal.client_secret - }) - destination = "/home/terraform/cluster-setup-values.yaml" - } - - provisioner "remote-exec" { - inline = [ - "helm repo add 2martens https://repo.2martens.de/charts/", - "helm install setup 2martens/cluster_setup --values /home/terraform/cluster-setup-values.yaml", - ] - } - - depends_on = [null_resource.join_nodes, hcloud_server.manager] -} - resource "inwx_nameserver_record" "argocd_a" { domain = var.domain name = format("%s.%s", "argocd", local.kube_api_server_domain) @@ -45,64 +12,3 @@ resource "inwx_nameserver_record" "argocd_aaaa" { type = "AAAA" ttl = 3600 } - -resource "null_resource" "install_argocd_single" { - count = var.number_nodes < 3 ? 1 : 0 - - connection { - host = hcloud_server.manager[0].ipv6_address - user = "terraform" - type = "ssh" - private_key = var.terraform_private_ssh_key - timeout = "10m" - } - - provisioner "remote-exec" { - inline = [ - "while (test -z `command -v helm`); do echo \"Waiting for cloud init to finish...\";sleep 5;done" - ] - } - - provisioner "remote-exec" { - inline = [ - "helm repo add 2martens https://repo.2martens.de/charts/", - "helm install setup 2martens/cluster_setup", - "helm install argocd 2martens/argocd --set environment=${var.argocd_environment}", - ] - } - - depends_on = [null_resource.join_nodes, hcloud_server.manager] -} - -resource "null_resource" "install_argocd_ha" { - count = var.number_nodes >= 3 ? 1 : 0 - - connection { - host = hcloud_server.manager[0].ipv6_address - user = "terraform" - type = "ssh" - private_key = var.terraform_private_ssh_key - timeout = "10m" - } - - provisioner "remote-exec" { - inline = [ - "while (test -z `command -v helm`); do echo \"Waiting for cloud init to finish...\";sleep 5;done" - ] - } - - provisioner "file" { - content = file("${path.module}/templates/argocd-values-ha.yaml") - destination = "/home/terraform/argocd-values-ha.yaml" - } - - provisioner "remote-exec" { - inline = [ - "helm repo add 2martens https://repo.2martens.de/charts/", - "helm install setup 2martens/cluster_setup", - "helm install argocd 2martens/argocd --set environment=${var.argocd_environment} --values /home/terraform/argocd-values-ha.yaml", - ] - } - - depends_on = [null_resource.join_nodes, hcloud_server.manager] -} \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/manager-node.tf b/terraform/modules/k8s-cluster/manager-node.tf index 6219514..c0552b3 100644 --- a/terraform/modules/k8s-cluster/manager-node.tf +++ b/terraform/modules/k8s-cluster/manager-node.tf @@ -80,15 +80,38 @@ resource "hcloud_server" "manager" { "manager" : "yes" } - user_data = templatefile("${path.module}/templates/cloud-init-k8s.tftpl", { - node_ip : var.private_node_ips[count.index], - admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name), - terraform_public_ssh_key : var.terraform_public_ssh_key, - microk8s_channel : var.microk8s_channel, - admin_user : var.admin_user, - microk8s_config : base64encode(templatefile("${path.module}/templates/microk8s-config.tftpl", { - node_ip : var.private_node_ips[count.index], + user_data = templatefile("${path.module}/templates/cloud-init-k8s-manager.tftpl", { + admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name) + admin_user : var.admin_user + terraform_public_ssh_key : var.terraform_public_ssh_key + main_node : count.index == 0 + microk8s_config : base64encode(templatefile("${path.module}/templates/microk8s-config-manager.tftpl", { + node_ip : var.private_node_ips[count.index] api_server_domain : local.kube_api_server_domain + main_node : count.index == 0 + main_node_ip : var.private_node_ips[0] + api_server_ip : var.create_loadbalancer ? hcloud_load_balancer.kubernetes[0].ipv4 : hcloud_primary_ip.ipv4_manager_address[0].ip_address + cluster_token : random_id.cluster_token.hex + })) + packages_setup : base64encode(file("${path.module}/templates/install-packages.sh")) + firewall_setup : base64encode(templatefile("${path.module}/templates/firewall-setup.sh", { + node_ip : var.private_node_ips[count.index] + })) + ssh_setup : base64encode(templatefile("${path.module}/templates/ssh-setup.sh", { + admin_user : var.admin_user + })) + microk8s_setup : base64encode(templatefile("${path.module}/templates/microk8s-setup.sh", { + microk8s_channel : var.microk8s_channel + main_node = count.index == 0 + })) + cluster_setup_values : base64encode(templatefile("${path.module}/templates/cluster-setup-values.yaml", { + client_id : var.vault_service_principal.client_id + client_secret : var.vault_service_principal.client_secret + })) + cluster_setup_values : base64encode(file("${path.module}/templates/argocd-values-ha.yaml")) + cluster_setup : base64encode(templatefile("${path.module}/templates/cluster-setup.sh", { + argocd_environment : var.argocd_environment + high_availability : var.number_nodes > 2 })) }) @@ -107,96 +130,84 @@ resource "random_id" "cluster_token" { byte_length = 16 } -resource "null_resource" "setup_tokens" { - count = var.number_nodes > 1 ? 1 : 0 - - triggers = { - rerun = random_id.cluster_token.hex - } - - connection { - host = hcloud_server.manager[0].ipv6_address - user = "terraform" - type = "ssh" - private_key = var.terraform_private_ssh_key - timeout = "10m" - } - - provisioner "local-exec" { - interpreter = ["bash", "-c"] - command = < /tmp/current_joining_node.txt - echo "0" > /tmp/current_joining_worker_node.txt - EOT - } - - provisioner "remote-exec" { - inline = [ - "while (test -z `command -v microk8s`); do echo \"Waiting for cloud init to finish...\";sleep 5;done" - ] - } - - provisioner "file" { - content = templatefile("${path.module}/templates/add-node.sh", - { - cluster_token = random_id.cluster_token.hex - cluster_token_ttl_seconds = var.cluster_token_ttl_seconds - }) - destination = "/usr/local/bin/add-node.sh" - } - - provisioner "remote-exec" { - inline = [ - "sh /usr/local/bin/add-node.sh", - ] - } -} - - -resource "null_resource" "join_nodes" { - count = var.number_nodes - 1 < 1 ? 0 : var.number_nodes - 1 - - triggers = { - rerun = random_id.cluster_token.hex - } - connection { - host = element(hcloud_server.manager.*.ipv6_address, count.index + 1) - user = "terraform" - type = "ssh" - private_key = var.terraform_private_ssh_key - timeout = "20m" - } - - provisioner "local-exec" { - interpreter = ["bash", "-c"] - command = "while [[ $(cat /tmp/current_joining_node.txt) != \"${count.index + 1}\" ]]; do echo \"${count.index + 1} is waiting...\";sleep 5;done" - } - - provisioner "remote-exec" { - inline = [ - "while (test -z `command -v microk8s`); do echo \"Waiting for cloud init to finish...\";sleep 5;done" - ] - } - - provisioner "file" { - content = templatefile("${path.module}/templates/join.sh", - { - cluster_token = random_id.cluster_token.hex - main_node_ip = hcloud_server_network.manager_private[0].ip - }) - destination = "/usr/local/bin/join.sh" - } - - provisioner "remote-exec" { - inline = [ - "sh /usr/local/bin/join.sh" - ] - } - - provisioner "local-exec" { - interpreter = ["bash", "-c"] - command = "echo \"${count.index + 2}\" > /tmp/current_joining_node.txt" - } - - depends_on = [null_resource.setup_tokens] -} \ No newline at end of file +#resource "null_resource" "setup_tokens" { +# count = var.number_nodes > 1 ? 1 : 0 +# +# triggers = { +# rerun = random_id.cluster_token.hex +# } +# +# connection { +# host = hcloud_server.manager[0].ipv6_address +# user = "terraform" +# type = "ssh" +# private_key = var.terraform_private_ssh_key +# timeout = "10m" +# } +# +# provisioner "local-exec" { +# interpreter = ["bash", "-c"] +# command = < /tmp/current_joining_node.txt +# echo "0" > /tmp/current_joining_worker_node.txt +# EOT +# } +# +# provisioner "file" { +# content = templatefile("${path.module}/templates/add-node.sh", +# { +# cluster_token = random_id.cluster_token.hex +# cluster_token_ttl_seconds = var.cluster_token_ttl_seconds +# }) +# destination = "/usr/local/bin/add-node.sh" +# } +# +# provisioner "remote-exec" { +# inline = [ +# "sh /usr/local/bin/add-node.sh", +# ] +# } +#} +# +# +#resource "null_resource" "join_nodes" { +# count = var.number_nodes - 1 < 1 ? 0 : var.number_nodes - 1 +# +# triggers = { +# rerun = random_id.cluster_token.hex +# } +# connection { +# host = element(hcloud_server.manager.*.ipv6_address, count.index + 1) +# user = "terraform" +# type = "ssh" +# private_key = var.terraform_private_ssh_key +# timeout = "20m" +# } +# +# provisioner "local-exec" { +# interpreter = ["bash", "-c"] +# command = "while [[ $(cat /tmp/current_joining_node.txt) != \"${count.index + 1}\" ]]; do echo \"${count.index + 1} is waiting...\";sleep 5;done" +# } +# +# provisioner "file" { +# content = templatefile("${path.module}/templates/join.sh", +# { +# cluster_token = random_id.cluster_token.hex +# main_node_ip = hcloud_server_network.manager_private[0].ip +# }) +# destination = "/usr/local/bin/join.sh" +# } +# +# provisioner "remote-exec" { +# inline = [ +# "sh /usr/local/bin/join.sh" +# ] +# } +# +# provisioner "local-exec" { +# interpreter = ["bash", "-c"] +# command = "echo \"${count.index + 2}\" > /tmp/current_joining_node.txt" +# } +# +# depends_on = [null_resource.setup_tokens] +#} \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/templates/add-node.sh b/terraform/modules/k8s-cluster/templates/add-node.sh index 4985659..f548866 100644 --- a/terraform/modules/k8s-cluster/templates/add-node.sh +++ b/terraform/modules/k8s-cluster/templates/add-node.sh @@ -1,5 +1,9 @@ #!/bin/sh +while test -z "\${command -v microk8s}"; + sleep 5; do echo \"Waiting for cloud init to finish...\"; +done + until microk8s.status --wait-ready; do sleep 5; echo "waiting for worker status.."; done diff --git a/terraform/modules/k8s-cluster/templates/cloud-init-k8s-manager.tftpl b/terraform/modules/k8s-cluster/templates/cloud-init-k8s-manager.tftpl new file mode 100644 index 0000000..2147d11 --- /dev/null +++ b/terraform/modules/k8s-cluster/templates/cloud-init-k8s-manager.tftpl @@ -0,0 +1,66 @@ +#cloud-config +users: + - name: ${admin_user} + groups: users, admin + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + ssh_authorized_keys: + - ${admin_public_ssh_key} + - name: terraform + groups: users, admin + sudo: ALL=(ALL) NOPASSWD:ALL + ssh_authorized_keys: + - ${terraform_public_ssh_key} +package_update: true +package_upgrade: true +package_reboot_if_required: true +write_files: + - owner: 2martensAdmin:2martensAdmin + content: "${microk8s_config}" + path: /home/2martensAdmin/.microk8s.yaml + encoding: base64 + - owner: root:root + content: "${packages_setup}" + path: /usr/local/bin/install-packages.sh + encoding: base64 + permissions: "0755" + - owner: root:root + content: "${firewall_setup}" + path: /usr/local/bin/firewall-setup.sh + encoding: base64 + permissions: "0755" + - owner: root:root + content: "${ssh_setup}" + path: /usr/local/bin/ssh-setup.sh + encoding: base64 + permissions: "0755" + - owner: root:root + content: "${microk8s_setup}" + path: /usr/local/bin/microk8s-setup.sh + encoding: base64 + permissions: "0755" +%{ if main_node ~} + - owner: root:root + content: "${cluster_setup}" + path: /usr/local/bin/cluster-setup.sh + encoding: base64 + permissions: "0755" + - owner: root:root + content: "${cluster_setup_values}" + path: /home/terraform/cluster-setup-values.yaml + encoding: base64 + permissions: "0755" + - owner: root:root + content: "${argocd_ha_values}" + path: /home/terraform/argocd-values-ha.yaml + encoding: base64 + permissions: "0755" +%{ endif ~} +runcmd: + - /usr/local/bin/install-packages.sh + - /usr/local/bin/firewall-setup.sh + - /usr/local/bin/ssh-setup.sh + - /usr/local/bin/microk8s-setup.sh +%{ if main_node ~} + - /usr/local/bin/cluster-setup.sh +%{ endif ~} \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/templates/cloud-init-k8s-worker.tftpl b/terraform/modules/k8s-cluster/templates/cloud-init-k8s-worker.tftpl new file mode 100644 index 0000000..bebc068 --- /dev/null +++ b/terraform/modules/k8s-cluster/templates/cloud-init-k8s-worker.tftpl @@ -0,0 +1,52 @@ +#cloud-config +users: + - name: ${admin_user} + groups: users, admin + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + ssh_authorized_keys: + - ${admin_public_ssh_key} + - name: terraform + groups: users, admin + sudo: ALL=(ALL) NOPASSWD:ALL + ssh_authorized_keys: + - ${terraform_public_ssh_key} +package_update: true +package_upgrade: true +package_reboot_if_required: true +write_files: + - owner: 2martensAdmin:2martensAdmin + content: "${microk8s_config}" + path: /home/2martensAdmin/.microk8s.yaml + encoding: base64 + - owner: root:root + content: "${packages_setup}" + path: /usr/local/bin/install-packages.sh + encoding: base64 + permissions: "0755" + - owner: root:root + content: "${firewall_setup}" + path: /usr/local/bin/firewall-setup.sh + encoding: base64 + permissions: "0755" + - owner: root:root + content: "${ssh_setup}" + path: /usr/local/bin/ssh-setup.sh + encoding: base64 + permissions: "0755" + - owner: root:root + content: "${microk8s_setup}" + path: /usr/local/bin/microk8s-setup.sh + encoding: base64 + permissions: "0755" + - owner: root:root + content: "${join_worker}" + path: /usr/local/bin/join-worker.sh + encoding: base64 + permissions: "0755" +runcmd: + - /usr/local/bin/install-packages.sh + - /usr/local/bin/firewall-setup.sh + - /usr/local/bin/ssh-setup.sh + - /usr/local/bin/microk8s-setup.sh + diff --git a/terraform/modules/k8s-cluster/templates/cloud-init-k8s.tftpl b/terraform/modules/k8s-cluster/templates/cloud-init-k8s.tftpl deleted file mode 100644 index 8f5b1da..0000000 --- a/terraform/modules/k8s-cluster/templates/cloud-init-k8s.tftpl +++ /dev/null @@ -1,56 +0,0 @@ -#cloud-config -users: - - name: ${admin_user} - groups: users, admin - sudo: ALL=(ALL) NOPASSWD:ALL - shell: /bin/bash - ssh_authorized_keys: - - ${admin_public_ssh_key} - - name: terraform - groups: users, admin - sudo: ALL=(ALL) NOPASSWD:ALL - ssh_authorized_keys: - - ${terraform_public_ssh_key} -packages: - - fail2ban - - ufw -package_update: true -package_upgrade: true -package_reboot_if_required: true -write_files: - - owner: root:root - content: "${microk8s_config}" - path: /tmp/.microk8s.yaml - encoding: base64 -runcmd: - - printf "[sshd]\nenabled = true\nbanaction = iptables-multiport" > /etc/fail2ban/jail.local - - systemctl enable fail2ban - - ufw allow OpenSSH - - ufw allow 16443 comment "Kubernetes API endpoint" - - ufw allow 80 comment HTTP - - ufw allow 443 comment HTTPS - - ufw allow to ${node_ip} port 9100 comment "Metrics endpoint" - - ufw allow to ${node_ip} port 10250 comment "cadvisor port" - - ufw allow to ${node_ip} port 10259 comment "kube-scheduler port" - - ufw allow to ${node_ip} port 10257 comment "kube-controller-manager port" - - ufw enable - - sed -i -e '/^\(#\|\)PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config - - sed -i -e '/^\(#\|\)PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config - - sed -i -e '/^\(#\|\)X11Forwarding/s/^.*$/X11Forwarding no/' /etc/ssh/sshd_config - - sed -i -e '/^\(#\|\)MaxAuthTries/s/^.*$/MaxAuthTries 2/' /etc/ssh/sshd_config - - sed -i -e '/^\(#\|\)AllowTcpForwarding/s/^.*$/AllowTcpForwarding no/' /etc/ssh/sshd_config - - sed -i -e '/^\(#\|\)AllowAgentForwarding/s/^.*$/AllowAgentForwarding no/' /etc/ssh/sshd_config - - sed -i -e '/^\(#\|\)AuthorizedKeysFile/s/^.*$/AuthorizedKeysFile .ssh\/authorized_keys/' /etc/ssh/sshd_config - - sed -i '$a AllowUsers ${admin_user} terraform' /etc/ssh/sshd_config - - curl https://packages.hetzner.com/hcloud/deb/hc-utils_0.0.4-1_all.deb -o /tmp/hc-utils_0.0.4-1_all.deb -s - - apt install /tmp/hc-utils_0.0.4-1_all.deb - - apt-get -y install snapd - - snap install kubectl --classic - - mkdir -p /var/snap/microk8s/common/etc/launcher - - cp /tmp/.microk8s.yaml /var/snap/microk8s/common/etc/launcher/.microk8s.yaml - - snap install microk8s --channel "${microk8s_channel}" - - mkdir -p /home/terraform/.kube - - microk8s kubectl config view --raw > /home/terraform/.kube/config - - chown -R terraform:terraform /home/terraform/.kube - - chmod go-r /home/terraform/.kube/config - - microk8s.status --wait-ready \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/templates/cluster-setup.sh b/terraform/modules/k8s-cluster/templates/cluster-setup.sh new file mode 100644 index 0000000..0413b3c --- /dev/null +++ b/terraform/modules/k8s-cluster/templates/cluster-setup.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +helm repo add 2martens https://repo.2martens.de/charts/ +helm install setup 2martens/cluster_setup --values /home/terraform/cluster-setup-values.yaml +%{ if high_availability ~} +helm install argocd 2martens/argocd --set environment=${argocd_environment} --values /home/terraform/argocd-values-ha.yaml +%{ else ~} +helm install argocd 2martens/argocd --set environment=${argocd_environment} +%{ endif ~} \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/templates/firewall-setup.sh b/terraform/modules/k8s-cluster/templates/firewall-setup.sh new file mode 100644 index 0000000..a0bfd76 --- /dev/null +++ b/terraform/modules/k8s-cluster/templates/firewall-setup.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +printf "[sshd]\nenabled = true\nbanaction = iptables-multiport" > /etc/fail2ban/jail.local +systemctl enable fail2ban + +ufw allow OpenSSH +ufw allow 16443 comment "Kubernetes API endpoint" +ufw allow 80 comment HTTP +ufw allow 443 comment HTTPS +ufw allow to ${node_ip} port 9100 comment "Metrics endpoint" +ufw allow to ${node_ip} port 10250 comment "cadvisor port" +ufw allow to ${node_ip} port 10259 comment "kube-scheduler port" +ufw allow to ${node_ip} port 10257 comment "kube-controller-manager port" +ufw allow to ${node_ip} port 25000 comment "port for joining nodes" +ufw enable \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/templates/install-packages.sh b/terraform/modules/k8s-cluster/templates/install-packages.sh new file mode 100644 index 0000000..4267ff8 --- /dev/null +++ b/terraform/modules/k8s-cluster/templates/install-packages.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +apt-get -y install fail2ban +apt-get -y install ufw +apt-get -y install snapd + +curl https://packages.hetzner.com/hcloud/deb/hc-utils_0.0.4-1_all.deb -o /tmp/hc-utils_0.0.4-1_all.deb -s +apt install /tmp/hc-utils_0.0.4-1_all.deb +snap install kubectl --classic \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/templates/join-worker.sh b/terraform/modules/k8s-cluster/templates/join-worker.sh index 16b74a9..89241ed 100644 --- a/terraform/modules/k8s-cluster/templates/join-worker.sh +++ b/terraform/modules/k8s-cluster/templates/join-worker.sh @@ -1,5 +1,9 @@ #!/bin/sh +while test -z "\${command -v microk8s}"; + sleep 5; do echo \"Waiting for cloud init to finish...\"; +done + until microk8s.status --wait-ready; do sleep 3; echo "waiting for worker status.."; done diff --git a/terraform/modules/k8s-cluster/templates/join.sh b/terraform/modules/k8s-cluster/templates/join.sh index 693b664..ae9fd37 100644 --- a/terraform/modules/k8s-cluster/templates/join.sh +++ b/terraform/modules/k8s-cluster/templates/join.sh @@ -1,5 +1,9 @@ #!/bin/sh +while test -z "\${command -v microk8s}"; + sleep 5; do echo \"Waiting for cloud init to finish...\"; +done + until microk8s.status --wait-ready; do sleep 3; echo "waiting for worker status.."; done diff --git a/terraform/modules/k8s-cluster/templates/microk8s-config.tftpl b/terraform/modules/k8s-cluster/templates/microk8s-config-manager.tftpl similarity index 72% rename from terraform/modules/k8s-cluster/templates/microk8s-config.tftpl rename to terraform/modules/k8s-cluster/templates/microk8s-config-manager.tftpl index 9255849..fa2e656 100644 --- a/terraform/modules/k8s-cluster/templates/microk8s-config.tftpl +++ b/terraform/modules/k8s-cluster/templates/microk8s-config-manager.tftpl @@ -1,7 +1,14 @@ --- version: 0.2.0 +%{ if main_node ~} +persistentClusterToken: ${cluster_token} +%{ else ~} +join: + url: "${main_node_ip}:25000/${cluster_token}" + worker: false +%{ endif ~} extraKubeAPIServerArgs: - --advertise-address=${node_ip} + --advertise-address=${api_server_ip} --authorization-mode=RBAC,Node extraKubeletArgs: --node-ip=${node_ip} diff --git a/terraform/modules/k8s-cluster/templates/microk8s-config-worker.tftpl b/terraform/modules/k8s-cluster/templates/microk8s-config-worker.tftpl new file mode 100644 index 0000000..c10da38 --- /dev/null +++ b/terraform/modules/k8s-cluster/templates/microk8s-config-worker.tftpl @@ -0,0 +1,9 @@ +--- +version: 0.2.0 +join: + url: "${main_node_ip}:25000/${cluster_token}" + worker: true +extraKubeletArgs: + --node-ip=${node_ip} + --cluster-dns=10.152.183.10 + --cluster-domain=cluster.local diff --git a/terraform/modules/k8s-cluster/templates/microk8s-setup.sh b/terraform/modules/k8s-cluster/templates/microk8s-setup.sh new file mode 100644 index 0000000..3831e13 --- /dev/null +++ b/terraform/modules/k8s-cluster/templates/microk8s-setup.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +mkdir -p /var/snap/microk8s/common/etc/launcher +cp /home/terraform/.microk8s.yaml /var/snap/microk8s/common/etc/launcher/.microk8s.yaml +chown root:root /var/snap/microk8s/common/etc/launcher/.microk8s.yaml +%{ if !main_node ~} +sleep 120 +%{ endif ~} +snap install microk8s --channel "${microk8s_channel}" diff --git a/terraform/modules/k8s-cluster/templates/ssh-setup.sh b/terraform/modules/k8s-cluster/templates/ssh-setup.sh new file mode 100644 index 0000000..cf3c8d8 --- /dev/null +++ b/terraform/modules/k8s-cluster/templates/ssh-setup.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +sed -i -e '/^\(#\|\)PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config +sed -i -e '/^\(#\|\)PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config +sed -i -e '/^\(#\|\)X11Forwarding/s/^.*$/X11Forwarding no/' /etc/ssh/sshd_config +sed -i -e '/^\(#\|\)MaxAuthTries/s/^.*$/MaxAuthTries 2/' /etc/ssh/sshd_config +sed -i -e '/^\(#\|\)AllowTcpForwarding/s/^.*$/AllowTcpForwarding no/' /etc/ssh/sshd_config +sed -i -e '/^\(#\|\)AllowAgentForwarding/s/^.*$/AllowAgentForwarding no/' /etc/ssh/sshd_config +sed -i -e '/^\(#\|\)AuthorizedKeysFile/s/^.*$/AuthorizedKeysFile .ssh\/authorized_keys/' /etc/ssh/sshd_config +sed -i '$a AllowUsers ${admin_user} terraform' /etc/ssh/sshd_config \ No newline at end of file diff --git a/terraform/modules/k8s-cluster/worker-node.tf b/terraform/modules/k8s-cluster/worker-node.tf index a073e80..b5f83d2 100644 --- a/terraform/modules/k8s-cluster/worker-node.tf +++ b/terraform/modules/k8s-cluster/worker-node.tf @@ -59,15 +59,38 @@ resource "hcloud_server" "worker" { "worker" : "yes" } - user_data = templatefile("${path.module}/templates/cloud-init-k8s.tftpl", { - node_ip : var.private_worker_node_ips[count.index], - admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name), - terraform_public_ssh_key : var.terraform_public_ssh_key, - microk8s_channel : var.microk8s_channel, - admin_user : var.admin_user, - microk8s_config : base64encode(templatefile("${path.module}/templates/microk8s-config.tftpl", { - node_ip : var.private_worker_node_ips[count.index], + user_data = templatefile("${path.module}/templates/cloud-init-k8s-worker.tftpl", { + admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name) + admin_user : var.admin_user + terraform_public_ssh_key : var.terraform_public_ssh_key + main_node : count.index == 0 + microk8s_config : base64encode(templatefile("${path.module}/templates/microk8s-config-worker.tftpl", { + node_ip : var.private_node_ips[count.index] api_server_domain : local.kube_api_server_domain + main_node : count.index == 0 + main_node_ip : hcloud_server_network.manager_private[0].ip + api_server_ip : var.create_loadbalancer ? hcloud_load_balancer.kubernetes[0].ipv4 : hcloud_server.manager[0].ipv4_address + cluster_token : random_id.cluster_token.hex + })) + packages_setup : base64encode(file("${path.module}/templates/install-packages.sh")) + firewall_setup : base64encode(templatefile("${path.module}/templates/firewall-setup.sh", { + node_ip : var.private_node_ips[count.index] + })) + ssh_setup : base64encode(templatefile("${path.module}/templates/ssh-setup.sh", { + admin_user : var.admin_user + })) + microk8s_setup : base64encode(templatefile("${path.module}/templates/microk8s-setup.sh", { + microk8s_channel : var.microk8s_channel + main_node : false + })) + cluster_setup_values : base64encode(templatefile("${path.module}/templates/cluster-setup-values.yaml", { + client_id : var.vault_service_principal.client_id + client_secret : var.vault_service_principal.client_secret + })) + cluster_setup_values : base64encode(file("${path.module}/templates/argocd-values-ha.yaml")) + cluster_setup : base64encode(templatefile("${path.module}/templates/cluster-setup.sh", { + argocd_environment : var.argocd_environment + high_availability : var.number_nodes > 2 })) }) @@ -82,50 +105,44 @@ resource "hcloud_server" "worker" { } } -resource "null_resource" "join_workers" { - count = var.number_worker_nodes - - triggers = { - rerun = random_id.cluster_token.hex - } - connection { - host = element(hcloud_server.worker.*.ipv6_address, count.index) - user = "terraform" - type = "ssh" - private_key = var.terraform_private_ssh_key - timeout = "20m" - } - - provisioner "local-exec" { - interpreter = ["bash", "-c"] - command = "while [[ $(cat /tmp/current_joining_worker_node.txt) != \"${count.index}\" ]]; do echo \"${count.index} is waiting...\";sleep 5;done" - } - - provisioner "remote-exec" { - inline = [ - "while (test -z `command -v microk8s`); do echo \"Waiting for cloud init to finish...\";sleep 5;done" - ] - } - - provisioner "file" { - content = templatefile("${path.module}/templates/join-worker.sh", - { - cluster_token = random_id.cluster_token.hex - main_node_ip = hcloud_server_network.manager_private[0].ip - }) - destination = "/usr/local/bin/join-worker.sh" - } - - provisioner "remote-exec" { - inline = [ - "sh /usr/local/bin/join-worker.sh" - ] - } - - provisioner "local-exec" { - interpreter = ["bash", "-c"] - command = "echo \"${count.index + 1}\" > /tmp/current_joining_worker_node.txt" - } - - depends_on = [null_resource.setup_tokens, null_resource.join_nodes] -} \ No newline at end of file +#resource "null_resource" "join_workers" { +# count = var.number_worker_nodes +# +# triggers = { +# rerun = random_id.cluster_token.hex +# } +# connection { +# host = element(hcloud_server.worker.*.ipv6_address, count.index) +# user = "terraform" +# type = "ssh" +# private_key = var.terraform_private_ssh_key +# timeout = "20m" +# } +# +# provisioner "local-exec" { +# interpreter = ["bash", "-c"] +# command = "while [[ $(cat /tmp/current_joining_worker_node.txt) != \"${count.index}\" ]]; do echo \"${count.index} is waiting...\";sleep 5;done" +# } +# +# provisioner "file" { +# content = templatefile("${path.module}/templates/join-worker.sh", +# { +# cluster_token = random_id.cluster_token.hex +# main_node_ip = hcloud_server_network.manager_private[0].ip +# }) +# destination = "/usr/local/bin/join-worker.sh" +# } +# +# provisioner "remote-exec" { +# inline = [ +# "sh /usr/local/bin/join-worker.sh" +# ] +# } +# +# provisioner "local-exec" { +# interpreter = ["bash", "-c"] +# command = "echo \"${count.index + 1}\" > /tmp/current_joining_worker_node.txt" +# } +# +# depends_on = [null_resource.setup_tokens, null_resource.join_nodes] +#} \ No newline at end of file