2martens/terraform
1
0
Fork 0
mirror of https://github.com/2martens/terraform.git synced 2026-10-17 14:36:50 +02:00
Code Issues Activity

Added module to create k8s cluster with single or multiple nodes

Browse Source
This commit is contained in:
2martens
2023-11-03 08:48:16 +01:00
parent a8424ad1d4
commit 27133a507f
25 changed files with 687 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
terraform/.terraform/modules/modules.json
+1 -1
View File
@@ -1 +1 @@
{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"api_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"argocd_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"drone_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"gitea_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"helm-repo_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"keycloak_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"nextcloud_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"personal_website_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"statping_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"wahlrecht_domain","Source":"./modules/domain","Dir":"modules/domain"}]}
{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"api_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"argocd_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"drone_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"gitea_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"helm-repo_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"keycloak_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"nextcloud_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"personal_website_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"prometheus_k8s_test_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"statping_domain","Source":"./modules/domain","Dir":"modules/domain"},{"Key":"test_cluster","Source":"./modules/k8s-cluster","Dir":"modules/k8s-cluster"},{"Key":"wahlrecht_domain","Source":"./modules/domain","Dir":"modules/domain"}]}
terraform/domains.tf
+17 -9
View File
@@ -17,7 +17,7 @@ resource "inwx_domain_contact" "admin" {
}
resource "inwx_domain" "twomartens_de" {
name = "2martens.de"
name = local.domain
nameservers = [
"ns.inwx.de",
"ns2.inwx.de",
@@ -59,24 +59,24 @@ resource "inwx_domain" "twomartens_eu" {
// NS records
resource "inwx_nameserver_record" "twomartens_de_ns1" {
domain = "2martens.de"
name = "2martens.de"
domain = local.domain
name = local.domain
content = "ns.inwx.de"
type = "NS"
ttl = 86400
}
resource "inwx_nameserver_record" "twomartens_de_ns2" {
domain = "2martens.de"
name = "2martens.de"
domain = local.domain
name = local.domain
content = "ns2.inwx.de"
type = "NS"
ttl = 86400
}
resource "inwx_nameserver_record" "twomartens_de_ns3" {
domain = "2martens.de"
name = "2martens.de"
domain = local.domain
name = local.domain
content = "ns3.inwx.eu"
type = "NS"
ttl = 86400
@@ -109,11 +109,15 @@ resource "inwx_nameserver_record" "twomartens_eu_ns3" {
// SOAs
resource "inwx_nameserver_record" "twomartens_de_soa" {
domain = "2martens.de"
name = "2martens.de"
domain = local.domain
name = local.domain
content = "ns.inwx.de hostmaster.inwx.de 2023103008 10800 3600 604800 3600"
type = "SOA"
ttl = 86400
lifecycle {
ignore_changes = [content]
}
}
resource "inwx_nameserver_record" "twomartens_eu_soa" {
@@ -122,6 +126,10 @@ resource "inwx_nameserver_record" "twomartens_eu_soa" {
content = "ns.inwx.de hostmaster.inwx.de 2023103001 10800 3600 604800 3600"
type = "SOA"
ttl = 86400
lifecycle {
ignore_changes = [content]
}
}
// redirect 2martens.eu to 2martens.de
terraform/hetzner.tf
+11 -24
View File
@@ -1,25 +1,5 @@
// data sources
data "hcloud_location" "falkenstein" {
name = "fsn1"
}
data "hcloud_location" "nuremberg" {
name = "nbg1"
}
data "hcloud_datacenter" "falkenstein" {
name = "fsn1-dc14"
}
data "hcloud_datacenter" "nuremberg" {
name = "nbg1-dc3"
}
data "hcloud_ssh_key" "macos" {
name = "2martens@Jims-Air.fritz.box"
}
data "hcloud_placement_group" "default" {
name = "default"
}
@@ -198,9 +178,16 @@ resource "hcloud_server" "server_k8s_test" {
}
firewall_ids = [hcloud_firewall.basic-firewall.id, hcloud_firewall.k8s-firewall.id]
ssh_keys = [data.hcloud_ssh_key.macos.id]
user_data = templatefile("${path.module}/templates/cloud-init-k8s.tftpl", {
user_data = templatefile("${path.module}/modules/k8s-cluster/templates/cloud-init-k8s.tftpl", {
node_ip : "10.0.0.2",
cluster_token : "DkZaYY5JUAptUtq4HbXVb2x4HHD3HP7U"
admin_public_ssh_key : format("%s %s", data.hcloud_ssh_key.macos.public_key, data.hcloud_ssh_key.macos.name),
terraform_public_ssh_key : var.terraform_public_ssh_key,
microk8s_channel : "1.28-strict/stable",
admin_user : "2martensAdmin",
"microk8s_config" : templatefile("${path.module}/modules/k8s-cluster/templates/microk8s-config.tftpl", {
node_ip : "10.0.0.2",
api_server_domain : ""
})
})
shutdown_before_deletion = true
@@ -219,13 +206,13 @@ resource "hcloud_server" "server_k8s_test" {
module "drone_domain" {
source = "./modules/domain"
domain = "2martens.de"
domain = local.domain
subdomain = "ci"
ipv4 = "49.12.69.146"
ipv6 = "2a01:4f8:c012:1fef::1"
}
resource "inwx_nameserver_record" "twomartens_de_ci-ownercheck_txt" {
domain = "2martens.de"
domain = local.domain
name = "ownercheck.ci.2martens.de"
content = "e1eb600b"
type = "TXT"
terraform/ingress-apps.tf
+3 -3
View File
@@ -1,6 +1,6 @@
module "argocd_domain" {
source = "./modules/domain"
domain = "2martens.de"
domain = local.domain
subdomain = "argocd"
ipv4 = "49.13.77.70"
ipv6 = "2a01:4f8:c012:4d25::1"
@@ -8,7 +8,7 @@ module "argocd_domain" {
module "api_domain" {
source = "./modules/domain"
domain = "2martens.de"
domain = local.domain
subdomain = "api"
ipv4 = "49.13.77.70"
ipv6 = "2a01:4f8:c012:4d25::1"
@@ -16,7 +16,7 @@ module "api_domain" {
module "prometheus_k8s_test_domain" {
source = "./modules/domain"
domain = "2martens.de"
domain = local.domain
subdomain = "prometheus.k8s.test"
ipv4 = "49.13.77.70"
ipv6 = "2a01:4f8:c012:4d25::1"
terraform/main.tf
+53 -17
View File
@@ -1,19 +1,55 @@
terraform {
cloud {
organization = "2martens"
data "hcloud_location" "falkenstein" {
name = "fsn1"
}
workspaces {
name = "cloud-configuration"
}
}
required_providers {
inwx = {
source = "inwx/inwx"
version = "1.3.2"
}
hcloud = {
source = "hetznercloud/hcloud"
version = "1.44.1"
}
}
data "hcloud_location" "nuremberg" {
name = "nbg1"
}
data "hcloud_datacenter" "falkenstein" {
name = "fsn1-dc14"
}
data "hcloud_datacenter" "nuremberg" {
name = "nbg1-dc3"
}
data "hcloud_ssh_key" "macos" {
name = "2martens@Jims-Air.fritz.box"
}
locals {
domain = "2martens.de"
}
module "test_cluster" {
source = "./modules/k8s-cluster"
admin_ssh_key = data.hcloud_ssh_key.macos
admin_user = "2martensAdmin"
terraform_public_ssh_key = var.terraform_public_ssh_key
terraform_private_ssh_key = var.terraform_private_ssh_key
basic_firewall_id = hcloud_firewall.basic-firewall.id
k8s_firewall_id = hcloud_firewall.k8s-firewall.id
locations = [{
id = data.hcloud_location.falkenstein.id,
name = data.hcloud_location.falkenstein.name,
datacenter_name = data.hcloud_datacenter.falkenstein.name,
network_zone = data.hcloud_location.falkenstein.network_zone
}, {
id = data.hcloud_location.nuremberg.id,
name = data.hcloud_location.nuremberg.name,
datacenter_name = data.hcloud_datacenter.nuremberg.name,
network_zone = data.hcloud_location.nuremberg.network_zone
}]
domain = local.domain
network_id = hcloud_network.kubernetes-network.id
private_node_ips = ["10.0.0.4", "10.0.0.5", "10.0.0.6"]
server_subnet_id = hcloud_network_subnet.k8s-network-subnet.id
cluster_role = "test"
number_nodes = 1
server_type = "cax21"
image_name = "ubuntu-22.04"
create_loadbalancer = false
loadbalancer_ip = "10.0.0.3"
}
terraform/modules/domain/main.tf
-9
View File
@@ -1,12 +1,3 @@
terraform {
required_providers {
inwx = {
source = "inwx/inwx"
version = "1.3.2"
}
}
}
resource "inwx_nameserver_record" "a" {
domain = var.domain
name = var.subdomain != "" ? format("%s.%s", var.subdomain, var.domain) : var.domain
terraform/modules/domain/versions.tf
+8
View File
@@ -0,0 +1,8 @@
terraform {
required_providers {
inwx = {
source = "inwx/inwx"
version = "1.3.2"
}
}
}
terraform/modules/k8s-cluster/ha.tf
View File
terraform/modules/k8s-cluster/helm.tf
+18
View File
@@ -0,0 +1,18 @@
resource "null_resource" "install_argocd" {
connection {
host = hcloud_server_network.manager_private[0].ip
user = "root"
type = "ssh"
private_key = var.terraform_private_ssh_key
timeout = "2m"
}
provisioner "remote-exec" {
inline = [
"helm repo add 2martens https://repo.2martens.de/charts/",
"helm install 2martens/argocd",
]
}
depends_on = [null_resource.join_nodes, hcloud_server.manager]
}
terraform/modules/k8s-cluster/inwx.tf
+14
View File
@@ -0,0 +1,14 @@
resource "inwx_nameserver_record" "kube_api_server_a" {
domain = var.domain
name = local.kube_api_server_domain
content = var.create_loadbalancer ? hcloud_load_balancer.kubernetes[0].ipv4 : hcloud_primary_ip.ipv4_manager_address[0].ip_address
type = "A"
ttl = 3600
}
resource "inwx_nameserver_record" "kube_api_server_aaaa" {
domain = var.domain
name = local.kube_api_server_domain
content = var.create_loadbalancer ? hcloud_load_balancer.kubernetes[0].ipv6 : hcloud_primary_ip.ipv6_manager_address[0].ip_address
type = "AAAA"
ttl = 3600
}
terraform/modules/k8s-cluster/main.tf
+37
View File
@@ -0,0 +1,37 @@
data "hcloud_placement_group" "default" {
name = "default"
}
locals {
kube_api_server_domain = format("%s.%s.%s", "k8s", var.cluster_role, var.domain)
}
resource "hcloud_load_balancer" "kubernetes" {
count = var.create_loadbalancer ? 1 : 0
load_balancer_type = var.loadbalancer_type
name = format("%s_%s", "k8s", var.cluster_role)
location = var.locations[count.index].name
network_zone = var.locations[count.index].network_zone
algorithm {
type = "round_robin"
}
}
resource "hcloud_load_balancer_network" "private" {
count = var.create_loadbalancer ? 1 : 0
load_balancer_id = hcloud_load_balancer.kubernetes[count.index].id
subnet_id = var.server_subnet_id
ip = var.loadbalancer_ip
}
resource "hcloud_load_balancer_target" "nodes" {
count = var.create_loadbalancer ? var.number_nodes : 0
type = "server"
load_balancer_id = hcloud_load_balancer.kubernetes[0].id
server_id = hcloud_server.manager[count.index].id
use_private_ip = true
}
terraform/modules/k8s-cluster/manager-node.tf
+160
View File
@@ -0,0 +1,160 @@
resource "hcloud_primary_ip" "ipv4_manager_address" {
count = var.create_loadbalancer ? 0 : var.number_nodes
name = format("%s_%s_%s_%d", "k8s", var.cluster_role, "ipv4", count.index)
datacenter = element(var.locations.*.datacenter_name, count.index)
type = "ipv4"
assignee_type = "server"
auto_delete = false
}
resource "hcloud_primary_ip" "ipv6_manager_address" {
count = var.create_loadbalancer ? 0 : var.number_nodes
name = format("%s_%s_%s_%d", "k8s", var.cluster_role, "ipv6", count.index)
datacenter = element(var.locations.*.datacenter_name, count.index)
type = "ipv6"
assignee_type = "server"
auto_delete = false
}
resource "hcloud_server_network" "manager_private" {
count = var.number_nodes
server_id = hcloud_server.manager[count.index].id
subnet_id = var.server_subnet_id
ip = var.private_node_ips[count.index]
}
resource "hcloud_server" "manager" {
count = var.number_nodes
name = format("%s_%s_%s_%d", "k8s", var.cluster_role, "node", count.index)
image = var.image_name
allow_deprecated_images = false
server_type = var.server_type
location = element(var.locations.*.name, count.index)
public_net {
ipv4_enabled = var.create_loadbalancer ? false : true
ipv4 = var.create_loadbalancer ? null : hcloud_primary_ip.ipv4_manager_address[count.index].id
ipv6_enabled = var.create_loadbalancer ? false : true
ipv6 = var.create_loadbalancer ? null : hcloud_primary_ip.ipv6_manager_address[count.index].id
}
ignore_remote_firewall_ids = false
keep_disk = false
placement_group_id = data.hcloud_placement_group.default.id
labels = {
"kubernetes" : "yes",
"manager" : "yes"
}
firewall_ids = [var.basic_firewall_id, var.k8s_firewall_id]
ssh_keys = [var.admin_ssh_key.id]
user_data = templatefile("${path.module}/templates/cloud-init-k8s.tftpl", {
node_ip : var.private_node_ips[count.index],
admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name),
terraform_public_ssh_key : var.terraform_public_ssh_key,
microk8s_channel : var.microk8s_channel,
admin_user : var.admin_user,
microk8s_config : templatefile("${path.module}/templates/microk8s-config.tftpl", {
node_ip : var.private_node_ips[count.index],
api_server_domain : local.kube_api_server_domain
})
})
shutdown_before_deletion = true
network {
network_id = var.network_id
ip = var.private_node_ips[count.index]
alias_ips = []
}
lifecycle {
ignore_changes = [ssh_keys, user_data]
}
}
resource "random_id" "cluster_token" {
byte_length = 16
}
resource "null_resource" "setup_tokens" {
count = var.number_nodes > 1 ? 1 : 0
triggers = {
rerun = random_id.cluster_token.hex
}
connection {
host = hcloud_server_network.manager_private[0].ip
user = "root"
type = "ssh"
private_key = var.terraform_private_ssh_key
timeout = "2m"
}
provisioner "local-exec" {
interpreter = ["bash", "-c"]
command = <<EOT
echo "1" > /tmp/current_joining_node.txt
echo "0" > /tmp/current_joining_worker_node.txt
EOT
}
provisioner "file" {
content = templatefile("${path.module}/templates/add-node.sh",
{
cluster_token = random_id.cluster_token.hex
cluster_token_ttl_seconds = var.cluster_token_ttl_seconds
})
destination = "/usr/local/bin/add-node.sh"
}
provisioner "remote-exec" {
inline = [
"sh /usr/local/bin/add-node.sh",
]
}
}
resource "null_resource" "join_nodes" {
count = var.number_nodes - 1 < 1 ? 0 : var.number_nodes - 1
triggers = {
rerun = random_id.cluster_token.hex
}
connection {
host = element(hcloud_server_network.manager_private.*.ip, count.index + 1)
user = "root"
type = "ssh"
private_key = var.terraform_private_ssh_key
timeout = "20m"
}
provisioner "local-exec" {
interpreter = ["bash", "-c"]
command = "while [[ $(cat /tmp/current_joining_node.txt) != \"${count.index + 1}\" ]]; do echo \"${count.index + 1} is waiting...\";sleep 5;done"
}
provisioner "file" {
content = templatefile("${path.module}/templates/join.sh",
{
cluster_token = random_id.cluster_token.hex
main_node_ip = hcloud_server_network.manager_private[0].ip
})
destination = "/usr/local/bin/join.sh"
}
provisioner "remote-exec" {
inline = [
"sh /usr/local/bin/join.sh"
]
}
provisioner "local-exec" {
interpreter = ["bash", "-c"]
command = "echo \"${count.index + 2}\" > /tmp/current_joining_node.txt"
}
depends_on = [null_resource.setup_tokens]
}
terraform/modules/k8s-cluster/outputs.tf
View File
terraform/modules/k8s-cluster/templates/add-node.sh
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
until microk8s.status --wait-ready;
do sleep 5; echo "waiting for worker status..";
done
sleep 10
microk8s add-node --token ${cluster_token} --token-ttl ${cluster_token_ttl_seconds}
terraform/templates/cloud-init-k8s.tftpl → terraform/modules/k8s-cluster/templates/cloud-init-k8s.tftpl
+18 -8
View File
@@ -1,16 +1,24 @@
#cloud-config
users:
- name: 2martensAdmin
- name: ${admin_user}
groups: users, admin
sudo: ALL=(ALL) NOPASSWD:ALL
shell: /bin/bash
ssh_authorized_keys:
- "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBjYkz73HbsFtRAOfS5QB9xDbOajU9C4gpDp+OjQkLir 2martens@Jims-Air.fritz.box"
- ${admin_public_ssh_key}
- name: terraform
groups: users, admin
sudo: ALL=(ALL) NOPASSWD:ALL
ssh_authorized_keys:
- ${terraform_public_ssh_key}
packages:
- fail2ban
- ufw
package_update: true
package_upgrade: true
snap:
commands:
- snap install kubectl --classic
runcmd:
- printf "[sshd]\nenabled = true\nbanaction = iptables-multiport" > /etc/fail2ban/jail.local
- systemctl enable fail2ban
@@ -30,12 +38,14 @@ runcmd:
- sed -i -e '/^\(#\|\)AllowTcpForwarding/s/^.*$/AllowTcpForwarding no/' /etc/ssh/sshd_config
- sed -i -e '/^\(#\|\)AllowAgentForwarding/s/^.*$/AllowAgentForwarding no/' /etc/ssh/sshd_config
- sed -i -e '/^\(#\|\)AuthorizedKeysFile/s/^.*$/AuthorizedKeysFile .ssh\/authorized_keys/' /etc/ssh/sshd_config
- sed -i '$a AllowUsers 2martensAdmin' /etc/ssh/sshd_config
- sed -i '$a AllowUsers ${admin_user} terraform' /etc/ssh/sshd_config
- curl https://packages.hetzner.com/hcloud/deb/hc-utils_0.0.4-1_all.deb -o /tmp/hc-utils_0.0.4-1_all.deb -s
- apt install /tmp/hc-utils_0.0.4-1_all.deb
- mkdir -p /var/snap/microk8s/common/
- curl https://repo.2martens.de/files/microk8s-config.yaml -o /var/snap/microk8s/common/.microk8s.yaml
- sed -i -e '/\$NODE_IP/s//${node_ip}/' /var/snap/microk8s/common/.microk8s.yaml
- sed -i -e '/\$CLUSTER_TOKEN/s//${cluster_token}/' /var/snap/microk8s/common/.microk8s.yaml
- snap install microk8s --channel 1.28-strict/stable
- reboot
- echo ${microk8s_config} > /var/snap/microk8s/common/.microk8s.yaml
- snap install microk8s --channel ${microk8s_channel}
- mkdir -p /home/terraform/.kube
- microk8s kubectl config view --raw > /home/terraform/.kube/config
- chown -R terraform:terraform /home/terraform/.kube
- chmod go-r /home/terraform/.kube/config
- microk8s.status --wait-ready
terraform/modules/k8s-cluster/templates/join-worker.sh
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
until microk8s.status --wait-ready;
do sleep 3; echo "waiting for worker status..";
done
if microk8s status | grep "datastore master nodes: 127.0.0.1:19001" > /dev/null 2>&1; then
microk8s join ${main_node_ip}:25000/${cluster_token} --worker
else
echo "Join process already done. Nothing to do"
fi
terraform/modules/k8s-cluster/templates/join.sh
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
until microk8s.status --wait-ready;
do sleep 3; echo "waiting for worker status..";
done
if microk8s status | grep "datastore master nodes: 127.0.0.1:19001" > /dev/null 2>&1; then
microk8s join "${main_node_ip}:25000/${cluster_token}"
else
echo "Join process already done. Nothing to do"
fi
terraform/modules/k8s-cluster/templates/microk8s-config.tftpl
+20
View File
@@ -0,0 +1,20 @@
---
version: 0.1.0
extraKubeAPIServerArgs:
--advertise-address=${node_ip}
--authorization-mode=RBAC,Node
extraKubeletArgs:
--node-ip=${node_ip}
--cluster-dns=10.152.183.10
--cluster-domain=cluster.local
extraSANs:
- ${api_server_domain}
addons:
- name: dns
- name: cis-hardening
- name: community
- name: helm3
- name: ingress
- name: ha-cluster
- name: rbac
- name: cert-manager
terraform/modules/k8s-cluster/variables.tf
+130
View File
@@ -0,0 +1,130 @@
variable "admin_ssh_key" {
description = "SSH key to provide to each server for the admin"
type = object({
id : number,
name : string
public_key : string
})
}
variable "admin_user" {
description = "Linux username for the admin user on each server"
type = string
}
variable "terraform_private_ssh_key" {
description = "Private SSH key to be used by Terraform to connect with remote machines"
type = string
sensitive = true
}
variable "terraform_public_ssh_key" {
description = "Public SSH key for the private key used by Terraform to connect with remote machines"
type = string
}
variable "locations" {
description = "Locations in which the resources should be created. Singular resources will always be created in the first. Must be all in the same network zone."
type = list(object({
id : number,
name : string,
datacenter_name : string,
network_zone : string
}))
}
variable "domain" {
description = "The registered domain that should be used as the base for all subdomains."
type = string
}
variable "network_id" {
description = "ID of the private network used to connect servers with each other and the loadbalancer"
type = number
}
variable "server_subnet_id" {
description = "ID of the subnet of the private network that is supposed to house the servers and loadbalancer"
type = string
}
variable "private_node_ips" {
description = "One IP in the private network subnet for each manager node created."
type = list(string)
}
variable "private_worker_node_ips" {
description = "One IP in the private network subnet for each worker node created."
type = list(string)
default = []
}
variable "basic_firewall_id" {
description = "ID of the basic firewall for each server"
type = number
}
variable "k8s_firewall_id" {
description = "ID of the K8s specific firewall"
type = number
}
variable "cluster_role" {
description = "The role the cluster serves. For example test, qs, prod. Will be used for names and subdomains and must be unique within the Hetzner project."
type = string
default = "test"
}
variable "number_nodes" {
description = "Number of manager nodes in the cluster."
type = number
default = 1
}
variable "number_worker_nodes" {
description = "Number of worker nodes in the cluster."
type = number
default = 0
}
variable "image_name" {
description = "Name of the image to use for the servers."
type = string
default = "ubuntu-22.04"
}
variable "server_type" {
description = "Hetzner server type like cax21 (default)"
type = string
default = "cax21"
}
variable "create_loadbalancer" {
description = "Specifies if a loadbalancer is created as gateway in front of the k8s servers. If false, the first server acts as gateway."
type = bool
default = false
}
variable "loadbalancer_type" {
description = "Type of the loadbalancer used. By default lb11 is used."
type = string
default = "lb11"
}
variable "loadbalancer_ip" {
description = "The private IP of the loadbalancer. Variable contains valid IP but must be overriden if a loadbalancer is used."
type = string
default = "0.0.0.0"
}
variable "cluster_token_ttl_seconds" {
description = "Number of seconds the join token is valid"
type = number
default = 3600
}
variable "microk8s_channel" {
description = "The snap channel used for the microk8s installation"
type = string
default = "1.28-strict/stable"
}
terraform/modules/k8s-cluster/versions.tf
+17
View File
@@ -0,0 +1,17 @@
terraform {
required_providers {
helm = {
source = "hashicorp/helm"
version = "2.11.0"
}
inwx = {
source = "inwx/inwx"
version = "1.3.2"
}
hcloud = {
source = "hetznercloud/hcloud"
version = "1.44.1"
}
}
}
terraform/modules/k8s-cluster/worker-node.tf
+94
View File
@@ -0,0 +1,94 @@
resource "hcloud_server_network" "worker_private" {
count = var.number_worker_nodes
server_id = hcloud_server.worker[count.index].id
subnet_id = var.server_subnet_id
ip = var.private_worker_node_ips[count.index]
}
resource "hcloud_server" "worker" {
count = var.number_worker_nodes
name = format("%s_%s_%s_%d", "k8s", var.cluster_role, "node", count.index)
image = var.image_name
allow_deprecated_images = false
server_type = var.server_type
location = element(var.locations.*.name, count.index)
public_net {
ipv4_enabled = false
ipv6_enabled = false
}
ignore_remote_firewall_ids = false
keep_disk = false
placement_group_id = data.hcloud_placement_group.default.id
labels = {
"kubernetes" : "yes",
"worker" : "yes"
}
firewall_ids = [var.basic_firewall_id, var.k8s_firewall_id]
ssh_keys = [var.admin_ssh_key.id]
user_data = templatefile("${path.module}/templates/cloud-init-k8s.tftpl", {
node_ip : var.private_worker_node_ips[count.index],
admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name),
terraform_public_ssh_key : var.terraform_public_ssh_key,
microk8s_channel : var.microk8s_channel,
admin_user : var.admin_user,
microk8s_config : templatefile("${path.module}/templates/microk8s-config.tftpl", {
node_ip : var.private_worker_node_ips[count.index],
api_server_domain : local.kube_api_server_domain
})
})
shutdown_before_deletion = true
network {
network_id = var.network_id
ip = var.private_worker_node_ips[count.index]
alias_ips = []
}
lifecycle {
ignore_changes = [ssh_keys, user_data]
}
}
resource "null_resource" "join_workers" {
count = var.number_worker_nodes
triggers = {
rerun = random_id.cluster_token.hex
}
connection {
host = element(hcloud_server_network.worker_private.*.ip, count.index)
user = "root"
type = "ssh"
private_key = var.terraform_private_ssh_key
timeout = "20m"
}
provisioner "local-exec" {
interpreter = ["bash", "-c"]
command = "while [[ $(cat /tmp/current_joining_worker_node.txt) != \"${count.index}\" ]]; do echo \"${count.index} is waiting...\";sleep 5;done"
}
provisioner "file" {
content = templatefile("${path.module}/templates/join-worker.sh",
{
cluster_token = random_id.cluster_token.hex
main_node_ip = hcloud_server_network.manager_private[0].ip
})
destination = "/usr/local/bin/join-worker.sh"
}
provisioner "remote-exec" {
inline = [
"sh /usr/local/bin/join-worker.sh"
]
}
provisioner "local-exec" {
interpreter = ["bash", "-c"]
command = "echo \"${count.index + 1}\" > /tmp/current_joining_worker_node.txt"
}
depends_on = [null_resource.setup_tokens, null_resource.join_nodes]
}
terraform/nameserver_records.tf
+14 -14
View File
@@ -1,6 +1,6 @@
// A records
resource "inwx_nameserver_record" "twomartens_de_allris_a" {
domain = "2martens.de"
domain = local.domain
name = "allris.2martens.de"
content = "185.26.156.105"
type = "A"
@@ -8,7 +8,7 @@ resource "inwx_nameserver_record" "twomartens_de_allris_a" {
}
resource "inwx_nameserver_record" "twomartens_de_autoconfig_a" {
domain = "2martens.de"
domain = local.domain
name = "autoconfig.2martens.de"
content = "185.26.156.22"
type = "A"
@@ -16,7 +16,7 @@ resource "inwx_nameserver_record" "twomartens_de_autoconfig_a" {
}
resource "inwx_nameserver_record" "twomartens_de_autodiscover_a" {
domain = "2martens.de"
domain = local.domain
name = "autodiscover.2martens.de"
content = "185.26.156.22"
type = "A"
@@ -24,7 +24,7 @@ resource "inwx_nameserver_record" "twomartens_de_autodiscover_a" {
}
resource "inwx_nameserver_record" "twomartens_de_jim_a" {
domain = "2martens.de"
domain = local.domain
name = "jim.2martens.de"
content = "185.26.156.65"
type = "A"
@@ -32,7 +32,7 @@ resource "inwx_nameserver_record" "twomartens_de_jim_a" {
}
resource "inwx_nameserver_record" "twomartens_de_pad_a" {
domain = "2martens.de"
domain = local.domain
name = "pad.2martens.de"
content = "185.26.156.132"
type = "A"
@@ -40,7 +40,7 @@ resource "inwx_nameserver_record" "twomartens_de_pad_a" {
}
resource "inwx_nameserver_record" "twomartens_de_short_a" {
domain = "2martens.de"
domain = local.domain
name = "short.2martens.de"
content = "95.143.172.56"
type = "A"
@@ -50,7 +50,7 @@ resource "inwx_nameserver_record" "twomartens_de_short_a" {
// AAAA records
resource "inwx_nameserver_record" "twomartens_de_allris_aaaa" {
domain = "2martens.de"
domain = local.domain
name = "allris.2martens.de"
content = "2a00:d0c0:200:0:b9:1a:9c:61"
type = "AAAA"
@@ -58,7 +58,7 @@ resource "inwx_nameserver_record" "twomartens_de_allris_aaaa" {
}
resource "inwx_nameserver_record" "twomartens_de_autoconfig_aaaa" {
domain = "2martens.de"
domain = local.domain
name = "autoconfig.2martens.de"
content = "2a00:d0c0:200:0:b9:1a:9c16:5d"
type = "AAAA"
@@ -66,7 +66,7 @@ resource "inwx_nameserver_record" "twomartens_de_autoconfig_aaaa" {
}
resource "inwx_nameserver_record" "twomartens_de_autodiscover_aaaa" {
domain = "2martens.de"
domain = local.domain
name = "autodiscover.2martens.de"
content = "2a00:d0c0:200:0:b9:1a:9c16:5d"
type = "AAAA"
@@ -74,7 +74,7 @@ resource "inwx_nameserver_record" "twomartens_de_autodiscover_aaaa" {
}
resource "inwx_nameserver_record" "twomartens_de_jim_aaaa" {
domain = "2martens.de"
domain = local.domain
name = "jim.2martens.de"
content = "2a00:d0c0:200:0:b9:1a:9c:40"
type = "AAAA"
@@ -82,7 +82,7 @@ resource "inwx_nameserver_record" "twomartens_de_jim_aaaa" {
}
resource "inwx_nameserver_record" "twomartens_de_pad_aaaa" {
domain = "2martens.de"
domain = local.domain
name = "pad.2martens.de"
content = "2a00:d0c0:200:0:c0bf:e0ff:fec1:af72"
type = "AAAA"
@@ -90,7 +90,7 @@ resource "inwx_nameserver_record" "twomartens_de_pad_aaaa" {
}
resource "inwx_nameserver_record" "twomartens_de_short_aaaa" {
domain = "2martens.de"
domain = local.domain
name = "short.2martens.de"
content = "2001:1a50:11:0:3854:d4ff:fe62:e0d8"
type = "AAAA"
@@ -100,7 +100,7 @@ resource "inwx_nameserver_record" "twomartens_de_short_aaaa" {
// CNAME records
resource "inwx_nameserver_record" "twomartens_de_cdn_cname" {
domain = "2martens.de"
domain = local.domain
name = "cdn.2martens.de"
content = "d1fvxyvcoii67h.cloudfront.net"
type = "CNAME"
@@ -108,7 +108,7 @@ resource "inwx_nameserver_record" "twomartens_de_cdn_cname" {
}
resource "inwx_nameserver_record" "twomartens_de_aws-validations-cdn_cname" {
domain = "2martens.de"
domain = local.domain
name = "_4741e2d9c8a605950eebfa048029e4ef.cdn.2martens.de"
content = "_cf71e85bb40cc12717f2718e4590ad23.acm-validations.aws"
type = "CNAME"
terraform/uberspace.tf
+10 -10
View File
@@ -1,7 +1,7 @@
// Identity management
module "keycloak_domain" {
source = "./modules/domain"
domain = "2martens.de"
domain = local.domain
subdomain = "id"
ipv4 = "95.143.172.184"
ipv6 = "2001:1a50:11:0:98b3:92ff:fe3e:bba1"
@@ -14,7 +14,7 @@ module "keycloak_domain" {
// Angular frontends
module "wahlrecht_domain" {
source = "./modules/domain"
domain = "2martens.de"
domain = local.domain
subdomain = "wahlrecht"
ipv4 = "185.26.156.43"
ipv6 = "2a00:d0c0:200:0:470:9fff:fe83:16bc"
@@ -23,7 +23,7 @@ module "wahlrecht_domain" {
// CI/CD
module "gitea_domain" {
source = "./modules/domain"
domain = "2martens.de"
domain = local.domain
subdomain = "git"
ipv4 = "185.26.156.91"
ipv6 = "2a00:d0c0:200:0:b9:1a:9c:5a"
@@ -31,7 +31,7 @@ module "gitea_domain" {
module "helm-repo_domain" {
source = "./modules/domain"
domain = "2martens.de"
domain = local.domain
subdomain = "repo"
ipv4 = "185.26.156.49"
ipv6 = "2a00:d0c0:200:0:2ca6:bff:fe78:832f"
@@ -40,7 +40,7 @@ module "helm-repo_domain" {
// Monitoring
module "statping_domain" {
source = "./modules/domain"
domain = "2martens.de"
domain = local.domain
subdomain = "status"
ipv4 = "185.26.156.33"
ipv6 = "2a00:d0c0:200:0:f8e2:6fff:fec2:7a92"
@@ -49,7 +49,7 @@ module "statping_domain" {
// Personal Website
module "personal_website_domain" {
source = "./modules/domain"
domain = "2martens.de"
domain = local.domain
subdomain = ""
ipv4 = "185.26.156.65"
ipv6 = "2a00:d0c0:200:0:b9:1a:9c:40"
@@ -59,8 +59,8 @@ module "personal_website_domain" {
mxSpf = "v=spf1 include:spf.uberspace.de"
}
resource "inwx_nameserver_record" "twomartens_de_google-verification_txt" {
domain = "2martens.de"
name = "2martens.de"
domain = local.domain
name = local.domain
content = "google-site-verification=nUFiHQFxBpBMdX96ELH3TcfyIfXf2ZlwMFYtXjVq5lo"
type = "TXT"
ttl = 3600
@@ -69,7 +69,7 @@ resource "inwx_nameserver_record" "twomartens_de_google-verification_txt" {
// Nextcloud
module "nextcloud_domain" {
source = "./modules/domain"
domain = "2martens.de"
domain = local.domain
subdomain = "cloud"
ipv4 = "185.26.156.194"
ipv6 = "2a00:d0c0:200:0:b9:1a:9c:95"
@@ -81,7 +81,7 @@ module "nextcloud_domain" {
// domain key
resource "inwx_nameserver_record" "twomartens_de_uberspace-domainkey_txt" {
domain = "2martens.de"
domain = local.domain
name = "uberspace._domainkey.id.2martens.de"
content = "v=DKIM1;t=s;n=core;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqs3bFL8W77z7VJHCFbsJ3o68Y1uBbVsbIS1y35CMfRv6wp7si7aIG2ZeKqzdh2dTMsvtvSMaYVq1gT/EtRmTbU/BYC21ff8sqcVqn/ll5nNsk5jKWXTYAlTQp4LBQN7icw94ZNGr5/SDYcnv2nsBYFf2GUviObWXGHX4RaBFNj9NVUNWNin/HicvW+LsbfYq37QtlhjmUn9K96VCwKcTV1mx+Ek0osErYefcOVNawqWIlVRQDLkHZhk1StLsOpRqV+qEjhzTk4n4ZiNtLJG1D9CpHl24d5DKsQDyVFdEfqHimFTSNgiitlkuXg+i+NMiRA9G3gNJtvw8uvfN8f+stQIDAQAB"
type = "TXT"
terraform/variables.tf
+11
View File
@@ -36,4 +36,15 @@ variable "admin_email" {
variable "hcloud_token" {
type = string
sensitive = true
}
variable "terraform_private_ssh_key" {
description = "Private SSH key to be used by Terraform to connect with remote machines"
type = string
sensitive = true
}
variable "terraform_public_ssh_key" {
description = "Public SSH key for the private key used by Terraform to connect with remote machines"
type = string
}
terraform/versions.tf
+19
View File
@@ -0,0 +1,19 @@
terraform {
cloud {
organization = "2martens"
workspaces {
name = "cloud-configuration"
}
}
required_providers {
inwx = {
source = "inwx/inwx"
version = "1.3.2"
}
hcloud = {
source = "hetznercloud/hcloud"
version = "1.44.1"
}
}
}