Finish docker network module

terraform/modules/docker-network/main.tf

@@ -5,19 +5,3 @@ data "hcloud_placement_group" "default" {
 locals {
   docker_network_domain = format("%s.%s.%s", "docker", var.stage_name, var.domain)
 }
-
-# TODO: use Hetzner DNS
-# resource "inwx_nameserver_record" "kube_api_server_a" {
-#   domain  = var.domain
-#   name    = local.docker_network_domain
-#   content = hcloud_server.nginx.ipv4_address
-#   type    = "A"
-#   ttl     = 3600
-# }
-# resource "inwx_nameserver_record" "kube_api_server_aaaa" {
-#   domain  = var.domain
-#   name    = local.docker_network_domain
-#   content = hcloud_server.nginx.ipv6_address
-#   type    = "AAAA"
-#   ttl     = 3600
-# }

terraform/modules/docker-network/nginx-vm.tf

@@ -1,96 +0,0 @@
-# TODO: Use Hetzner DNS
-# resource "inwx_nameserver_record" "manager_aaaa" {
-#   domain  = var.domain
-#   name = format("%s.%s", "nginx", local.docker_network_domain)
-#   content = hcloud_server.nginx.ipv6_address
-#   type    = "AAAA"
-#   ttl     = 3600
-# }
-
-resource "hcloud_primary_ip" "ipv4_nginx_address" {
-  name = format("%s_%s_%s", "docker", var.stage_name, "ipv4")
-  datacenter = element(var.locations.*.datacenter_name, 0)
-  type          = "ipv4"
-  assignee_type = "server"
-  auto_delete   = false
-}
-
-resource "hcloud_rdns" "ipv4_nginx" {
-  primary_ip_id = hcloud_primary_ip.ipv4_nginx_address.id
-  ip_address    = hcloud_primary_ip.ipv4_nginx_address.ip_address
-  # TODO: use Hetzner DNS
-  # dns_ptr       = inwx_nameserver_record.manager_aaaa.name
-}
-
-resource "hcloud_primary_ip" "ipv6_nginx_address" {
-  name = format("%s_%s_%s", "k8s", var.stage_name, "ipv6")
-  datacenter = element(var.locations.*.datacenter_name, 0)
-  type          = "ipv6"
-  assignee_type = "server"
-  auto_delete   = false
-}
-
-resource "hcloud_rdns" "ipv6_nginx" {
-  primary_ip_id = hcloud_primary_ip.ipv6_nginx_address.id
-  ip_address    = hcloud_server.nginx.ipv6_address
-  # TODO: use Hetzner DNS
-  # dns_ptr       = inwx_nameserver_record.manager_aaaa.name
-}
-
-resource "hcloud_server_network" "nginx_private" {
-  server_id = hcloud_server.nginx.id
-  subnet_id = var.server_subnet_id
-  ip        = var.private_nginx_vm_ip
-}
-
-resource "hcloud_server" "nginx" {
-  name = format("%s-%s-%s", "docker", var.stage_name, "nginx")
-  image                   = var.image_name
-  allow_deprecated_images = false
-  server_type             = var.server_type
-  location = element(var.locations.*.name, 0)
-
-  public_net {
-    ipv4_enabled = true
-    ipv4         = hcloud_primary_ip.ipv4_nginx_address.id
-    ipv6_enabled = true
-    ipv6         = hcloud_primary_ip.ipv6_nginx_address.id
-  }
-
-  ignore_remote_firewall_ids = false
-  keep_disk                  = false
-  placement_group_id         = data.hcloud_placement_group.default.id
-  firewall_ids = [var.basic_firewall_id, var.docker_firewall_id]
-  ssh_keys = [var.admin_ssh_key.id]
-  shutdown_before_deletion   = true
-
-  labels = {
-    "docker" : "yes",
-    "nginx" : "yes"
-  }
-
-  user_data = templatefile("${path.module}/templates/cloud-init-docker-with-nginx.yaml.tftpl", {
-    admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name)
-    admin_user : var.admin_user
-    terraform_public_ssh_key : var.terraform_public_ssh_key
-    packages_setup : base64encode(file("${path.module}/templates/scripts/install-packages.sh"))
-    firewall_setup : base64encode(templatefile("${path.module}/templates/scripts/firewall-setup.sh", {
-      node_ip : var.private_nginx_vm_ip
-    }))
-    ssh_setup : base64encode(templatefile("${path.module}/templates/scripts/ssh-setup.sh", {
-      admin_user : var.admin_user
-    }))
-    sysctl_setup : base64encode(file("${path.module}/templates/scripts/sysctl-setup.sh"))
-    nginx_setup : base64encode(file("${path.module}/templates/scripts/nginx-setup.sh"))
-  })
-
-  network {
-    network_id = var.network_id
-    ip         = var.private_nginx_vm_ip
-    alias_ips = []
-  }
-
-  lifecycle {
-    ignore_changes = [ssh_keys, user_data]
-  }
-}

terraform/modules/docker-network/outputs.tf

@@ -1,7 +1,11 @@
-output "nginx_server_ipv4_address" {
+output "node_details" {
-  value = hcloud_server.nginx.ipv4_address
+  value = {
-}
+    for idx, instance in hcloud_server.node :
-
+    "node-${idx}" => {
-output "nginx_server_ipv6_address" {
+      id      = instance.id
-  value = hcloud_server.nginx.ipv6_address
+      name    = instance.name
+      ipv4    = instance.ipv4_address
+      ipv6    = instance.ipv6_address
+    }
+  }
 }

terraform/modules/docker-network/swarm-node.tf

@@ -0,0 +1,101 @@
+resource "hcloud_primary_ip" "ipv4_node_address" {
+  count = var.number_nodes
+
+  name = format("%s_%s_%s_%d", "docker", var.stage_name, "ipv4", count.index)
+  datacenter    = var.locations[count.index].datacenter_name
+  type          = "ipv4"
+  assignee_type = "server"
+  auto_delete   = false
+}
+
+resource "hcloud_rdns" "ipv4_node" {
+  count = var.number_nodes
+
+  primary_ip_id = hcloud_primary_ip.ipv4_node_address[count.index].id
+  ip_address    = hcloud_primary_ip.ipv4_node_address[count.index].ip_address
+  dns_ptr = format("%s-%d.%s", "node", count.index, local.docker_network_domain)
+}
+
+resource "hcloud_primary_ip" "ipv6_node_address" {
+  count = var.number_nodes
+
+  name = format("%s_%s_%s_%d", "docker", var.stage_name, "ipv6", count.index)
+  datacenter    = var.locations[count.index].datacenter_name
+  type          = "ipv6"
+  assignee_type = "server"
+  auto_delete   = false
+}
+
+resource "hcloud_rdns" "ipv6_node" {
+  count = var.number_nodes
+
+  primary_ip_id = hcloud_primary_ip.ipv6_node_address[count.index].id
+  ip_address    = hcloud_server.node[count.index].ipv6_address
+  dns_ptr = format("%s-%d.%s", "node", count.index, local.docker_network_domain)
+}
+
+resource "hcloud_server_network" "nginx_private" {
+  count = var.number_nodes
+
+  server_id = hcloud_server.node[count.index].id
+  subnet_id = var.server_subnet_id
+  ip        = var.private_node_ips[count.index]
+}
+
+resource "hcloud_server" "node" {
+  count = var.number_nodes
+
+  name = format("%s-%s-%s-%d", "docker", var.stage_name, "node", count.index)
+  image                   = var.image_name
+  allow_deprecated_images = false
+  server_type             = var.server_type
+  location                = var.locations[count.index].name
+
+  public_net {
+    ipv4_enabled = true
+    ipv4         = hcloud_primary_ip.ipv4_node_address[count.index].id
+    ipv6_enabled = true
+    ipv6         = hcloud_primary_ip.ipv6_node_address[count.index].id
+  }
+
+  ignore_remote_firewall_ids = false
+  keep_disk                  = false
+  placement_group_id         = data.hcloud_placement_group.default.id
+  firewall_ids = [var.basic_firewall_id, var.docker_firewall_id]
+  ssh_keys = [var.admin_ssh_key.id]
+  shutdown_before_deletion   = true
+
+  labels = {
+    "docker" : "yes",
+    "swarm" : "yes"
+  }
+
+  user_data = templatefile("cloud-init-docker.yaml.tftpl", {
+    admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name)
+    admin_user : var.admin_user
+    terraform_public_ssh_key : var.terraform_public_ssh_key
+    main_node : count.index == 0
+    packages_setup : base64encode(file("${path.module}/templates/scripts/install-packages.sh"))
+    firewall_setup : base64encode(templatefile("${path.module}/templates/scripts/firewall-setup.sh.tftpl", {
+      node_ip : var.private_node_ips[count.index]
+    }))
+    ssh_setup : base64encode(templatefile("${path.module}/templates/scripts/ssh-setup.sh.tftpl", {
+      admin_user : var.admin_user
+    }))
+    sysctl_setup : base64encode(file("${path.module}/templates/scripts/sysctl-setup.sh"))
+    swarm_setup : base64encode(templatefile("${path.module}/templates/scripts/swarm-setup.sh.tftpl", {
+      main_node : count.index == 0
+      main_node_private_ip : var.private_node_ips[0]
+    }))
+  })
+
+  network {
+    network_id = var.network_id
+    ip         = var.private_node_ips[count.index]
+    alias_ips = []
+  }
+
+  lifecycle {
+    ignore_changes = [ssh_keys, user_data]
+  }
+}

terraform/modules/docker-network/templates/cloud-init-docker-no-nginx.yaml.tftpl

@@ -1,43 +0,0 @@
-#cloud-config
-users:
-  - name: ${admin_user}
-    groups: users, admin, microk8s
-    sudo: ALL=(ALL) NOPASSWD:ALL
-    shell: /bin/bash
-    ssh_authorized_keys:
-      - ${admin_public_ssh_key}
-  - name: terraform
-    groups: users, admin
-    sudo: ALL=(ALL) NOPASSWD:ALL
-    ssh_authorized_keys:
-      - ${terraform_public_ssh_key}
-package_update: true
-package_upgrade: true
-package_reboot_if_required: true
-write_files:
-  - owner: root:root
-    content: "${packages_setup}"
-    path: /run/tmpfiles.d/install-packages.sh
-    encoding: base64
-    defer: true
-  - owner: root:root
-    content: "${firewall_setup}"
-    path: /run/tmpfiles.d/firewall-setup.sh
-    encoding: base64
-    defer: true
-  - owner: root:root
-    content: "${ssh_setup}"
-    path: /run/tmpfiles.d/ssh-setup.sh
-    encoding: base64
-    defer: true
-  - owner: root:root
-    content: "${sysctl_setup}"
-    path: /run/tmpfiles.d/sysctl-setup.sh
-    encoding: base64
-    defer: true
-runcmd:
-  - install /run/tmpfiles.d/*.sh /usr/local/bin
-  - /usr/local/bin/install-packages.sh
-  - /usr/local/bin/firewall-setup.sh
-  - /usr/local/bin/ssh-setup.sh
-  - /usr/local/bin/sysctl-setup.sh

terraform/modules/docker-network/templates/cloud-init-docker.yaml.tftpl

@@ -35,15 +35,19 @@ write_files:
     path: /run/tmpfiles.d/sysctl-setup.sh
     encoding: base64
     defer: true
+%{ if main_node ~}
   - owner: root:root
-    content: "${nginx_setup}"
+    content: "${swarm_setup}"
-    path: /run/tmpfiles.d/nginx-setup.sh
+    path: /run/tmpfiles.d/swarm-setup.sh
     encoding: base64
     defer: true
+%{ endif ~}
 runcmd:
   - install /run/tmpfiles.d/*.sh /usr/local/bin
   - /usr/local/bin/install-packages.sh
   - /usr/local/bin/firewall-setup.sh
   - /usr/local/bin/ssh-setup.sh
   - /usr/local/bin/sysctl-setup.sh
-  - /usr/local/bin/nginx-setup.sh
+%{ if main_node ~}
+  - /usr/local/bin/swarm-setup.sh
+%{ endif ~}

terraform/modules/docker-network/templates/scripts/nginx-setup.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-apt-get -y install gnupg2 ubuntu-keyring
-
-curl https://nginx.org/keys/nginx_signing.key | gpg --dearmor \
-    | tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null
-
-echo \
-  "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] http://nginx.org/packages/mainline/ubuntu \
-  $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") nginx" | \
-  tee /etc/apt/sources.list.d/nginx.list
-
-echo -e "Package: *\nPin: origin nginx.org\nPin: release o=nginx\nPin-Priority: 900\n" \
-    | tee /etc/apt/preferences.d/99nginx
-
-apt-get update
-apt-get -y install nginx
-
-# Remove the default Nginx configuration file
-rm /etc/nginx/sites-enabled/default
-
-# Enable the Nginx site configuration
-ln -s /etc/nginx/sites-available/default /etc/nginx/sites-enabled/default
-
-nginx

terraform/modules/docker-network/templates/scripts/swarm-setup.sh.tftpl

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+%{ if main_node ~}
+docker swarm init --advertise-addr ${main_node_private_ip}
+docker network create --driver overlay --attachable internal_net
+%{ endif ~}

terraform/modules/docker-network/variables.tf

@@ -48,13 +48,8 @@ variable "server_subnet_id" {
   type        = string
 }
 
-variable "private_nginx_vm_ip" {
+variable "private_node_ips" {
-  description = "IP in the private network subnet for the VM containing Nginx"
+  description = "One IP in the private network subnet for each Docker Swarm node created."
-  type        = string
-}
-
-variable "private_worker_vm_ips" {
-  description = "One IP in the private network subnet for each VM without Nginx created."
   type = list(string)
 }
 
@@ -74,8 +69,8 @@ variable "stage_name" {
   default     = "test"
 }
 
-variable "number_worker_vms" {
+variable "number_nodes" {
-  description = "Number of worker VMs in the docker network."
+  description = "Number of nodes in the docker swarm."
   type        = number
   default     = 0
 }

terraform/modules/docker-network/worker-vm.tf

@@ -1,86 +0,0 @@
-# TODO: Use Hetzner DNS
-# resource "inwx_nameserver_record" "worker_aaaa" {
-#   count = var.number_worker_vms
-#
-#   domain  = var.domain
-#   name    = format("%s-%d.%s", "node", count.index, local.docker_network_domain)
-#   content = hcloud_server.worker[count.index].ipv6_address
-#   type    = "AAAA"
-#   ttl     = 3600
-# }
-
-resource "hcloud_primary_ip" "ipv6_worker_address" {
-  count = var.number_worker_vms
-
-  name          = format("%s_%s_%s_%d", "docker", var.stage_name, "ipv6", count.index)
-  datacenter    = element(var.locations.*.datacenter_name, count.index)
-  type          = "ipv6"
-  assignee_type = "server"
-  auto_delete   = false
-}
-
-resource "hcloud_rdns" "ipv6_worker" {
-  count = var.number_worker_vms
-
-  primary_ip_id = hcloud_primary_ip.ipv6_worker_address[count.index].id
-  ip_address    = hcloud_server.worker[count.index].ipv6_address
-  # TODO: Use Hetzner DNS
-  # dns_ptr       = inwx_nameserver_record.worker_aaaa[count.index].name
-}
-
-resource "hcloud_server_network" "worker_private" {
-  count = var.number_worker_vms
-
-  server_id = hcloud_server.worker[count.index].id
-  subnet_id = var.server_subnet_id
-  ip        = var.private_worker_vm_ips[count.index]
-}
-
-resource "hcloud_server" "worker" {
-  count = var.number_worker_vms
-
-  name                    = format("%s-%s-%s-%d", "docker", var.stage_name, "node", count.index)
-  image                   = var.image_name
-  allow_deprecated_images = false
-  server_type             = var.server_type
-  location                = element(var.locations.*.name, count.index)
-  public_net {
-    ipv4_enabled = false
-    ipv6_enabled = true
-    ipv6         = hcloud_primary_ip.ipv6_worker_address[count.index].id
-  }
-  ignore_remote_firewall_ids = false
-  keep_disk                  = false
-  placement_group_id         = data.hcloud_placement_group.default.id
-  firewall_ids               = [var.basic_firewall_id, var.docker_firewall_id]
-  ssh_keys                   = [var.admin_ssh_key.id]
-  shutdown_before_deletion   = true
-
-  labels = {
-    "docker" : "yes"
-  }
-
-  user_data = templatefile("${path.module}/templates/cloud-init-docker-no-nginx.yaml.tftpl", {
-    admin_public_ssh_key : format("%s %s", var.admin_ssh_key.public_key, var.admin_ssh_key.name)
-    admin_user : var.admin_user
-    terraform_public_ssh_key : var.terraform_public_ssh_key
-    packages_setup : base64encode(file("${path.module}/templates/scripts/install-packages.sh"))
-    firewall_setup : base64encode(templatefile("${path.module}/templates/scripts/firewall-setup.sh", {
-      node_ip : var.private_worker_vm_ips[count.index]
-    }))
-    ssh_setup : base64encode(templatefile("${path.module}/templates/scripts/ssh-setup.sh", {
-      admin_user : var.admin_user
-    }))
-    sysctl_setup : base64encode(file("${path.module}/templates/scripts/sysctl-setup.sh"))
-  })
-
-  network {
-    network_id = var.network_id
-    ip         = var.private_worker_vm_ips[count.index]
-    alias_ips  = []
-  }
-
-  lifecycle {
-    ignore_changes = [ssh_keys, user_data]
-  }
-}