Install cilium directly from Helm, not as addon

Also install cert-manager directly from Helm to enable experimental Gateway API support.

terraform/modules/k8s-cluster/manager-node.tf

@@ -131,6 +131,9 @@ resource "hcloud_server" "manager" {
       high_availability : var.number_nodes > 2
       admin_user : var.admin_user
     }))
+    cilium_values : base64encode(templatefile("${path.module}/templates/helm/cilium-values.yaml", {
+      high_availability : var.number_nodes > 1
+    }))
   })
 
   network {

terraform/modules/k8s-cluster/templates/cloud-init-k8s-manager.tftpl

@@ -66,6 +66,11 @@ write_files:
     path: /run/tmpfiles.d/argocd-values.yaml
     encoding: base64
     defer: true
+  - owner: root:root
+    content: "${cilium_values}"
+    path: /run/tmpfiles.d/cilium-values.yaml
+    encoding: base64
+    defer: true
 %{ endif ~}
 runcmd:
   - install /run/tmpfiles.d/*.sh /usr/local/bin

terraform/modules/k8s-cluster/templates/helm/cilium-values.yaml

@@ -0,0 +1,69 @@
+rolloutCiliumPods: true
+annotateK8sNode: true
+bpf:
+  hostLegacyRouting: false
+cni:
+  install: true
+  uninstall: false
+  customConf: false
+  confPath: "/var/snap/microk8s/current/args/cni-network"
+  binPath: "/var/snap/microk8s/current/opt/cni/bin"
+  logFile: "/var/snap/microk8s/current/var/run/cilium/cilium-cni.log"
+daemon:
+  runPath: /var/snap/microk8s/current/var/run/cilium
+gatewayAPI:
+  enabled: false
+endpointStatus:
+  enabled: true
+  status: health
+hubble:
+  enabled: true
+  metrics:
+    enabled:
+      - dns
+      - drop
+      - tcp
+      - flow
+      - port-distribution
+      - icmp
+      - httpV2:exemplars=true;labelsContext=source_ip,source_namespace,source_workload,destination_ip,destination_namespace,destination_workload,traffic_direction
+    serviceMonitor:
+      enabled: true
+  tls:
+    auto:
+      method: helm
+#      certManagerIssuerRef:
+#        group: cert-manager.io
+#        kind: ClusterIssuer
+#        name: lets-encrypt
+  backend:
+    livenessProbe:
+      enabled: true
+    readinessProbe:
+      enabled: true
+ipam:
+  operator:
+    clusterPoolIPv4PodCIDRList:
+      - "10.1.0.0/16"
+kubeProxyReplacement: true
+nodePort:
+  enabled: true
+  enableHealthCheckLoadBalancerIP: true
+prometheus:
+  enabled: true
+  serviceMonitor:
+    enabled: true
+routingMode: ""
+wellKnownIdentities:
+  enabled: true
+operator:
+%{ if high_availability ~}
+  replicas: 2
+%{ else ~}
+  replicas: 1
+%{ endif ~}
+  rollOutPods: true
+  prometheus:
+    enabled: true
+    serviceMonitor:
+      enabled: true

terraform/modules/k8s-cluster/templates/scripts/cluster-setup.sh

@@ -10,12 +10,10 @@ chmod 600 /home/${admin_user}/.kube/config
 chown -R ${admin_user}:${admin_user} /home/${admin_user}/.kube
 /snap/bin/microk8s.helm3 repo add 2martens https://repo.2martens.de/charts/
 /snap/bin/microk8s.helm3 repo add argo https://argoproj.github.io/argo-helm
-/snap/bin/microk8s.helm3 repo add cilium https://helm.cilium.io
-/snap/bin/microk8s.helm3 upgrade cilium cilium/cilium --version 1.13.4 --namespace kube-system --reuse-values --set prometheus.enabled=true --set operator.prometheus.enabled=true --set hubble.metrics.enableOpenMetrics=true --set hubble.metrics.enabled="{dns,drop,tcp,flow,port-distribution,icmp,httpV2:exemplars=true;labelsContext=source_ip\,source_namespace\,source_workload\,destination_ip\,destination_namespace\,destination_workload\,traffic_direction}"
 /snap/bin/microk8s.helm3 install setup 2martens/cluster_setup --values /run/tmpfiles.d/cluster-setup-values.yaml
 %{ if high_availability ~}
 /snap/bin/microk8s.helm3 install argo-cd argo/argo-cd --create-namespace -n argocd --values /run/tmpfiles.d/argocd-values-ha.yaml
 %{ else ~}
 /snap/bin/microk8s.helm3 install argo-cd argo/argo-cd --create-namespace -n argocd --values /run/tmpfiles.d/argocd-values.yaml
 %{ endif ~}
-/snap/bin/microk8s.helm3 install argocd-config 2martens/argocd-configuration -n argocd --set environment=${argocd_environment}
+/snap/bin/microk8s.helm3 install argocd-config 2martens/argocd-configuration -n argocd --set environment=${argocd_environment} --set gateway.enabled=true

terraform/modules/k8s-cluster/templates/scripts/microk8s-setup.sh

@@ -11,9 +11,20 @@ snap install microk8s --classic --channel "${microk8s_channel}"
 /snap/bin/microk8s addons repo add community https://github.com/canonical/microk8s-community-addons
 /snap/bin/microk8s enable cis-hardening
 /snap/bin/microk8s enable rbac
+/snap/bin/microk8s.kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/experimental-install.yaml
 %{ if manager_node ~}
+/snap/bin/microk8s.helm3 repo add prometheus-community https://prometheus-community.github.io/helm-charts
+/snap/bin/microk8s.helm3 repo add cilium https://helm.cilium.io
+/snap/bin/microk8s.helm3 repo add jetstack https://charts.jetstack.io
+/snap/bin/microk8s.helm3 install prometheus-crds prometheus-community/prometheus-operator-crds
+/snap/bin/microk8s.kubectl delete -f "/var/snap/microk8s/current/args/cni-network/cni.yaml"
+sleep 2s
+sudo mv /var/snap/microk8s/current/args/cni-network/cni.yaml /var/snap/microk8s/current/args/cni-network/cni.yaml.disabled
+/snap/bin/microk8s.helm3 install cilium cilium/cilium --namespace kube-system --values /run/tmpfiles.d/cilium-values.yaml
+/snap/bin/microk8s.helm3 install cert-manager jetstack/cert-manager \
+    --namespace cert-manager \
+    --set installCRDs=true \
+    --create-namespace \
+    --set "extraArgs={--feature-gates=ExperimentalGatewayAPISupport=true}"
 /snap/bin/microk8s enable ingress
-/snap/bin/microk8s enable cert-manager
-/snap/bin/microk8s enable cilium
-/snap/bin/microk8s cilium hubble enable
 %{ endif ~}