diff --git a/terraform/modules/k8s-cluster/manager-node.tf b/terraform/modules/k8s-cluster/manager-node.tf index f01f584..9f0123a 100644 --- a/terraform/modules/k8s-cluster/manager-node.tf +++ b/terraform/modules/k8s-cluster/manager-node.tf @@ -131,6 +131,9 @@ resource "hcloud_server" "manager" { high_availability : var.number_nodes > 2 admin_user : var.admin_user })) + cilium_values : base64encode(templatefile("${path.module}/templates/helm/cilium-values.yaml", { + high_availability : var.number_nodes > 1 + })) }) network { diff --git a/terraform/modules/k8s-cluster/templates/cloud-init-k8s-manager.tftpl b/terraform/modules/k8s-cluster/templates/cloud-init-k8s-manager.tftpl index a25e3ec..a359552 100644 --- a/terraform/modules/k8s-cluster/templates/cloud-init-k8s-manager.tftpl +++ b/terraform/modules/k8s-cluster/templates/cloud-init-k8s-manager.tftpl @@ -66,6 +66,11 @@ write_files: path: /run/tmpfiles.d/argocd-values.yaml encoding: base64 defer: true + - owner: root:root + content: "${cilium_values}" + path: /run/tmpfiles.d/cilium-values.yaml + encoding: base64 + defer: true %{ endif ~} runcmd: - install /run/tmpfiles.d/*.sh /usr/local/bin diff --git a/terraform/modules/k8s-cluster/templates/helm/cilium-values.yaml b/terraform/modules/k8s-cluster/templates/helm/cilium-values.yaml new file mode 100644 index 0000000..3270c57 --- /dev/null +++ b/terraform/modules/k8s-cluster/templates/helm/cilium-values.yaml @@ -0,0 +1,69 @@ +rolloutCiliumPods: true +annotateK8sNode: true +bpf: + hostLegacyRouting: false +cni: + install: true + uninstall: false + customConf: false + confPath: "/var/snap/microk8s/current/args/cni-network" + binPath: "/var/snap/microk8s/current/opt/cni/bin" + logFile: "/var/snap/microk8s/current/var/run/cilium/cilium-cni.log" +daemon: + runPath: /var/snap/microk8s/current/var/run/cilium +gatewayAPI: + enabled: false +endpointStatus: + enabled: true + status: health +hubble: + enabled: true + metrics: + enabled: + - dns + - drop + - tcp + - flow + - port-distribution + - icmp + - httpV2:exemplars=true;labelsContext=source_ip,source_namespace,source_workload,destination_ip,destination_namespace,destination_workload,traffic_direction + serviceMonitor: + enabled: true + tls: + auto: + method: helm +# certManagerIssuerRef: +# group: cert-manager.io +# kind: ClusterIssuer +# name: lets-encrypt + backend: + livenessProbe: + enabled: true + readinessProbe: + enabled: true +ipam: + operator: + clusterPoolIPv4PodCIDRList: + - "10.1.0.0/16" +kubeProxyReplacement: true +nodePort: + enabled: true + enableHealthCheckLoadBalancerIP: true +prometheus: + enabled: true + serviceMonitor: + enabled: true +routingMode: "" +wellKnownIdentities: + enabled: true +operator: +%{ if high_availability ~} + replicas: 2 +%{ else ~} + replicas: 1 +%{ endif ~} + rollOutPods: true + prometheus: + enabled: true + serviceMonitor: + enabled: true diff --git a/terraform/modules/k8s-cluster/templates/scripts/cluster-setup.sh b/terraform/modules/k8s-cluster/templates/scripts/cluster-setup.sh index 1082ac3..0af9632 100644 --- a/terraform/modules/k8s-cluster/templates/scripts/cluster-setup.sh +++ b/terraform/modules/k8s-cluster/templates/scripts/cluster-setup.sh @@ -10,12 +10,10 @@ chmod 600 /home/${admin_user}/.kube/config chown -R ${admin_user}:${admin_user} /home/${admin_user}/.kube /snap/bin/microk8s.helm3 repo add 2martens https://repo.2martens.de/charts/ /snap/bin/microk8s.helm3 repo add argo https://argoproj.github.io/argo-helm -/snap/bin/microk8s.helm3 repo add cilium https://helm.cilium.io -/snap/bin/microk8s.helm3 upgrade cilium cilium/cilium --version 1.13.4 --namespace kube-system --reuse-values --set prometheus.enabled=true --set operator.prometheus.enabled=true --set hubble.metrics.enableOpenMetrics=true --set hubble.metrics.enabled="{dns,drop,tcp,flow,port-distribution,icmp,httpV2:exemplars=true;labelsContext=source_ip\,source_namespace\,source_workload\,destination_ip\,destination_namespace\,destination_workload\,traffic_direction}" /snap/bin/microk8s.helm3 install setup 2martens/cluster_setup --values /run/tmpfiles.d/cluster-setup-values.yaml %{ if high_availability ~} /snap/bin/microk8s.helm3 install argo-cd argo/argo-cd --create-namespace -n argocd --values /run/tmpfiles.d/argocd-values-ha.yaml %{ else ~} /snap/bin/microk8s.helm3 install argo-cd argo/argo-cd --create-namespace -n argocd --values /run/tmpfiles.d/argocd-values.yaml %{ endif ~} -/snap/bin/microk8s.helm3 install argocd-config 2martens/argocd-configuration -n argocd --set environment=${argocd_environment} +/snap/bin/microk8s.helm3 install argocd-config 2martens/argocd-configuration -n argocd --set environment=${argocd_environment} --set gateway.enabled=true diff --git a/terraform/modules/k8s-cluster/templates/scripts/microk8s-setup.sh b/terraform/modules/k8s-cluster/templates/scripts/microk8s-setup.sh index 4e0a56f..51fd1a7 100644 --- a/terraform/modules/k8s-cluster/templates/scripts/microk8s-setup.sh +++ b/terraform/modules/k8s-cluster/templates/scripts/microk8s-setup.sh @@ -11,9 +11,20 @@ snap install microk8s --classic --channel "${microk8s_channel}" /snap/bin/microk8s addons repo add community https://github.com/canonical/microk8s-community-addons /snap/bin/microk8s enable cis-hardening /snap/bin/microk8s enable rbac +/snap/bin/microk8s.kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/experimental-install.yaml %{ if manager_node ~} +/snap/bin/microk8s.helm3 repo add prometheus-community https://prometheus-community.github.io/helm-charts +/snap/bin/microk8s.helm3 repo add cilium https://helm.cilium.io +/snap/bin/microk8s.helm3 repo add jetstack https://charts.jetstack.io +/snap/bin/microk8s.helm3 install prometheus-crds prometheus-community/prometheus-operator-crds +/snap/bin/microk8s.kubectl delete -f "/var/snap/microk8s/current/args/cni-network/cni.yaml" +sleep 2s +sudo mv /var/snap/microk8s/current/args/cni-network/cni.yaml /var/snap/microk8s/current/args/cni-network/cni.yaml.disabled +/snap/bin/microk8s.helm3 install cilium cilium/cilium --namespace kube-system --values /run/tmpfiles.d/cilium-values.yaml +/snap/bin/microk8s.helm3 install cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --set installCRDs=true \ + --create-namespace \ + --set "extraArgs={--feature-gates=ExperimentalGatewayAPISupport=true}" /snap/bin/microk8s enable ingress -/snap/bin/microk8s enable cert-manager -/snap/bin/microk8s enable cilium -/snap/bin/microk8s cilium hubble enable %{ endif ~}