Added kafka and springBoot roles, minor cleanup, and use ansible-vault to hide sensitive data

.gitignore

@@ -1,6 +1,4 @@
 # ---> Ansible
 *.retry
-hosts
-**/*_vars.yaml
-inventories/*
-!inventories/*.example
+**/private.yaml
+vault_password

ansible/applications.yaml

@@ -1,6 +1,20 @@
 ---
+- name: Create server
+  hosts: setup
+  gather_facts: false
+  roles:
+    - server_setup
 - name: Set up domain
   hosts: frontend
+  gather_facts: false
   remote_user: "{{ ssh_user }}"
   roles:
     - frontend
+- name: Set up Spring Boot applications
+  hosts: springBoot
+  gather_facts: false
+  become: true
+  become_method: ansible.builtin.sudo
+  remote_user: "{{ ssh_user }}"
+  roles:
+    - spring_boot

ansible/devops.yaml

@@ -1,10 +1,12 @@
 ---
 - name: Create server
-  hosts: localhost
+  hosts: setup
+  gather_facts: false
   roles:
-    - server-setup
+    - server_setup
 - name: Set up drone
   hosts: ci
+  gather_facts: false
   become: true
   become_method: ansible.builtin.sudo
   remote_user: "{{ ssh_user }}"
@@ -12,6 +14,7 @@
     - drone
 - name: Set up Gitea
   hosts: git
+  gather_facts: false
   remote_user: "{{ ssh_user }}"
   roles:
     - gitea

ansible/inventories/applications

@@ -0,0 +1,8 @@
+[setup]
+applications ansible_host=localhost ansible_connection=local ansible_python_interpreter=python3
+
+[frontend]
+wahlrecht-frontend ansible_host=wahlfron.uber.space ansible_python_interpreter=/usr/bin/python
+
+[springBoot]
+api-server ansible_host=49.13.77.70 ansible_python_interpreter=python3

ansible/inventories/applications.example

@@ -1,2 +0,0 @@
-[frontend]
-user.uber.space ansible_python_interpreter=/usr/bin/python

ansible/inventories/devops

@@ -0,0 +1,8 @@
+[setup]
+devops ansible_host=localhost ansible_connection=local ansible_python_interpreter=python3
+
+[ci]
+devops ansible_host=49.12.69.146 ansible_python_interpreter=python3
+
+[git]
+gitea ansible_host=giteajim.uber.space ansible_python_interpreter=/usr/bin/python

ansible/inventories/devops.example

@@ -1,7 +0,0 @@
-localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3
-
-[ci]
-alias ansible_host=IPorDomain ansible_python_interpreter=python3
-
-[git]
-user.uber.space ansible_python_interpreter=/usr/bin/python

ansible/inventories/group_vars/ci/ci_vars.yaml.example

@@ -1,5 +0,0 @@
----
-hcloud_api_token: someAPIToken
-ssh_user: remoteUser
-admin_mail: mail@example.com
-ci_domain: yourDomainForCi

ansible/inventories/group_vars/ci/private_vault.yaml

@@ -0,0 +1,12 @@
+$ANSIBLE_VAULT;1.1;AES256
+38366333313464396137373963613664666465353139393330316631316336376662646430323739
+3165353235393532613663646230336162356231313831610a386335386436633265623266366538
+62353035316332386435316531613533613738623662396563373765656332343265613264376636
+3831366363313331330a326133336565356433663264373830653135373863656665666163653135
+34613531323030663765303264626136636564666432313364303264613364323635653436666163
+38323262383634663937666533323833316364656561633532626331343238623838366339363835
+30396235343837376135613132643766343364633034386634343235366263306434666565313366
+33616632313731626639316166383464626335346435356161393334373334323762336364336437
+65343535383732353237626437613063626631313065613763393430306432653238383165346364
+32363062356236323266613731383862376566383933633535383733343331373763626461363638
+623264356630636563613665313431323433

ansible/inventories/group_vars/ci/public.yaml

@@ -0,0 +1,6 @@
+---
+ssh_user: 2martensAdmin
+admin_mail: admin@2martens.de
+ci_domain: ci.2martens.de
+gitea_user: 2martens
+gitea_url: https://git.2martens.de

ansible/inventories/group_vars/frontend/public.yaml

@@ -0,0 +1,3 @@
+---
+ssh_user: wahlfron
+frontend_domain: wahlrecht.2martens.de

ansible/inventories/group_vars/git/private_vault.yaml

@@ -0,0 +1,19 @@
+$ANSIBLE_VAULT;1.1;AES256
+30623066653736656465303465356166646633623333346362363039336634373266666235303038
+6232303430353137303739306565356165616434656139650a303632313061616362336662356365
+64373935376334333930356465336565666132653434623063343131656262623639303337366230
+3230613932336636330a663362393635663234623539616330373963363166636538343234653263
+32633366343966333034303664383563356633663737396338356565663933383264666162316638
+39366132626466313138356164333430363734636265356637333166396363306161656262626131
+66353264343466643536343766373062363230306431663035363263373137363362326634353538
+33373034323538656666366338653733663838336133613038303739356332666636666665383735
+33366439623332393932323930616463373335326432663031663135363138386236393634333032
+65383135393938653466656430653162396430633336306234336462316463663364303439346232
+61333066626239633235386235613139656633356233663061396433613239386139643765623666
+63646163343261663234353134656533313439393032616161383265393562653339356163373031
+34356466386238666334633438633331336235316664666534363466636537353633346462306536
+65313565313433646366663533633333356137303235333735333863343732663461306235626639
+63653035303431623737626638636137656431313139386663646330663033376465663335653431
+35633165346565313938343665653337393063376537356333343532666434616231376365636266
+31663336356435366364316562326665646335663830613631376262323638383134363037633735
+6262343530653162343539656530653032663535383862333935

ansible/inventories/group_vars/git/public.yaml

@@ -0,0 +1,8 @@
+---
+ssh_user: giteajim
+host_architecture: amd64
+gitea_domain: git.2martens.de
+gitea_port: 64745
+gitea_name: 2martens Gitea
+gitea_backend: http:64745
+gitea_mail_domain: kohoutek.uberspace.de

ansible/inventories/group_vars/setup/private_vault.yaml

@@ -0,0 +1,10 @@
+$ANSIBLE_VAULT;1.1;AES256
+64656265623838666261343137623538386437623736356462313164326461633562383730393434
+3238303631303331363833643937663937663263383361340a363234376566393533623030666632
+66323464663730636461303466633733336130386631333362343631366132366332346166323332
+3565626362653161610a363237613963313661323439653031326430363337343638633630386430
+36336237376137316638393034363834643038613930363965323232336131643335653331343939
+39376237396539393931616363633031323932333566386362393836633864653262323633613533
+64393237313230373933316638373436616137303236323536663664373732383539656236373530
+38613332316439363738663365373638373061666566643164333166356366346337393038393032
+3233

ansible/inventories/group_vars/setup/public.yaml

@@ -0,0 +1,4 @@
+---
+admin_user: 2martensAdmin
+ssh_key_admin_user: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBjYkz73HbsFtRAOfS5QB9xDbOajU9C4gpDp+OjQkLir 2martens@Jims-Air.fritz.box
+ssh_key_admin_user_name: 2martens@Jims-Air.fritz.box

ansible/inventories/group_vars/springBoot/public.yaml

@@ -0,0 +1,3 @@
+---
+ssh_user: 2martensAdmin
+admin_mail: admin@2martens.de

ansible/inventories/host_vars/api-server/private_vault.yaml

@@ -0,0 +1,18 @@
+$ANSIBLE_VAULT;1.1;AES256
+36346336333535323261633361383234303961646434646635363930653938313537306262666535
+3630343635653831396234373663353437663762613064330a323836643966636135343331353634
+34303539343362623730653762636139663036323464643635323633326166383631303637383739
+3132323236383566370a393830663962383838383332623932633866323837636464656339656439
+33323863373761363133643336323731346331376364373833653562643133366134643833393031
+61316164393833393732643431663965393766393135376433316337646561653037373738616531
+31663166343964626664663335343638613832353038313264613230316432656530306265323138
+37633238393663343535326336326263333131373533303338626366656633376634346162363033
+61353763366465663431333339306239656464373231396166623763636234393534323062323631
+38323436663361333965326665383432643439636230396339646362336531643965613763323266
+65613330333432396664656234626439633331653230366264386263653738323133623938616538
+39396662396263383832616534316661336638366438643461336338313937636230336136666239
+61383838636132643834363736383933663238663635656665363734633733356661666366396563
+37616338303637636161366331343635393332343466333965633034633532366439393437313264
+36343337666561323863336366656337616463326330313334316335323165366431336537376563
+31373738383465366366313061303466663863613664303431366662616539643136636335636535
+37666535653432643134363964326639323935303335333139626136633939333635

ansible/inventories/host_vars/api-server/public.yaml

@@ -0,0 +1,23 @@
+---
+spring_domain: api.2martens.de
+spring_use_nginx: true
+spring_docker_path: api
+configserver_image_version: 0.1.8
+wahlrecht_image_version: 0.1.27
+configserver_environment:
+  - "SPRING_PROFILES_ACTIVE=prod"
+spring_apps:
+  - name: configserver
+    port: 8888
+    image: 2martens/configserver
+    path: "= /monitor"
+    expose: true
+    has_environment: true
+  - name: wahlrecht
+    port: 12000
+    image: 2martens/wahlrecht
+    path: /wahlrecht
+    expose: true
+    has_environment: true
+    depends_on:
+      - configserver

ansible/inventories/host_vars/applications/public.yaml

@@ -0,0 +1,3 @@
+---
+server_name: applications
+server_type: cax11

ansible/inventories/host_vars/devops/public.yaml

@@ -0,0 +1,3 @@
+---
+server_name: devops
+server_type: cax21

ansible/inventories/host_vars/localhost/localhost_vars.yaml.example

@@ -1,6 +0,0 @@
----
-hcloud_api_token: someAPIToken
-admin_user: nameForNewAdminUser
-ssh_key_admin_user: publicKeyForNewAdminUser
-ssh_key_admin_user_name: nameForSSHKeyForNewAdminUser
-server_name: nameOfTheNewServer

ansible/roles/apache/tasks/letsencrypt.yaml

@@ -0,0 +1,5 @@
+---
+- name: Install python3-certbot-apache
+  ansible.builtin.apt:
+    name: python3-certbot-apache
+    update_cache: true

ansible/roles/apache/tasks/main.yml

@@ -1,4 +1,6 @@
 ---
+- name: Install Apache certbot
+  ansible.builtin.import_tasks: letsencrypt.yaml
 - name: Install apache 2
   ansible.builtin.apt:
     name: apache2

ansible/roles/drone/tasks/drone.yml

@@ -11,13 +11,6 @@
     cmd: openssl rand -hex 16
   register: drone_rpc_secret
   changed_when: false
-- name: Create drone config directory
-  ansible.builtin.file:
-    state: directory
-    path: /etc/drone
-    mode: "755"
-    owner: root
-    group: root
 - name: Copy docker compose file for drone
   ansible.builtin.template:
     src: etc/drone/docker-compose.yml.j2

ansible/roles/gitea/tasks/gitea.yaml

@@ -47,9 +47,9 @@
   ansible.builtin.command:
     chdir: /home/{{ ssh_user }}/gitea/custom/conf
     argv:
-    - bash
-    - -c
-    - grep -e ^SECRET_KEY app.ini | cut -f 3 -d ' '
+      - bash
+      - -c
+      - grep -e ^SECRET_KEY app.ini | cut -f 3 -d ' '
 - name: Read existing internal token
   register: existing_internal_token
   when: config_file.stat.exists

ansible/roles/kafka/meta/main.yaml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: docker

ansible/roles/kafka/tasks/main.yaml

@@ -0,0 +1,24 @@
+---
+- name: Create Kafka directory
+  ansible.builtin.file:
+    state: directory
+    path: /opt/kafka
+    owner: root
+    group: root
+    mode: "755"
+- name: Copy docker compose file for kafka
+  ansible.builtin.template:
+    src: opt/kafka/docker-compose.yaml.j2
+    dest: /opt/kafka/docker-compose.yaml
+    owner: root
+    group: root
+    mode: "644"
+    lstrip_blocks: true
+- name: Create Kafka network
+  community.docker.docker_network:
+    name: kafka
+    state: present
+    appends: true
+- name: Start kafka docker container
+  community.docker.docker_compose:
+    project_src: /opt/kafka

ansible/roles/kafka/templates/opt/kafka/docker-compose.yaml.j2

@@ -0,0 +1,41 @@
+version: '2'
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:latest
+    restart: always
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+    expose:
+      - "2181"
+    networks:
+      - kafka
+
+  kafka:
+    image: confluentinc/cp-kafka:latest
+    depends_on:
+      - zookeeper
+    expose:
+      - "9092"
+    networks:
+      - kafka
+    restart: always
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_LISTENERS: PLAINTEXT://kafka:9092
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+
+  kafkacat:
+    image: confluentinc/cp-kafkacat:7.0.10
+    command: sleep infinity
+    networks:
+      - kafka
+
+networks:
+  kafka:
+    name: kafka
+    external: true

ansible/roles/letsencrypt/tasks/main.yml

@@ -3,7 +3,3 @@
   ansible.builtin.apt:
     name: certbot
     update_cache: true
-- name: Install python3-certbot-apache
-  ansible.builtin.apt:
-    name: python3-certbot-apache
-    update_cache: true

ansible/roles/nginx/handlers/main.yaml

@@ -0,0 +1,9 @@
+---
+- name: Restart nginx
+  ansible.builtin.service:
+    name: nginx
+    state: restarted
+- name: Reload nginx
+  ansible.builtin.service:
+    name: nginx
+    state: reloaded

ansible/roles/nginx/meta/main.yaml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: letsencrypt

ansible/roles/nginx/tasks/letsencrypt.yaml

@@ -0,0 +1,12 @@
+---
+- name: Install python3-certbot-nginx
+  ansible.builtin.apt:
+    name: python3-certbot-nginx
+    update_cache: true
+- name: Create directory for acme challenge
+  ansible.builtin.file:
+    state: directory
+    path: /var/www/html/.well-known/acme-challenge
+    owner: root
+    group: root
+    mode: "700"

ansible/roles/nginx/tasks/main.yaml

@@ -0,0 +1,11 @@
+---
+- name: Install nginx
+  ansible.builtin.apt:
+    name: nginx
+    update_cache: true
+- name: Allow nginx through firewall
+  community.general.ufw:
+    name: Nginx Full
+    rule: allow
+- name: Prepare for Let's Encrypt
+  ansible.builtin.import_tasks: letsencrypt.yaml

ansible/roles/spring_boot/meta/main.yaml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  - role: docker
+  - role: letsencrypt
+  - role: nginx
+  - role: kafka

ansible/roles/spring_boot/tasks/docker.yaml

@@ -0,0 +1,19 @@
+---
+- name: Create Spring Boot application directory
+  ansible.builtin.file:
+    state: directory
+    path: /opt/{{ spring_docker_path }}
+    owner: root
+    group: root
+    mode: "755"
+- name: Copy docker compose file for application
+  ansible.builtin.template:
+    src: opt/application/docker-compose.yaml.j2
+    dest: /opt/{{ spring_docker_path }}/docker-compose.yaml
+    owner: root
+    group: root
+    mode: "644"
+    lstrip_blocks: true
+- name: Start application docker container
+  community.docker.docker_compose:
+    project_src: /opt/{{ spring_docker_path }}

ansible/roles/spring_boot/tasks/main.yaml

@@ -0,0 +1,6 @@
+---
+- name: Set up nginx for Spring Boot application
+  ansible.builtin.import_tasks: nginx.yaml
+  when: spring_use_nginx
+- name: Set up docker-compose
+  ansible.builtin.import_tasks: docker.yaml

ansible/roles/spring_boot/tasks/nginx.yaml

@@ -0,0 +1,38 @@
+---
+- name: Add HTTP site
+  ansible.builtin.template:
+    src: etc/nginx/sites-available/spring_domain.conf.j2
+    dest: /etc/nginx/sites-available/{{ spring_domain }}.conf
+    owner: root
+    group: root
+    mode: "644"
+    force: false
+    lstrip_blocks: true
+- name: Disable default site
+  ansible.builtin.command:
+    cmd: rm /etc/nginx/sites-enabled/default
+    removes: /etc/nginx/sites-enabled/default
+- name: Enable HTTP site
+  ansible.builtin.command:
+    cmd: ln -sf /etc/nginx/sites-available/{{ spring_domain }}.conf /etc/nginx/sites-enabled/{{ spring_domain }}.conf
+    creates: /etc/nginx/sites-enabled/{{ spring_domain }}.conf
+  notify: Reload nginx
+- name: Check certificate existence
+  ansible.builtin.stat:
+    path: "/etc/letsencrypt/live/{{ spring_domain }}/cert.pem"
+  register: certificate
+- name: Generate certificates
+  ansible.builtin.command:
+    cmd: >
+      certbot --nginx --non-interactive --keep-until-expiring
+      --no-eff-email
+      --email {{ admin_mail }}
+      --redirect
+      --renew-with-new-domains --agree-tos -d {{ spring_domain }}
+    creates: "/etc/letsencrypt/live/{{ spring_domain }}/cert.pem"
+  register: certbot_certificate_created
+  notify: Reload nginx
+  failed_when:
+    - "'Successfully received certificate.' not in certbot_certificate_created.stdout"
+  when:
+    - not certificate.stat.exists

ansible/roles/spring_boot/templates/etc/nginx/sites-available/spring_domain.conf.j2

@@ -0,0 +1,18 @@
+server {
+    listen 80;
+    listen [::]:80;
+
+    server_name {{ spring_domain }};
+
+    location ^~ /.well-known/acme-challenge/ {
+        default_type "text/plain";
+    }
+
+    {% for app in spring_apps %}
+    {% if app.expose %}
+    location {{ app.path }} {
+        proxy_pass http://localhost:{{ app.port }};
+    }
+    {% endif %}
+    {% endfor %}
+}

ansible/roles/spring_boot/templates/opt/application/docker-compose.yaml.j2

@@ -0,0 +1,28 @@
+version: '2'
+
+services:
+  {% for app in spring_apps %}
+  {{ app.name }}:
+    image: {{ app.image }}:{{ lookup('ansible.builtin.vars', app.name + '_image_version') }}
+    {% if app.expose %}
+    ports:
+      - "{{ app.port }}:{{ app.port }}"
+    {% else %}
+    expose:
+      - "{{ app.port }}"
+    {% endif %}
+    {% if 'depends_on' in app %}
+    depends_on: {{ app.depends_on }}
+    {% endif %}
+    restart: always
+    networks:
+      - kafka
+    {% if app.has_environment %}
+    environment: {{ lookup('ansible.builtin.vars', app.name + '_environment') }}
+    {% endif %}
+  {% endfor %}
+
+networks:
+  kafka:
+    name: kafka
+    external: true

ansible/site.yaml

@@ -1,2 +1,3 @@
 ---
 - include: devops.yaml
+- include: applications.yaml