From f5f45e7afb52f9e5d943ce8200694bd59d86b194 Mon Sep 17 00:00:00 2001
From: Ilia Denisov <evilcart@gmail.com>
Date: Sun, 21 Jun 2026 23:54:57 +0200
Subject: [PATCH] feat(deploy): Ansible provisioning for prod hosts (Stage 18)

Idempotent playbooks under deploy/ansible/ prepare both production hosts:
docker-ce + compose plugin, a non-sudo deploy service account holding the CI
deploy key, key-only sshd, default-deny ufw, fail2ban, unattended upgrades and
chrony. The main host also opens 80/443/9443 and creates the external edge
network; the tg host verifies direct Bot API egress (the no-VPN assumption).

The application is deployed separately by the prod-deploy workflow (later
phase), running as the deploy account this playbook provisions.
---
 deploy/ansible/README.md                      |  49 +++++
 deploy/ansible/ansible.cfg                    |  11 ++
 deploy/ansible/group_vars/all.yml             |  21 +++
 deploy/ansible/inventory.ini                  |  19 ++
 deploy/ansible/roles/common/handlers/main.yml |  15 ++
 deploy/ansible/roles/common/tasks/main.yml    | 167 ++++++++++++++++++
 .../roles/common/templates/daemon.json.j2     |   8 +
 .../roles/common/templates/jail.local.j2      |   9 +
 .../common/templates/sshd-hardening.conf.j2   |   6 +
 deploy/ansible/roles/main/tasks/main.yml      |  18 ++
 deploy/ansible/roles/tg/tasks/main.yml        |  19 ++
 deploy/ansible/site.yml                       |  31 ++++
 12 files changed, 373 insertions(+)
 create mode 100644 deploy/ansible/README.md
 create mode 100644 deploy/ansible/ansible.cfg
 create mode 100644 deploy/ansible/group_vars/all.yml
 create mode 100644 deploy/ansible/inventory.ini
 create mode 100644 deploy/ansible/roles/common/handlers/main.yml
 create mode 100644 deploy/ansible/roles/common/tasks/main.yml
 create mode 100644 deploy/ansible/roles/common/templates/daemon.json.j2
 create mode 100644 deploy/ansible/roles/common/templates/jail.local.j2
 create mode 100644 deploy/ansible/roles/common/templates/sshd-hardening.conf.j2
 create mode 100644 deploy/ansible/roles/main/tasks/main.yml
 create mode 100644 deploy/ansible/roles/tg/tasks/main.yml
 create mode 100644 deploy/ansible/site.yml

diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md
new file mode 100644
index 0000000..36fd44e
--- /dev/null
+++ b/deploy/ansible/README.md
@@ -0,0 +1,49 @@
+# Prod host provisioning (Stage 18)
+
+Idempotent Ansible that prepares the two production hosts. It installs Docker, a
+non-sudo `deploy` service account, SSH hardening, a default-deny firewall,
+fail2ban, unattended security upgrades and time sync. It does **not** deploy the
+application — that is `.gitea/workflows/prod-deploy.yaml`'s job, running as the
+`deploy` account this playbook creates.
+
+Hosts are referenced by `~/.ssh/config` aliases (`scrabble-main-ops`,
+`scrabble-tg-ops`), so no IPs or key paths live in the repo.
+
+## Prerequisites (controller)
+
+- `ansible` with the bundled collections (`community.general`, `community.docker`,
+  `ansible.posix`).
+- The two hosts reachable as root via the ssh-config aliases, host keys already
+  accepted into `known_hosts` (`host_key_checking = True`).
+
+## One-time: the CI deploy key
+
+The CI prod-deploy workflow logs into the hosts as `deploy` using a dedicated
+key. Generate it once on the controller, authorize its public half via the
+playbook, and store its private half **only** in the Gitea `PROD_SSH_KEY` secret:
+
+```sh
+ssh-keygen -t ed25519 -N '' -C scrabble-ci-deploy \
+  -f ~/.ssh/scrabble_ci_deploy_ed25519
+# private half -> Gitea secret PROD_SSH_KEY (set via API); never commit it
+```
+
+## Run
+
+```sh
+cd deploy/ansible
+ansible-playbook site.yml
+```
+
+The playbook reads the public key from `~/.ssh/scrabble_ci_deploy_ed25519.pub` by
+default; override with `-e deploy_ci_pubkey_path=/path/to/key.pub`. Re-running is
+safe (idempotent) and survives a host resize.
+
+## What each host gets
+
+- **both** (`common`): docker-ce + compose plugin, `daemon.json` (live-restore,
+  10m×3 log rotation), `deploy` user (docker group, no sudo), key-only sshd,
+  `ufw` default-deny incoming + allow SSH, fail2ban sshd jail, unattended
+  upgrades, chrony, `/opt/scrabble/{config,certs,dumps,images}`.
+- **main**: `ufw` opens 80/443/9443; the external `edge` docker network.
+- **tg**: verifies direct `api.telegram.org` egress (the no-VPN assumption).
diff --git a/deploy/ansible/ansible.cfg b/deploy/ansible/ansible.cfg
new file mode 100644
index 0000000..f846548
--- /dev/null
+++ b/deploy/ansible/ansible.cfg
@@ -0,0 +1,11 @@
+[defaults]
+inventory = inventory.ini
+roles_path = roles
+interpreter_python = /usr/bin/python3
+host_key_checking = True
+stdout_callback = yaml
+deprecation_warnings = False
+retry_files_enabled = False
+
+[ssh_connection]
+pipelining = True
diff --git a/deploy/ansible/group_vars/all.yml b/deploy/ansible/group_vars/all.yml
new file mode 100644
index 0000000..bb1f07a
--- /dev/null
+++ b/deploy/ansible/group_vars/all.yml
@@ -0,0 +1,21 @@
+---
+# Service account the CI prod-deploy workflow uses to drive docker on the hosts.
+# Membership in the docker group is root-equivalent (docker socket access), which
+# is all the deploy workflow needs; the account is deliberately not given sudo.
+deploy_user: deploy
+
+# Public half of the dedicated CI deploy SSH key, read from the controller at run
+# time. The private half is generated on the controller during provisioning and
+# stored ONLY in the Gitea PROD_SSH_KEY secret; it is never committed. Override the
+# path with -e deploy_ci_pubkey_path=/path/to/key.pub if the key lives elsewhere.
+deploy_ci_pubkey_path: "{{ lookup('env', 'HOME') }}/.ssh/scrabble_ci_deploy_ed25519.pub"
+deploy_ci_pubkey: "{{ lookup('file', deploy_ci_pubkey_path) }}"
+
+# Base directory the deploy workflow rsyncs compose files, config, certs and dumps
+# into. Owned by deploy_user so the workflow needs no elevation.
+scrabble_base_dir: /opt/scrabble
+
+# Docker daemon json-file log rotation, mirroring the compose x-logging anchor so
+# the host's own containers (and any ad-hoc runs) rotate identically.
+docker_log_max_size: "10m"
+docker_log_max_file: "3"
diff --git a/deploy/ansible/inventory.ini b/deploy/ansible/inventory.ini
new file mode 100644
index 0000000..25b80df
--- /dev/null
+++ b/deploy/ansible/inventory.ini
@@ -0,0 +1,19 @@
+# Production inventory for Stage 18.
+#
+# Hosts resolve through the operator's ~/.ssh/config aliases, so HostName (public
+# IP), User and IdentityFile live there — no IPs or key paths are committed here.
+#   scrabble-main-ops -> main stack host (public IP, domain erudit-game.ru)
+#   scrabble-tg-ops   -> Telegram bot host (direct Bot API egress, no VPN)
+
+[main]
+scrabble-main-ops
+
+[tg]
+scrabble-tg-ops
+
+[prod:children]
+main
+tg
+
+[prod:vars]
+ansible_user=root
diff --git a/deploy/ansible/roles/common/handlers/main.yml b/deploy/ansible/roles/common/handlers/main.yml
new file mode 100644
index 0000000..017e54b
--- /dev/null
+++ b/deploy/ansible/roles/common/handlers/main.yml
@@ -0,0 +1,15 @@
+---
+- name: restart docker
+  ansible.builtin.service:
+    name: docker
+    state: restarted
+
+- name: reload sshd
+  ansible.builtin.service:
+    name: ssh
+    state: reloaded
+
+- name: restart fail2ban
+  ansible.builtin.service:
+    name: fail2ban
+    state: restarted
diff --git a/deploy/ansible/roles/common/tasks/main.yml b/deploy/ansible/roles/common/tasks/main.yml
new file mode 100644
index 0000000..5c0a7a3
--- /dev/null
+++ b/deploy/ansible/roles/common/tasks/main.yml
@@ -0,0 +1,167 @@
+---
+# Common baseline applied to both prod hosts: Docker engine, a non-sudo deploy
+# service account, SSH hardening, a default-deny firewall, fail2ban, unattended
+# security upgrades and time sync. Every task is idempotent.
+
+- name: Install base packages
+  ansible.builtin.apt:
+    name:
+      - ca-certificates
+      - curl
+      - gnupg
+      - ufw
+      - fail2ban
+      - unattended-upgrades
+      - chrony
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+
+# --- Docker engine (official repo; trixie is published upstream) ---------------
+
+- name: Create apt keyring directory
+  ansible.builtin.file:
+    path: /etc/apt/keyrings
+    state: directory
+    mode: "0755"
+
+- name: Install Docker apt GPG key
+  ansible.builtin.get_url:
+    url: https://download.docker.com/linux/debian/gpg
+    dest: /etc/apt/keyrings/docker.asc
+    mode: "0644"
+
+- name: Add Docker apt repository
+  ansible.builtin.apt_repository:
+    repo: >-
+      deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc]
+      https://download.docker.com/linux/debian {{ ansible_distribution_release }} stable
+    filename: docker
+    state: present
+
+- name: Install Docker engine and the compose plugin
+  ansible.builtin.apt:
+    name:
+      - docker-ce
+      - docker-ce-cli
+      - containerd.io
+      - docker-buildx-plugin
+      - docker-compose-plugin
+    state: present
+    update_cache: true
+
+- name: Configure the Docker daemon (live-restore + log rotation)
+  ansible.builtin.template:
+    src: daemon.json.j2
+    dest: /etc/docker/daemon.json
+    mode: "0644"
+  notify: restart docker
+
+- name: Enable and start Docker
+  ansible.builtin.service:
+    name: docker
+    enabled: true
+    state: started
+
+# --- Deploy service account ----------------------------------------------------
+
+- name: Create the deploy service account
+  ansible.builtin.user:
+    name: "{{ deploy_user }}"
+    groups: docker
+    append: true
+    shell: /bin/bash
+    create_home: true
+
+- name: Ensure the deploy .ssh directory
+  ansible.builtin.file:
+    path: "/home/{{ deploy_user }}/.ssh"
+    state: directory
+    owner: "{{ deploy_user }}"
+    group: "{{ deploy_user }}"
+    mode: "0700"
+
+- name: Authorize the CI deploy SSH key (exclusive)
+  ansible.builtin.copy:
+    dest: "/home/{{ deploy_user }}/.ssh/authorized_keys"
+    content: "{{ deploy_ci_pubkey }}\n"
+    owner: "{{ deploy_user }}"
+    group: "{{ deploy_user }}"
+    mode: "0600"
+
+# --- SSH hardening -------------------------------------------------------------
+
+- name: Harden sshd (key-only auth)
+  ansible.builtin.template:
+    src: sshd-hardening.conf.j2
+    dest: /etc/ssh/sshd_config.d/10-scrabble-hardening.conf
+    mode: "0644"
+    validate: sshd -t -f %s
+  notify: reload sshd
+
+# --- Firewall (default deny incoming) ------------------------------------------
+# SSH is allowed before the policy flips so enabling ufw never locks us out.
+
+- name: Allow SSH through the firewall
+  community.general.ufw:
+    rule: allow
+    name: OpenSSH
+
+- name: Default-deny incoming, allow outgoing
+  community.general.ufw:
+    direction: "{{ item.direction }}"
+    policy: "{{ item.policy }}"
+  loop:
+    - { direction: incoming, policy: deny }
+    - { direction: outgoing, policy: allow }
+
+- name: Enable the firewall
+  community.general.ufw:
+    state: enabled
+
+# --- fail2ban ------------------------------------------------------------------
+
+- name: Configure the fail2ban sshd jail
+  ansible.builtin.template:
+    src: jail.local.j2
+    dest: /etc/fail2ban/jail.local
+    mode: "0644"
+  notify: restart fail2ban
+
+- name: Enable and start fail2ban
+  ansible.builtin.service:
+    name: fail2ban
+    enabled: true
+    state: started
+
+# --- Unattended security upgrades + time sync ----------------------------------
+
+- name: Enable unattended upgrades
+  ansible.builtin.copy:
+    dest: /etc/apt/apt.conf.d/20auto-upgrades
+    mode: "0644"
+    content: |
+      APT::Periodic::Update-Package-Lists "1";
+      APT::Periodic::Unattended-Upgrade "1";
+
+- name: Enable and start chrony
+  ansible.builtin.service:
+    name: chrony
+    enabled: true
+    state: started
+
+# --- Deploy directories --------------------------------------------------------
+
+- name: Create the scrabble base directories
+  ansible.builtin.file:
+    path: "{{ scrabble_base_dir }}/{{ item }}"
+    state: directory
+    owner: "{{ deploy_user }}"
+    group: "{{ deploy_user }}"
+    mode: "0750"
+  loop:
+    - ""
+    - config
+    - certs
+    - dumps
+    - images
diff --git a/deploy/ansible/roles/common/templates/daemon.json.j2 b/deploy/ansible/roles/common/templates/daemon.json.j2
new file mode 100644
index 0000000..50f7040
--- /dev/null
+++ b/deploy/ansible/roles/common/templates/daemon.json.j2
@@ -0,0 +1,8 @@
+{
+  "live-restore": true,
+  "log-driver": "json-file",
+  "log-opts": {
+    "max-size": "{{ docker_log_max_size }}",
+    "max-file": "{{ docker_log_max_file }}"
+  }
+}
diff --git a/deploy/ansible/roles/common/templates/jail.local.j2 b/deploy/ansible/roles/common/templates/jail.local.j2
new file mode 100644
index 0000000..e037958
--- /dev/null
+++ b/deploy/ansible/roles/common/templates/jail.local.j2
@@ -0,0 +1,9 @@
+# Managed by Ansible (deploy/ansible).
+[DEFAULT]
+bantime = 1h
+findtime = 10m
+maxretry = 5
+backend = systemd
+
+[sshd]
+enabled = true
diff --git a/deploy/ansible/roles/common/templates/sshd-hardening.conf.j2 b/deploy/ansible/roles/common/templates/sshd-hardening.conf.j2
new file mode 100644
index 0000000..7293298
--- /dev/null
+++ b/deploy/ansible/roles/common/templates/sshd-hardening.conf.j2
@@ -0,0 +1,6 @@
+# Managed by Ansible (deploy/ansible). Key-only authentication.
+# root stays reachable by key (prohibit-password) for provisioning re-runs.
+PasswordAuthentication no
+PermitRootLogin prohibit-password
+PubkeyAuthentication yes
+KbdInteractiveAuthentication no
diff --git a/deploy/ansible/roles/main/tasks/main.yml b/deploy/ansible/roles/main/tasks/main.yml
new file mode 100644
index 0000000..3cb7cd7
--- /dev/null
+++ b/deploy/ansible/roles/main/tasks/main.yml
@@ -0,0 +1,18 @@
+---
+# Main stack host: public web + bot-link ports and the external 'edge' network
+# the compose stack attaches caddy to.
+
+- name: Open public web and bot-link ports
+  community.general.ufw:
+    rule: allow
+    port: "{{ item }}"
+    proto: tcp
+  loop:
+    - "80"    # HTTP (ACME challenge + redirect to HTTPS)
+    - "443"   # HTTPS (caddy edge)
+    - "9443"  # bot-link mTLS (remote bot dials in; mutual TLS gates access)
+
+- name: Ensure the external 'edge' docker network exists
+  community.docker.docker_network:
+    name: edge
+    state: present
diff --git a/deploy/ansible/roles/tg/tasks/main.yml b/deploy/ansible/roles/tg/tasks/main.yml
new file mode 100644
index 0000000..ad62307
--- /dev/null
+++ b/deploy/ansible/roles/tg/tasks/main.yml
@@ -0,0 +1,19 @@
+---
+# Telegram bot host: holds no inbound port beyond SSH (the bot dials out to the
+# Bot API and into the main host's bot-link). We only verify direct Bot API
+# egress here, since the "no VPN" decision depends on it.
+
+- name: Verify direct Telegram Bot API egress (no VPN on this host)
+  ansible.builtin.uri:
+    url: https://api.telegram.org/
+    method: GET
+    status_code: [200, 301, 302, 401, 404]   # any HTTP reply proves reachability
+    timeout: 10
+  register: tg_egress
+  failed_when: false
+
+- name: Report Telegram reachability
+  ansible.builtin.debug:
+    msg: >-
+      api.telegram.org reachable:
+      {{ (tg_egress.status | default(0) | int) > 0 }} (status {{ tg_egress.status | default('none') }})
diff --git a/deploy/ansible/site.yml b/deploy/ansible/site.yml
new file mode 100644
index 0000000..a977ee8
--- /dev/null
+++ b/deploy/ansible/site.yml
@@ -0,0 +1,31 @@
+---
+# Stage 18 host provisioning. Idempotent: safe to re-run after a host resize.
+# Prepares hosts only (docker, hardening, service account, firewall); the
+# application is deployed separately by .gitea/workflows/prod-deploy.yaml.
+
+- name: Common baseline (both hosts)
+  hosts: prod
+  become: true
+  pre_tasks:
+    - name: Require a well-formed CI deploy public key
+      ansible.builtin.assert:
+        that:
+          - deploy_ci_pubkey | length > 0
+          - deploy_ci_pubkey is search('^(ssh|ecdsa)-')
+        fail_msg: >-
+          deploy_ci_pubkey is empty or malformed. Generate the key first
+          (see deploy/ansible/README.md) or override deploy_ci_pubkey_path.
+  roles:
+    - common
+
+- name: Main stack host
+  hosts: main
+  become: true
+  roles:
+    - main
+
+- name: Telegram bot host
+  hosts: tg
+  become: true
+  roles:
+    - tg