From e922f5f3b9c574479570be99b0bc7e83deea2d25 Mon Sep 17 00:00:00 2001 From: Ilia Denisov Date: Thu, 9 Jul 2026 00:58:04 +0200 Subject: [PATCH] feat(deploy): continuous WAL archiving to S3 for point-in-time recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add pgBackRest-based PITR for the production database, shipped disarmed (archive_mode and the base-backup timer gated off) so it stays inert until the operator arms it before real payments — an un-armed deploy cannot pile WAL onto the disk. - Postgres now builds from a thin image carrying pgBackRest (archive_command runs in-process); the prod overlay wires an encrypted (AES-256-CBC), path-style S3 repository with 30-day retention and a daily full base-backup systemd timer. - Repository config/secrets flow through the PROD_PGBACKREST_* Gitea set and write-prod-env.sh; prod-rollback re-renders the same env so a rollback cannot disarm archiving. - Grafana alerts watch pg_stat_archiver (failing / stalled), absent-safe on the test contour, which never archives. - Fix the pre-migration pg_dump to snapshot the whole database (was backend-only, silently excluding payments). - Document the PITR runbook, the arming sequence and the restore drill in deploy/README.md; record the measured cost/perf assessment. --- .gitea/workflows/prod-deploy.yaml | 13 ++- .gitea/workflows/prod-rollback.yaml | 9 ++ PLAN.md | 65 ++++++++----- deploy/.env.example | 15 +++ deploy/README.md | 95 +++++++++++++++++-- deploy/ansible/README.md | 4 +- deploy/ansible/group_vars/all.yml | 9 ++ deploy/ansible/roles/main/tasks/main.yml | 27 ++++++ .../templates/pgbackrest-backup.service.j2 | 15 +++ .../main/templates/pgbackrest-backup.timer.j2 | 12 +++ deploy/docker-compose.prod.yml | 36 +++++++ deploy/docker-compose.yml | 8 +- .../grafana/provisioning/alerting/rules.yaml | 59 ++++++++++++ deploy/postgres/Dockerfile | 12 +++ deploy/prod-deploy.sh | 5 +- deploy/write-prod-env.sh | 11 +++ 16 files changed, 359 insertions(+), 36 deletions(-) create mode 100644 deploy/ansible/roles/main/templates/pgbackrest-backup.service.j2 create mode 100644 deploy/ansible/roles/main/templates/pgbackrest-backup.timer.j2 create mode 100644 deploy/postgres/Dockerfile diff --git a/.gitea/workflows/prod-deploy.yaml b/.gitea/workflows/prod-deploy.yaml index a7e0b3d..e851e5f 100644 --- a/.gitea/workflows/prod-deploy.yaml +++ b/.gitea/workflows/prod-deploy.yaml @@ -77,7 +77,7 @@ jobs: # The main-stack images via compose (reuses the build args, incl. VERSION); # the bot separately, since it is profiled out of the prod compose. docker compose -f docker-compose.yml -f docker-compose.prod.yml build - docker compose -f docker-compose.yml -f docker-compose.prod.yml push backend gateway landing validator renderer + docker compose -f docker-compose.yml -f docker-compose.prod.yml push postgres backend gateway landing validator renderer docker build -f ../platform/telegram/Dockerfile --target bot --build-arg VERSION="$TAG" -t "$REGISTRY/scrabble-telegram-bot:$TAG" .. docker push "$REGISTRY/scrabble-telegram-bot:$TAG" @@ -135,6 +135,17 @@ jobs: DICT_VERSION: ${{ vars.DICT_VERSION }} POSTGRES_DB: ${{ vars.PROD_POSTGRES_DB }} POSTGRES_USER: ${{ vars.PROD_POSTGRES_USER }} + # Point-in-time recovery (pgBackRest -> S3), prod main host only. Endpoint/bucket/ + # region + the archive-mode switch are variables; the S3 keys + the repository cipher + # passphrase are secrets. All empty/off until the operator arms archiving; flipping + # PROD_PGBACKREST_ARCHIVE_MODE=on activates it (deploy/README.md, arming). + PGBACKREST_ARCHIVE_MODE: ${{ vars.PROD_PGBACKREST_ARCHIVE_MODE }} + PGBACKREST_S3_ENDPOINT: ${{ vars.PROD_PGBACKREST_S3_ENDPOINT }} + PGBACKREST_S3_BUCKET: ${{ vars.PROD_PGBACKREST_S3_BUCKET }} + PGBACKREST_S3_REGION: ${{ vars.PROD_PGBACKREST_S3_REGION }} + PGBACKREST_S3_KEY: ${{ secrets.PROD_PGBACKREST_S3_KEY }} + PGBACKREST_S3_KEY_SECRET: ${{ secrets.PROD_PGBACKREST_S3_KEY_SECRET }} + PGBACKREST_CIPHER_PASS: ${{ secrets.PROD_PGBACKREST_CIPHER_PASS }} # TELEGRAM_MINIAPP_URL and GRAFANA_ROOT_URL are derived from PUBLIC_BASE_URL in # deploy/write-prod-env.sh, not stored variables. steps: diff --git a/.gitea/workflows/prod-rollback.yaml b/.gitea/workflows/prod-rollback.yaml index 9f1e71f..d846b7e 100644 --- a/.gitea/workflows/prod-rollback.yaml +++ b/.gitea/workflows/prod-rollback.yaml @@ -77,6 +77,15 @@ jobs: SERVICE_EMAIL: ${{ vars.PROD_SERVICE_EMAIL }} GRAFANA_SMTP_PORT: ${{ vars.GRAFANA_SMTP_PORT }} GF_SMTP_ENABLED: ${{ vars.PROD_GF_SMTP_ENABLED }} + # PITR archiving parity: a rollback must re-render the SAME env.sh, else it would + # silently disarm WAL archiving (PGBACKREST_ARCHIVE_MODE would fall back to off). + PGBACKREST_ARCHIVE_MODE: ${{ vars.PROD_PGBACKREST_ARCHIVE_MODE }} + PGBACKREST_S3_ENDPOINT: ${{ vars.PROD_PGBACKREST_S3_ENDPOINT }} + PGBACKREST_S3_BUCKET: ${{ vars.PROD_PGBACKREST_S3_BUCKET }} + PGBACKREST_S3_REGION: ${{ vars.PROD_PGBACKREST_S3_REGION }} + PGBACKREST_S3_KEY: ${{ secrets.PROD_PGBACKREST_S3_KEY }} + PGBACKREST_S3_KEY_SECRET: ${{ secrets.PROD_PGBACKREST_S3_KEY_SECRET }} + PGBACKREST_CIPHER_PASS: ${{ secrets.PROD_PGBACKREST_CIPHER_PASS }} INPUT_TARGET: ${{ inputs.target_version }} steps: - uses: actions/checkout@v4 diff --git a/PLAN.md b/PLAN.md index f345161..0e22e08 100644 --- a/PLAN.md +++ b/PLAN.md @@ -33,7 +33,7 @@ status — without re-deriving decisions. | E1 | Trusted platform signal | 1 | DONE | | E2 | Currency + benefit core | 1 | DONE | | E3 | Wallet UI | 1 | DONE | -| E4 | Durability (PITR) | 2 | TODO | +| E4 | Durability (PITR) | 2 | WIP | | E5 | Payment intake | 2 | TODO | | E6 | Ads | 2 | TODO | | E7 | Admin & reports | 2 | TODO | @@ -446,40 +446,57 @@ noun agreement). Svelte whitespace/`$state` naming gotchas apply. ## E4 — Durability (PITR) -**Status:** TODO · **Release 2** · depends on: E0 (schema exists) · mechanics: PAYMENTS §14. +**Status:** WIP — repo artifacts landed; **prod arming pending** (owner-coordinated, before +E5). · **Release 2** · depends on: E0 (schema exists) · mechanics: PAYMENTS §14 (D4). **Goal.** Continuous WAL archiving with point-in-time recovery, armed **before the first real money** is accepted (E5 prod). Protects both money and game data. -**Work.** +**Locked decisions (this stage).** -- Add WAL archiving to the prod Postgres (pgBackRest or WAL-G): base backups + continuous - WAL to a second host or object storage. Wire into `deploy/` (compose/prod overlay + - ansible `deploy/ansible/`), config + secrets under the `PROD_` prefix. -- Restore runbook in `deploy/README.md`: base + WAL replay to a timestamp; test the restore - on a scratch target. -- Keep the existing migration-time `pg_dump` (`deploy/prod-deploy.sh`) as belt-and-braces. +- **Tool: pgBackRest.** **Destination: Selectel S3** object storage (encrypted AES-256-CBC, + path-style addressing), **prod main host only** — the test contour never archives. (D4 left + the tool + destination open; resolved here. Warm replica stays deferred, per D4.) +- **Retention 30 days**, **daily full base backup** (a systemd timer) + continuous WAL + (`archive_command`, a 5-minute forced switch bounds the recovery point). Assessment + (owner-reviewed prod gate): measured prod WAL **~0.77 MB/day**, DB **~9.6 MB** → archive + **< 1 ₽/month** on Selectel S3, **negligible** perf impact. Recorded in `deploy/README.md`. +- **Archiving ships gated OFF** (`PGBACKREST_ARCHIVE_MODE` on the DB, `pitr_enabled` for the + timer — both default off), so the merged/redeployed artifact is inert until armed and an + un-armed prod deploy **cannot** pile WAL onto the disk. -**Mandatory pre-prod assessment (owner-required, gates the Release 2 prod rollout).** -Before the first money goes live, produce and record here: +**Work (landed in the artifacts PR into `development`).** -1. **Disk-storage cost estimate** for the WAL archive + base backups (retention window × - WAL volume; account for both hosts / object-storage pricing). This can change hosting - sizing. -2. **PG performance-degradation assessment** from continuous archiving (write amplification, - archive_command latency, backup I/O contention) on the prod-sized instance. +- pgBackRest in the DB image (`deploy/postgres/Dockerfile`); postgres becomes a built + pushed + image (prod overlay `image:` + `prod-deploy.yaml` build/push list). +- Repository config + `archive_mode` via `PGBACKREST_*` env on the prod-overlay postgres + service; secrets/vars rendered by `deploy/write-prod-env.sh` from the `PROD_PGBACKREST_*` + Gitea set; **parity added to `prod-rollback.yaml`** (a rollback must not disarm archiving). +- Daily base-backup systemd timer (Ansible `main` role, gated by `pitr_enabled`). +- Two Grafana alerts on the exporter's `pg_stat_archiver` metrics (failing / stalled), + absent/NaN-safe on the contour. +- Belt-and-braces `pg_dump` fixed to dump the **whole DB** (`deploy/prod-deploy.sh` was + `-n backend`, silently excluding `payments`); manual-restore runbook updated. +- Full PITR runbook + arming sequence + recorded assessment in `deploy/README.md`. -Neither blocks building the plan, but both **must** be completed and reviewed before the -prod release — surface the numbers to the owner (they may change host settings). +**Prod arming (SEPARATE — owner-coordinated, before E5; NOT the artifacts PR).** Owner creates +the Selectel S3 bucket + the `PROD_PGBACKREST_*` secrets/variables (incl. `ARCHIVE_MODE=on`); +then promote `development → master` → `prod-deploy` (archive_mode on behind the maintenance +window), `pgbackrest stanza-create` + first base backup + `check`, re-run Ansible with +`-e pitr_enabled=true`, and run the restore drill on an isolated one-shot target. Exact steps: +`deploy/README.md` (point-in-time recovery — arming). -**Tests.** Restore drill on a scratch instance (base + WAL → target timestamp) documented -and passing. No app-level tests. +**Tests.** Restore drill on an isolated one-shot instance (base + WAL → target timestamp), +recorded in `deploy/README.md`. No app-level tests. Local verification: `docker compose config` +valid + the custom PG image builds (archiving inert on the contour). -**Done-criteria.** WAL archiving live on prod PG; a timed restore verified; cost + perf -assessment recorded and owner-reviewed; runbook in `deploy/README.md`. +**Done-criteria.** Artifacts merged with archiving inert. **Fully done** at arming: WAL +archiving live on prod PG; a timed restore verified; cost + perf assessment reviewed; runbook +current in `deploy/README.md`. -**Notes/risks.** The tg host is weak (1 vCPU) — if the archive lands there, watch -contention. Migrations stay expand-contract so image rollback remains DB-safe alongside PITR. +**Notes/risks.** The repository **cipher passphrase is unrecoverable if lost** — stored apart +from the S3 keys. Enabling `archive_mode` restarts postgres (rides the prod-deploy maintenance +window). Migrations stay expand-contract so image rollback remains DB-safe alongside PITR. --- diff --git a/deploy/.env.example b/deploy/.env.example index 96bf39b..c757459 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -13,6 +13,21 @@ POSTGRES_DB=scrabble POSTGRES_USER=scrabble POSTGRES_PASSWORD=change-me # required +# --- Point-in-time recovery (pgBackRest -> S3; PROD main host only) ---------- +# Continuous WAL archiving for PITR. PROD-ONLY: the test contour never archives (these +# stay unset there and archive_mode stays off). The artifact ships DISARMED — +# PGBACKREST_ARCHIVE_MODE defaults off; arm it only after setting up the S3 repository + +# secrets and creating the stanza, then flip it on (deploy/README.md, point-in-time +# recovery). Gitea: PROD_PGBACKREST_* — endpoint/bucket/region/archive-mode are variables, +# the two S3 keys + the cipher passphrase are secrets. +PGBACKREST_ARCHIVE_MODE=off # on = archiving active (arm only after the stanza exists) +PGBACKREST_S3_ENDPOINT= # e.g. s3.ru-1.storage.selcloud.ru (Selectel S3, path-style addressing) +PGBACKREST_S3_BUCKET= # bucket name; lowercase, no dots/underscores +PGBACKREST_S3_REGION= # e.g. ru-1 +PGBACKREST_S3_KEY= # secret: S3 access key id +PGBACKREST_S3_KEY_SECRET= # secret: S3 secret access key +PGBACKREST_CIPHER_PASS= # secret: repo encryption passphrase — KEEP SAFE, stored apart from the S3 keys (losing it makes the archive unrecoverable) + # --- Dictionary ------------------------------------------------------------- # scrabble-dictionary release tag baked into the image as the SEED dictionary for a # FRESH volume (image build-arg; also labels the resident seed version). After first diff --git a/deploy/README.md b/deploy/README.md index fd09f33..93e4733 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -220,10 +220,86 @@ release tag, so any prior release is reachable. **Migrations** must be **expand-contract** (backward-compatible; goose is forward-only): the automatic rollback is image-only and never restores the DB. A deploy that changes `backend/internal/postgres/migrations/` opens a maintenance window — the backend (sole -writer) is stopped for a consistent `pg_dump` into `/opt/scrabble/dumps` before the new -backend migrates. **Manual DB restore** (only if a migration was destructive): -`docker exec -i scrabble-postgres psql -U scrabble -d scrabble -c 'DROP SCHEMA backend CASCADE'`, -then pipe the dump into the same `psql`, and redeploy the matching old tag. +writer) is stopped for a consistent **whole-database** `pg_dump` into `/opt/scrabble/dumps` +(it covers `backend` **and** `payments`) before the new backend migrates. **Manual DB +restore** (only if a migration was destructive): drop the app schemas in the same instance +and pipe the dump back — +`docker exec -i scrabble-postgres psql -U scrabble -d scrabble -c 'DROP SCHEMA IF EXISTS backend CASCADE; DROP SCHEMA IF EXISTS payments CASCADE;'`, +then `docker exec -i scrabble-postgres psql -U scrabble -d scrabble < the-dump.sql`, and +redeploy the matching old tag. This dump is a belt-and-braces net for a bad migration; +**point-in-time recovery** (below) is the primary recovery path once armed, and the only one +that survives losing the host. + +## Point-in-time recovery (PITR) + +The main host archives Postgres continuously with **pgBackRest** to **Selectel S3** +(encrypted at rest, path-style addressing) so the database can be restored to any moment — +protecting the money ledger and the game state against corruption or host loss. It is the +primary recovery path; the migration-window `pg_dump` above is the secondary net. + +**Shape.** A daily full **base backup** (a systemd timer on the main host, `04:00`) plus +**continuous WAL** archived by Postgres `archive_command` (a segment is forced at least every +5 minutes, so the recovery point is never more than a few minutes behind). Retention is **30 +days** (`repo1-retention-full=30`); the repository is AES-256-CBC encrypted. This is main-host +only — the test contour never archives. + +**Wiring.** pgBackRest ships inside the DB image (`deploy/postgres/Dockerfile`); the +repository + S3 credentials + cipher are the `PGBACKREST_*` environment on the postgres +service in `docker-compose.prod.yml`, rendered from the `PROD_` Gitea set by +`write-prod-env.sh`. Archiving is gated by **`PGBACKREST_ARCHIVE_MODE` (default `off`)**, so +shipping or redeploying this stack does **not** start archiving — the artifact is inert until +armed, which is why an un-armed prod deploy can never pile WAL onto the disk. The base-backup +timer is provisioned by the Ansible `main` role behind `pitr_enabled` (also default off). Two +Grafana alerts watch health: `WAL archiving failing` (`pg_stat_archiver_failed_count` rising) +and `WAL archiving stalled` (`pg_stat_archiver_last_archive_age` over 30 min) — both +absent/NaN-safe, so they stay quiet until archiving is armed. + +**Assessment (owner-reviewed; the gate before the first real money).** Measured on prod +`pg_stat_wal`: WAL is generated at **~0.77 MB/day** and the database is **~9.6 MB**. At +30-day retention on Selectel S3 (~2 ₽/GB·month) the archive is **under ~0.3 GB → well under +1 ₽/month** (compression halves it again); request volume is trivial. Performance impact is +**negligible**: archive-push moves tiny compressed segments, and the daily full base backup +is a ~10 MB, sub-second job on the 2 vCPU host. Revisit both if traffic grows ~100× (watch +`node_exporter` during a base backup). + +**Arming (owner-coordinated, once, before real payments).** Ships disarmed; to turn it on: + +1. **Owner (Selectel + Gitea):** create an S3 bucket (name **lowercase, no dots/underscores**) + and an S3 access key/secret; pick a repository **cipher passphrase** and store it **apart + from the S3 keys** (losing it makes the archive unrecoverable). Set the Gitea `PROD_` set — + variables `PROD_PGBACKREST_S3_ENDPOINT` / `_S3_BUCKET` / `_S3_REGION` and + `PROD_PGBACKREST_ARCHIVE_MODE=on`; secrets `PROD_PGBACKREST_S3_KEY` / `_S3_KEY_SECRET` / + `_CIPHER_PASS`. +2. Promote `development → master`, tag, and run **prod-deploy**. The roll recreates postgres + with `archive_mode=on` behind the maintenance page. (Archive pushes fail harmlessly for the + minute until step 3 creates the repository — the WAL is retained, not lost.) +3. On the main host, create the repository, take the first base backup, and verify: + ```sh + docker exec scrabble-postgres pgbackrest --stanza=scrabble stanza-create + docker exec scrabble-postgres pgbackrest --stanza=scrabble --type=full backup + docker exec scrabble-postgres pgbackrest --stanza=scrabble check + ``` +4. Enable the daily timer: `ansible-playbook site.yml -e pitr_enabled=true` (installs + starts + `pgbackrest-backup.timer` on the main host). +5. Confirm in Grafana that the two archiving alerts are green (`last_archive_age` now tracks a + real number, `failed_count` flat). + +**Restore drill (proves recoverability; re-run after arming and after any major change).** On +an **isolated, one-shot** target (a throwaway VM or a container with no ingress), pgBackRest, +the matching Postgres major, an empty `PGDATA`, and the same `PGBACKREST_*` environment: + +```sh +pgbackrest --stanza=scrabble --type=time "--target=YYYY-MM-DD HH:MM:SS+00" --delta restore +# start Postgres; it replays WAL to the target; then verify a known row / the ledger tail +``` + +The target holds **real money + personal data** while it exists — keep it network-isolated and +**destroy it (wipe `PGDATA` + the instance) afterwards**. Record each drill (date, target +timestamp, outcome) here: + +| Date | Target timestamp | Result | +| --- | --- | --- | +| _pending first arming_ | — | — | **bot-link cert rotation:** regenerate (`deploy/gen-certs.sh /tmp/c --force`), reset the five `PROD_BOTLINK_*` secrets from `/tmp/c`, and re-run the workflow — both hosts redeploy @@ -249,14 +325,17 @@ VITE_VK_APP_LINK, VITE_VK_APP_ID`. **Derived from `PUBLIC_BASE_URL` at deploy** `PROD_{POSTGRES_PASSWORD, GM_BASICAUTH_HASH, GRAFANA_ADMIN_PASSWORD, TELEGRAM_BOT_TOKEN, TELEGRAM_PROMO_BOT_TOKEN, EXPORT_SIGN_KEY, GATEWAY_HONEYTOKEN, REGISTRY_PASSWORD, SSH_KEY, SSH_KNOWN_HOSTS, BOTLINK_CA, BOTLINK_GATEWAY_CERT, BOTLINK_GATEWAY_KEY, BOTLINK_BOT_CERT, -BOTLINK_BOT_KEY}`; variables: +BOTLINK_BOT_KEY, PGBACKREST_S3_KEY, PGBACKREST_S3_KEY_SECRET, PGBACKREST_CIPHER_PASS}`; +variables: `PROD_{REGISTRY_USER, MAIN_HOST, TG_HOST, CADDY_SITE_ADDRESS, GM_BASICAUTH_USER, LOG_LEVEL, TELEGRAM_GAME_CHANNEL_ID, TELEGRAM_CHAT_ID, TELEGRAM_SUPPORT_CHAT_ID, TELEGRAM_BOT_USERNAME, VITE_TELEGRAM_BOT_ID, VITE_TELEGRAM_LINK, VITE_TELEGRAM_GAME_CHANNEL_NAME, SMTP_RELAY_FROM, PUBLIC_BASE_URL, SMTP_RELAY_ADMIN_FROM, ADMIN_EMAIL, SMTP_RELAY_SERVICE_FROM, SERVICE_EMAIL, -GF_SMTP_ENABLED}`. The test contour uses the same names under `TEST_`, minus the prod-only -infra (`MAIN_HOST`/`TG_HOST`/`REGISTRY_*`/`SSH_*`/`BOTLINK_*`, which the test deploy generates -or runs locally) and plus `TEST_AWG_CONF` (the bot's VPN egress). +GF_SMTP_ENABLED, PGBACKREST_S3_ENDPOINT, PGBACKREST_S3_BUCKET, PGBACKREST_S3_REGION, +PGBACKREST_ARCHIVE_MODE}`. The test contour uses the same names under `TEST_`, minus the +prod-only infra (`MAIN_HOST`/`TG_HOST`/`REGISTRY_*`/`SSH_*`/`BOTLINK_*` and the `PGBACKREST_*` +PITR set, which is prod-only — the test contour never archives) and plus `TEST_AWG_CONF` (the +bot's VPN egress). ## Host-side setup (outside this repo) diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index 9e25cce..4b0eeb2 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -45,5 +45,7 @@ safe (idempotent) and survives a host resize. 10m×3 log rotation), `deploy` user (docker group, no sudo), key-only sshd, `ufw` default-deny incoming + allow SSH, fail2ban sshd jail, unattended upgrades, chrony, `/opt/scrabble/{config,certs,dumps,images}`. -- **main**: `ufw` opens 80/443/9443; the external `edge` docker network. +- **main**: `ufw` opens 80/443/9443; the external `edge` docker network; the pgBackRest + daily base-backup systemd timer — installed only when `pitr_enabled=true` (off by default; + turned on as part of arming point-in-time recovery, see [`../README.md`](../README.md)). - **tg**: verifies direct `api.telegram.org` egress (the no-VPN assumption). diff --git a/deploy/ansible/group_vars/all.yml b/deploy/ansible/group_vars/all.yml index 0222db7..bd46535 100644 --- a/deploy/ansible/group_vars/all.yml +++ b/deploy/ansible/group_vars/all.yml @@ -27,3 +27,12 @@ docker_log_max_file: "3" # path (a slow service beats a killed one). Set swap_size to "0" to skip provisioning. swap_size: "1G" swap_swappiness: 10 + +# Point-in-time recovery (pgBackRest). The base-backup systemd timer on the main host is +# gated by pitr_enabled so provisioning stays inert until archiving is armed — the timer +# would fail nightly if it ran before the S3 repository + stanza exist. Arm it as part of +# the PITR rollout (deploy/README.md): flip it on with `-e pitr_enabled=true` (or a main +# host_var) once the stanza is created. Only the `main` role reads these. +pitr_enabled: false +# systemd OnCalendar for the daily full base backup — a low-traffic hour on the main host. +pitr_backup_oncalendar: "*-*-* 04:00:00" diff --git a/deploy/ansible/roles/main/tasks/main.yml b/deploy/ansible/roles/main/tasks/main.yml index 3cb7cd7..40ee5c7 100644 --- a/deploy/ansible/roles/main/tasks/main.yml +++ b/deploy/ansible/roles/main/tasks/main.yml @@ -16,3 +16,30 @@ community.docker.docker_network: name: edge state: present + +# --- pgBackRest base-backup schedule (point-in-time recovery) ------------------ +# A daily full base backup via a systemd timer that runs pgBackRest inside the postgres +# container; archive_command handles the continuous WAL between backups. Gated by +# pitr_enabled so the schedule stays inert until archiving is armed (the S3 repository + +# stanza must exist first, or the run fails nightly). Arm per deploy/README.md. +- name: Install the pgBackRest base-backup service + ansible.builtin.template: + src: pgbackrest-backup.service.j2 + dest: /etc/systemd/system/pgbackrest-backup.service + mode: "0644" + when: pitr_enabled | default(false) | bool + +- name: Install the pgBackRest base-backup timer + ansible.builtin.template: + src: pgbackrest-backup.timer.j2 + dest: /etc/systemd/system/pgbackrest-backup.timer + mode: "0644" + when: pitr_enabled | default(false) | bool + +- name: Enable and start the pgBackRest base-backup timer + ansible.builtin.systemd: + name: pgbackrest-backup.timer + enabled: true + state: started + daemon_reload: true + when: pitr_enabled | default(false) | bool diff --git a/deploy/ansible/roles/main/templates/pgbackrest-backup.service.j2 b/deploy/ansible/roles/main/templates/pgbackrest-backup.service.j2 new file mode 100644 index 0000000..7565a09 --- /dev/null +++ b/deploy/ansible/roles/main/templates/pgbackrest-backup.service.j2 @@ -0,0 +1,15 @@ +# Managed by Ansible (deploy/ansible). +# Daily full pgBackRest base backup for the payments/game database. Runs pgBackRest +# inside the postgres container (where the repository configuration and PGDATA live); +# the continuous WAL between base backups is handled by the container's archive_command. +# Installed only when pitr_enabled is set (see deploy/ansible/group_vars/all.yml). +[Unit] +Description=pgBackRest full base backup (scrabble) +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +# docker exec runs as the image's postgres user and inherits the container's PGBACKREST_* +# environment (the S3 repository + cipher), so no repository config is needed on the host. +ExecStart=/usr/bin/docker exec scrabble-postgres pgbackrest --stanza=scrabble --type=full backup diff --git a/deploy/ansible/roles/main/templates/pgbackrest-backup.timer.j2 b/deploy/ansible/roles/main/templates/pgbackrest-backup.timer.j2 new file mode 100644 index 0000000..4f0a8f8 --- /dev/null +++ b/deploy/ansible/roles/main/templates/pgbackrest-backup.timer.j2 @@ -0,0 +1,12 @@ +# Managed by Ansible (deploy/ansible). +# Fires the daily full base backup. Persistent=true catches up a run missed while the +# host was down. Installed only when pitr_enabled is set. +[Unit] +Description=Daily pgBackRest full base backup (scrabble) + +[Timer] +OnCalendar={{ pitr_backup_oncalendar }} +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/deploy/docker-compose.prod.yml b/deploy/docker-compose.prod.yml index 434eaec..2aa393e 100644 --- a/deploy/docker-compose.prod.yml +++ b/deploy/docker-compose.prod.yml @@ -45,6 +45,42 @@ services: memory: 384M postgres: + image: ${REGISTRY:?set REGISTRY}/scrabble-postgres:${TAG:?set TAG} + # Continuous WAL archiving to S3 via pgBackRest (point-in-time recovery), on the prod + # main host only. archive_mode is gated by PGBACKREST_ARCHIVE_MODE (default "off"), so + # merging/redeploying this stack does NOT start archiving until the operator arms it: + # archiving stays inert — and cannot pile WAL onto the disk — until the S3 repository is + # set up, the stanza is created and the switch is flipped on. See deploy/README.md + # (point-in-time recovery — arming). Non-secret settings are inline; the endpoint, + # bucket, region, S3 keys and repository cipher passphrase come from the prod env.sh + # (deploy/write-prod-env.sh, PROD_ secrets/variables). + environment: + PGBACKREST_STANZA: scrabble + PGBACKREST_PG1_PATH: /var/lib/postgresql/data + PGBACKREST_REPO1_TYPE: s3 + PGBACKREST_REPO1_PATH: /pgbackrest + PGBACKREST_REPO1_S3_URI_STYLE: path + PGBACKREST_REPO1_S3_ENDPOINT: ${PGBACKREST_S3_ENDPOINT:-} + PGBACKREST_REPO1_S3_BUCKET: ${PGBACKREST_S3_BUCKET:-} + PGBACKREST_REPO1_S3_REGION: ${PGBACKREST_S3_REGION:-} + PGBACKREST_REPO1_S3_KEY: ${PGBACKREST_S3_KEY:-} + PGBACKREST_REPO1_S3_KEY_SECRET: ${PGBACKREST_S3_KEY_SECRET:-} + PGBACKREST_REPO1_CIPHER_TYPE: aes-256-cbc + PGBACKREST_REPO1_CIPHER_PASS: ${PGBACKREST_CIPHER_PASS:-} + PGBACKREST_REPO1_RETENTION_FULL: "30" + PGBACKREST_COMPRESS_TYPE: zst + PGBACKREST_LOG_LEVEL_CONSOLE: info + PGBACKREST_LOG_LEVEL_FILE: "off" + # archive_command is inert while archive_mode is off, so it is always present and only + # the mode switches. A forced 5-minute segment switch bounds the recovery point. + command: + - postgres + - -c + - archive_mode=${PGBACKREST_ARCHIVE_MODE:-off} + - -c + - "archive_command=pgbackrest --stanza=scrabble archive-push %p" + - -c + - archive_timeout=300 deploy: resources: limits: diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index f6d25c8..919fee2 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -38,7 +38,13 @@ x-logging: &default-logging services: postgres: container_name: scrabble-postgres - image: postgres:17-alpine + # Built from postgres:17-alpine + pgBackRest (deploy/postgres/Dockerfile) so the prod + # main host can archive WAL for point-in-time recovery. Archiving is switched on only by + # the prod overlay (docker-compose.prod.yml); on this contour the extra binary is present + # but idle and this is a plain postgres. See deploy/README.md (point-in-time recovery). + image: scrabble-postgres:latest + build: + context: postgres restart: unless-stopped logging: *default-logging environment: diff --git a/deploy/grafana/provisioning/alerting/rules.yaml b/deploy/grafana/provisioning/alerting/rules.yaml index a63c62b..c56aae6 100644 --- a/deploy/grafana/provisioning/alerting/rules.yaml +++ b/deploy/grafana/provisioning/alerting/rules.yaml @@ -212,3 +212,62 @@ groups: - evaluator: { type: gt, params: [0.8] } labels: { severity: warning } annotations: { summary: 'Postgres using over 80% of max_connections.' } + + # Continuous WAL archiving health (pgBackRest -> S3, point-in-time recovery). Both rules + # read the postgres_exporter's built-in pg_stat_archiver metrics and are absent/NaN-safe: + # on the test contour (and any host before archiving is armed) archive_mode is off, so + # failed_count stays 0 and last_archive_age is NaN — neither threshold trips. + - orgId: 1 + name: scrabble-backup + folder: Alerts + interval: 1m + rules: + - uid: pg_archive_failing + title: WAL archiving failing + condition: C + for: 15m + noDataState: OK + execErrState: OK + data: + - refId: A + relativeTimeRange: { from: 900, to: 0 } + datasourceUid: prometheus + model: + refId: A + expr: increase(pg_stat_archiver_failed_count[15m]) + instant: true + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: { type: gt, params: [0] } + labels: { severity: critical } + annotations: { summary: 'pgBackRest archive_command is failing — PITR is degrading and pg_wal can fill the disk. Run `docker exec scrabble-postgres pgbackrest --stanza=scrabble check`.' } + + - uid: pg_archive_stalled + title: WAL archiving stalled + condition: C + for: 15m + noDataState: OK + execErrState: OK + data: + - refId: A + relativeTimeRange: { from: 300, to: 0 } + datasourceUid: prometheus + model: + refId: A + expr: pg_stat_archiver_last_archive_age + instant: true + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: { type: gt, params: [1800] } + labels: { severity: critical } + annotations: { summary: 'No WAL segment archived for over 30 minutes (archive_timeout is 5m) — archiving is stalled; the PITR recovery point is frozen and pg_wal may grow.' } diff --git a/deploy/postgres/Dockerfile b/deploy/postgres/Dockerfile new file mode 100644 index 0000000..8de0e71 --- /dev/null +++ b/deploy/postgres/Dockerfile @@ -0,0 +1,12 @@ +# Postgres for the Scrabble contour, extended with pgBackRest for continuous WAL +# archiving and point-in-time recovery. pgBackRest must live in the database container +# because Postgres runs archive_command in-process; the scheduled base backups and the +# manual restore drills invoke the same binary through `docker exec`, inheriting the +# repository configuration from the container environment. +# +# Archiving is enabled only on the production main host, where the prod overlay +# (docker-compose.prod.yml) sets archive_mode and supplies the S3 repository. On the test +# contour this image behaves exactly as a plain postgres:17-alpine. See deploy/README.md +# (point-in-time recovery). +FROM postgres:17-alpine +RUN apk add --no-cache pgbackrest diff --git a/deploy/prod-deploy.sh b/deploy/prod-deploy.sh index d26a65c..0bb8a59 100755 --- a/deploy/prod-deploy.sh +++ b/deploy/prod-deploy.sh @@ -140,7 +140,10 @@ if [ "$MIGRATION" = 1 ]; then echo "migration deploy: opening maintenance window (stopping the backend = the only writer)" dc stop backend dump="$DUMP_DIR/pre-$TAG-$(date +%Y%m%d-%H%M%S).sql" - if ! docker exec scrabble-postgres pg_dump -U "$PG_USER" -d "$PG_DB" -n backend > "$dump"; then + # Dump the WHOLE database (no -n): the payments schema — and any schema added later — + # must be in the belt-and-braces snapshot, not just backend. Restored manually into the + # same instance only if a migration turned out destructive (see deploy/README.md). + if ! docker exec scrabble-postgres pg_dump -U "$PG_USER" -d "$PG_DB" > "$dump"; then echo "pg_dump failed; restarting the old backend and aborting" dc start backend exit 1 diff --git a/deploy/write-prod-env.sh b/deploy/write-prod-env.sh index d0460cb..5ce91cf 100755 --- a/deploy/write-prod-env.sh +++ b/deploy/write-prod-env.sh @@ -73,4 +73,15 @@ export GF_SMTP_ENABLED='$GF_SMTP_ENABLED' export PUBLIC_BASE_URL='$PUBLIC_BASE_URL' export GATEWAY_HONEYTOKEN='$GATEWAY_HONEYTOKEN' export GATEWAY_ABUSE_BAN_ENABLED='true' +# Continuous WAL archiving (pgBackRest -> S3) for point-in-time recovery. The artifact +# ships disarmed: PGBACKREST_ARCHIVE_MODE defaults off, so archiving stays inert until the +# operator sets the S3 repository values + secrets, creates the stanza and flips the switch +# on (deploy/README.md). Empty repository values are harmless while the mode is off. +export PGBACKREST_ARCHIVE_MODE='${PGBACKREST_ARCHIVE_MODE:-off}' +export PGBACKREST_S3_ENDPOINT='$PGBACKREST_S3_ENDPOINT' +export PGBACKREST_S3_BUCKET='$PGBACKREST_S3_BUCKET' +export PGBACKREST_S3_REGION='$PGBACKREST_S3_REGION' +export PGBACKREST_S3_KEY='$PGBACKREST_S3_KEY' +export PGBACKREST_S3_KEY_SECRET='$PGBACKREST_S3_KEY_SECRET' +export PGBACKREST_CIPHER_PASS='$PGBACKREST_CIPHER_PASS' EOF