diff --git a/backend/internal/telemetry/middleware.go b/backend/internal/telemetry/middleware.go index 4485a5a..4a59b74 100644 --- a/backend/internal/telemetry/middleware.go +++ b/backend/internal/telemetry/middleware.go @@ -56,6 +56,9 @@ func Middleware(logger *zap.Logger) gin.HandlerFunc { zap.String("path", route), zap.Int("status", status), zap.Duration("latency", elapsed), + // The gateway forwards the real caller as X-Forwarded-For (and Caddy does for /_gm), which + // gin resolves here — so the access log carries the client IP, not the gateway's connection. + zap.String("client_ip", c.ClientIP()), } fields = append(fields, TraceFieldsFromContext(ctx)...) diff --git a/deploy/ansible/roles/common/tasks/main.yml b/deploy/ansible/roles/common/tasks/main.yml index a2da8ba..378b7b9 100644 --- a/deploy/ansible/roles/common/tasks/main.yml +++ b/deploy/ansible/roles/common/tasks/main.yml @@ -150,6 +150,13 @@ enabled: true state: started +# Pin every host to UTC so host-level timestamps (journald, file mtimes, cron) line up +# across the fleet — some VPS images ship a local zone (the tg host came up on MSK). The +# services themselves run in UTC regardless; this is about host-side log correlation. +- name: Set the system timezone to UTC + community.general.timezone: + name: Etc/UTC + # --- Swap file (OOM cushion) --------------------------------------------------- # The prod main host is tight (1.9 GiB) and the per-container memory caps # (docker-compose.prod.yml) sum to more than RAM, so a simultaneous spike could hit diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 25de66e..ff21a40 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -151,7 +151,10 @@ dropped). Horizontal scaling is explicit future work. and GCG are unaffected** (they stay decoded concrete characters, §9.1). - **gateway ↔ backend (sync)**: plain HTTP REST/JSON. The gateway injects `X-User-ID` (and the session's trusted `X-Platform`, §3) for authenticated - requests; `backend` never re-derives identity or platform from the body. Because every sync call targets the one backend host, the + requests, plus the caller's real IP as **`X-Forwarded-For`** on **every** call + (carried on the request context), so the backend records the real caller — the + account's last-login IP, chat/feedback moderation, the access log — rather than + the gateway's own connection address; `backend` never re-derives identity or platform from the body. Because every sync call targets the one backend host, the gateway's REST client widens its keep-alive pool well past the stdlib default of 2 idle connections per host; otherwise the per-request connection churn exhausts ephemeral ports and burns gateway CPU under load (see @@ -1496,7 +1499,8 @@ Two contours, two secret/variable prefixes (`TEST_` / `PROD_`): forwards the domain to `scrabble:80`, so the in-compose caddy serves plain HTTP (`CADDY_SITE_ADDRESS=:80`). The in-compose caddy **trusts X-Forwarded-For from private-range upstreams** (`trusted_proxies private_ranges`), so the real client IP — - used for chat-moderation logging and the gateway's per-IP rate limiting — survives the + used for the gateway's per-IP rate limiting and, forwarded on to the backend, the account's + last-login IP, chat/feedback moderation and the access log — survives the host-caddy hop; in prod (no host caddy) public clients are untrusted and Caddy uses the real peer, so the single config is correct and spoof-safe in both contours. The **bot-link mTLS material** (a private CA + gateway/bot leaves, CN=`gateway`) is diff --git a/gateway/internal/backendclient/client.go b/gateway/internal/backendclient/client.go index 9da6bfa..d66464e 100644 --- a/gateway/internal/backendclient/client.go +++ b/gateway/internal/backendclient/client.go @@ -139,6 +139,26 @@ func platformFromContext(ctx context.Context) string { return p } +// clientIPCtxKey types the request-context slot the originating client IP rides in. +type clientIPCtxKey struct{} + +// WithClientIP returns a copy of ctx carrying the originating client IP that do injects as +// X-Forwarded-For on every downstream backend request — so the backend records the real caller +// (the account's last-login IP, chat/feedback moderation, the access log) rather than the gateway's +// own connection. An empty IP leaves ctx unchanged, so no header is sent. +func WithClientIP(ctx context.Context, ip string) context.Context { + if ip == "" { + return ctx + } + return context.WithValue(ctx, clientIPCtxKey{}, ip) +} + +// clientIPFromContext returns the client IP stored by WithClientIP, or an empty string when none. +func clientIPFromContext(ctx context.Context) string { + ip, _ := ctx.Value(clientIPCtxKey{}).(string) + return ip +} + // do performs one REST call. userID, when non-empty, is forwarded as X-User-ID; // clientIP, when non-empty, as X-Forwarded-For (for chat moderation); the trusted // platform carried on ctx (see WithPlatform), when present, as X-Platform. A non-2xx @@ -160,6 +180,13 @@ func (c *Client) do(ctx context.Context, method, path, userID, clientIP string, if userID != "" { req.Header.Set("X-User-ID", userID) } + // The client IP rides X-Forwarded-For so the backend records the real caller. It comes from the + // explicit param (chat/feedback) or, for every other call, the request context (WithClientIP, set + // once per request in the Connect edge) — so the backend never falls back to the gateway's own + // connection address. + if clientIP == "" { + clientIP = clientIPFromContext(ctx) + } if clientIP != "" { req.Header.Set("X-Forwarded-For", clientIP) } diff --git a/gateway/internal/backendclient/client_test.go b/gateway/internal/backendclient/client_test.go index 68ea42b..e419303 100644 --- a/gateway/internal/backendclient/client_test.go +++ b/gateway/internal/backendclient/client_test.go @@ -122,3 +122,44 @@ func TestXPlatformInjection(t *testing.T) { } }) } + +// TestXForwardedForInjection verifies the client IP carried on the context (WithClientIP) rides every +// backend call as X-Forwarded-For — not just chat/feedback, which pass it explicitly — so the backend +// records the real caller (e.g. the account's last-login IP on the profile fetch) rather than the +// gateway's own connection. Absent when no client IP is set. +func TestXForwardedForInjection(t *testing.T) { + var gotXFF string + var hadHeader bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotXFF = r.Header.Get("X-Forwarded-For") + _, hadHeader = r.Header["X-Forwarded-For"] + _, _ = w.Write([]byte(`{}`)) + })) + defer srv.Close() + + c, err := backendclient.New(srv.URL, "localhost:9090", 2*time.Second) + if err != nil { + t.Fatalf("backendclient: %v", err) + } + defer func() { _ = c.Close() }() + + t.Run("client IP on ctx rides a non-chat call", func(t *testing.T) { + ctx := backendclient.WithClientIP(context.Background(), "203.0.113.7") + if _, err := c.Profile(ctx, "user-1"); err != nil { + t.Fatalf("Profile: %v", err) + } + if gotXFF != "203.0.113.7" { + t.Fatalf("X-Forwarded-For = %q, want 203.0.113.7", gotXFF) + } + }) + + t.Run("no client IP omits the header", func(t *testing.T) { + hadHeader = true + if _, err := c.Profile(context.Background(), "user-1"); err != nil { + t.Fatalf("Profile: %v", err) + } + if hadHeader { + t.Fatal("X-Forwarded-For must be absent when no client IP is set") + } + }) +} diff --git a/gateway/internal/connectsrv/server.go b/gateway/internal/connectsrv/server.go index b0826e6..d5b2cfd 100644 --- a/gateway/internal/connectsrv/server.go +++ b/gateway/internal/connectsrv/server.go @@ -418,6 +418,10 @@ func (s *Server) Execute(ctx context.Context, req *connect.Request[edgev1.Execut return nil, connect.NewError(connect.CodeNotFound, errUnknownMessageType(msgType)) } clientIP := peerIP(req.Peer().Addr, req.Header()) + // Carry the client IP on the context so the backend client injects it as X-Forwarded-For on every + // downstream REST call for this request — the account's last-login IP, chat/feedback moderation and + // the backend access log, not only the chat/feedback calls that pass it explicitly. + ctx = backendclient.WithClientIP(ctx, clientIP) tr := transcode.Request{Payload: req.Msg.GetPayload(), ClientIP: clientIP} if op.Auth {