feat(telemetry): local move-preview adoption metrics (Phase 4) #155

Merged
developer merged 1 commits from feature/local-eval-telemetry into development 2026-07-01 21:39:59 +00:00
12 changed files with 254 additions and 3 deletions
+1 -1
View File
@@ -53,7 +53,7 @@
# The game SPA and the Connect edge are served by the gateway. Strip any # The game SPA and the Connect edge are served by the gateway. Strip any
# client-supplied X-Scrabble-Honeypot here so the gateway only ever honours the # client-supplied X-Scrabble-Honeypot here so the gateway only ever honours the
# tag the honeypot block sets below (a client cannot self-tag a real request). # tag the honeypot block sets below (a client cannot self-tag a real request).
@gateway path /app /app/* /telegram /telegram/* /vk /vk/* /dict/* /scrabble.edge.v1.Gateway/* @gateway path /app /app/* /telegram /telegram/* /vk /vk/* /dict/* /metrics/* /scrabble.edge.v1.Gateway/*
handle @gateway { handle @gateway {
reverse_proxy gateway:8081 { reverse_proxy gateway:8081 {
header_up -X-Scrabble-Honeypot header_up -X-Scrabble-Honeypot
+28 -1
View File
@@ -4,7 +4,7 @@
"tags": ["scrabble"], "tags": ["scrabble"],
"timezone": "", "timezone": "",
"schemaVersion": 39, "schemaVersion": 39,
"version": 1, "version": 2,
"refresh": "30s", "refresh": "30s",
"time": { "from": "now-7d", "to": "now" }, "time": { "from": "now-7d", "to": "now" },
"panels": [ "panels": [
@@ -29,6 +29,33 @@
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 8 }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 8 },
"datasource": { "type": "prometheus", "uid": "prometheus" }, "datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "A", "expr": "sum(accounts_created_total) by (kind)", "legendFormat": "{{kind}}" }] "targets": [{ "refId": "A", "expr": "sum(accounts_created_total) by (kind)", "legendFormat": "{{kind}}" }]
},
{
"type": "timeseries",
"title": "App opens vs dictionary fills (cumulative)",
"description": "Local move-preview adoption. App opens are client cold starts; dictionary fills are IndexedDB downloads of a dawg. The gap between them is how many launches reuse an already-cached dictionary. Client-reported, best-effort (a batched beacon), so slightly lossy.",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{ "refId": "A", "expr": "sum(local_eval_cold_start_total)", "legendFormat": "app opens (cold start)" },
{ "refId": "B", "expr": "sum(local_eval_dict_load_total{result=\"fetched\"})", "legendFormat": "dictionary fills (IndexedDB)" }
]
},
{
"type": "timeseries",
"title": "Dictionary loads by result (rate)",
"description": "Client dictionary loads for the local preview, by tier: fetched (network download), cache_hit (IndexedDB), miss (a warm-up that failed or timed out — trips the bad-connection breaker).",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "A", "expr": "sum(rate(local_eval_dict_load_total[1h])) by (result)", "legendFormat": "{{result}}" }]
},
{
"type": "timeseries",
"title": "Move previews by path (rate)",
"description": "Where the move preview is computed: local (on-device, the accelerator) vs network (the fallback to game.evaluate). The local share is the backend load shed.",
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 24 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "A", "expr": "sum(rate(local_eval_preview_total[1h])) by (path)", "legendFormat": "{{path}}" }]
} }
] ]
} }
+14
View File
@@ -955,6 +955,20 @@ edits take effect on the next `profile.get` (open/reconnect/foreground), not mid
distinct accounts that performed an authenticated edge action in the window. The distinct accounts that performed an authenticated edge action in the window. The
gauge is single-process by design (single-instance MVP, §10): it is correct for one gauge is single-process by design (single-instance MVP, §10): it is correct for one
gateway, resets on restart, and is a live operational figure, not a billing count. gateway, resets on restart, and is a live operational figure, not a billing count.
- **Local-preview adoption (client-reported):** three gateway counters measure uptake of
the local move-preview accelerator (§5), fed by a small, best-effort client beacon
(`POST /metrics/local-eval`, session-gated) that batches counter deltas and posts them
on a 60 s timer and when the app is backgrounded — **never on the gameplay path** (the
in-app counters are plain in-memory increments; only the periodic flush touches the
network, fire-and-forget). `local_eval_cold_start_total` counts app cold starts (the
adoption denominator); `local_eval_dict_load_total` (`result` = fetched/cache_hit/miss)
splits a dawg download that fills IndexedDB from a cache hit and from a warm-up that
failed or timed out; `local_eval_preview_total` (`path` = local/network) is the share of
previews computed on-device versus the network `evaluate` fallback — the backend load
shed. It answers the owner's adoption question — how often the app opens versus how often
it fills IndexedDB — and is surfaced on the **Scrabble — Users** dashboard. Lossy by
design (a dropped beacon just retries its batch on the next flush); it is telemetry, never
a game input.
- **Rate-limit observability:** every limiter rejection increments the gateway - **Rate-limit observability:** every limiter rejection increments the gateway
counter `gateway_rate_limited_total` (`class` = user/public/email/admin — aggregate counter `gateway_rate_limited_total` (`class` = user/public/email/admin — aggregate
only, honouring the no-per-user-label discipline above) and logs one **Debug** line; only, honouring the no-per-user-label discipline above) and logs one **Debug** line;
+47 -1
View File
@@ -28,6 +28,10 @@ type serverMetrics struct {
rateLimited metric.Int64Counter rateLimited metric.Int64Counter
banned metric.Int64Counter banned metric.Int64Counter
active *activeUsers active *activeUsers
// Client-reported local move-preview adoption (see localEvalMetricsHandler).
localColdStart metric.Int64Counter
localDictLoad metric.Int64Counter
localPreview metric.Int64Counter
} }
// newServerMetrics builds the instruments on meter (nil selects a no-op meter), // newServerMetrics builds the instruments on meter (nil selects a no-op meter),
@@ -54,7 +58,12 @@ func newServerMetrics(meter metric.Meter) *serverMetrics {
if err != nil { if err != nil {
b, _ = noop.NewMeterProvider().Meter(meterName).Int64Counter("gateway_abuse_banned_total") b, _ = noop.NewMeterProvider().Meter(meterName).Int64Counter("gateway_abuse_banned_total")
} }
m := &serverMetrics{edge: h, rateLimited: c, banned: b, active: newActiveUsers()} m := &serverMetrics{
edge: h, rateLimited: c, banned: b, active: newActiveUsers(),
localColdStart: counterOf(meter, "local_eval_cold_start_total", "App cold starts reported by clients — the denominator for local-move-preview adoption."),
localDictLoad: counterOf(meter, "local_eval_dict_load_total", "Client dictionary loads for the local move preview, by result (fetched, cache_hit or miss)."),
localPreview: counterOf(meter, "local_eval_preview_total", "Client move previews, by path (local on-device, or network fallback)."),
}
gauge, err := meter.Int64ObservableGauge("active_users", gauge, err := meter.Int64ObservableGauge("active_users",
metric.WithDescription("Distinct accounts that performed an authenticated action within the window (in-memory, single gateway instance).")) metric.WithDescription("Distinct accounts that performed an authenticated action within the window (in-memory, single gateway instance)."))
@@ -98,3 +107,40 @@ func (m *serverMetrics) recordRateLimited(ctx context.Context, class string) {
func (m *serverMetrics) recordBan(ctx context.Context, reason string) { func (m *serverMetrics) recordBan(ctx context.Context, reason string) {
m.banned.Add(ctx, 1, metric.WithAttributes(attribute.String("reason", reason))) m.banned.Add(ctx, 1, metric.WithAttributes(attribute.String("reason", reason)))
} }
// localEvalReport is the client-reported local move-preview telemetry batch — deltas since
// the client's previous report. It backs the adoption dashboard: app cold starts vs cached
// dictionaries vs on-device previews.
type localEvalReport struct {
ColdStart int `json:"cold_start"`
DictFetched int `json:"dict_fetched"`
DictCacheHit int `json:"dict_cache_hit"`
DictMiss int `json:"dict_miss"`
PreviewLocal int `json:"preview_local"`
PreviewNetwork int `json:"preview_network"`
}
// recordLocalEval folds one client report into the edge's local-eval counters.
func (m *serverMetrics) recordLocalEval(ctx context.Context, r localEvalReport) {
add := func(c metric.Int64Counter, n int, opts ...metric.AddOption) {
if n > 0 {
c.Add(ctx, int64(n), opts...)
}
}
add(m.localColdStart, r.ColdStart)
add(m.localDictLoad, r.DictFetched, metric.WithAttributes(attribute.String("result", "fetched")))
add(m.localDictLoad, r.DictCacheHit, metric.WithAttributes(attribute.String("result", "cache_hit")))
add(m.localDictLoad, r.DictMiss, metric.WithAttributes(attribute.String("result", "miss")))
add(m.localPreview, r.PreviewLocal, metric.WithAttributes(attribute.String("path", "local")))
add(m.localPreview, r.PreviewNetwork, metric.WithAttributes(attribute.String("path", "network")))
}
// counterOf builds an Int64Counter on meter, falling back to a no-op on the rare
// construction error so metrics never fail startup.
func counterOf(meter metric.Meter, name, desc string) metric.Int64Counter {
c, err := meter.Int64Counter(name, metric.WithDescription(desc))
if err != nil {
c, _ = noop.NewMeterProvider().Meter(meterName).Int64Counter(name)
}
return c
}
+49
View File
@@ -9,7 +9,9 @@ package connectsrv
import ( import (
"context" "context"
"crypto/subtle" "crypto/subtle"
"encoding/json"
"errors" "errors"
"io"
"net" "net"
"net/http" "net/http"
"strings" "strings"
@@ -191,6 +193,8 @@ func (s *Server) HTTPHandler() http.Handler {
// The client-side local move preview pulls each game's pinned dictionary blob // The client-side local move preview pulls each game's pinned dictionary blob
// through this session-gated route (not public); see dictBytesHandler. // through this session-gated route (not public); see dictBytesHandler.
mux.Handle("/dict/", s.dictBytesHandler()) mux.Handle("/dict/", s.dictBytesHandler())
// The client posts its local-move-preview adoption telemetry here (session-gated).
mux.Handle("/metrics/local-eval", s.localEvalMetricsHandler())
// The embedded UI: the game SPA under /app/ (web), /telegram/ (the Telegram Mini // The embedded UI: the game SPA under /app/ (web), /telegram/ (the Telegram Mini
// App) and /vk/ (the VK Mini App) — the single-origin model (docs/ARCHITECTURE.md // App) and /vk/ (the VK Mini App) — the single-origin model (docs/ARCHITECTURE.md
// §13). All sit below the h2c wrap so the Connect edge (a more specific prefix) keeps // §13). All sit below the h2c wrap so the Connect edge (a more specific prefix) keeps
@@ -474,6 +478,51 @@ func parseDictPath(p string) (variant, version string, ok bool) {
return variant, version, true return variant, version, true
} }
// localEvalMetricsHandler ingests a client's local move-preview telemetry batch into the
// edge's adoption counters (docs/ARCHITECTURE.md §11). Session-gated so only real clients
// report; the values are aggregate (no per-user attributes) and clamped against a spoofed
// inflation. Only POST; the body is a small JSON of per-metric deltas.
func (s *Server) localEvalMetricsHandler() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
ip := peerIP(r.RemoteAddr, r.Header)
if _, _, err := s.resolve(r.Context(), r.Header, ip); err != nil {
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
var rep localEvalReport
if err := json.NewDecoder(io.LimitReader(r.Body, 512)).Decode(&rep); err != nil {
http.Error(w, "bad request", http.StatusBadRequest)
return
}
clampReport(&rep)
s.metrics.recordLocalEval(r.Context(), rep)
w.WriteHeader(http.StatusNoContent)
})
}
// clampReport bounds each counter of a client report against a spoofed inflation — one batch
// reflects at most a few minutes of a single client's activity.
func clampReport(r *localEvalReport) {
const maxPerField = 1000
clamp := func(n *int) {
if *n < 0 {
*n = 0
} else if *n > maxPerField {
*n = maxPerField
}
}
clamp(&r.ColdStart)
clamp(&r.DictFetched)
clamp(&r.DictCacheHit)
clamp(&r.DictMiss)
clamp(&r.PreviewLocal)
clamp(&r.PreviewNetwork)
}
// resolve extracts and resolves the Authorization bearer token to an account id // resolve extracts and resolves the Authorization bearer token to an account id
// and its guest flag, returning a Connect Unauthenticated error when it is missing // and its guest flag, returning a Connect Unauthenticated error when it is missing
// or unknown. // or unknown.
+3
View File
@@ -8,6 +8,7 @@
import Board from './Board.svelte'; import Board from './Board.svelte';
import Rack from './Rack.svelte'; import Rack from './Rack.svelte';
import { gateway } from '../lib/gateway'; import { gateway } from '../lib/gateway';
import { notePreviewLocal, notePreviewNetwork } from '../lib/localeval-metrics';
import { navigate } from '../lib/router.svelte'; import { navigate } from '../lib/router.svelte';
import { app, handleError, showToast, markChatRead, seedChatUnread } from '../lib/app.svelte'; import { app, handleError, showToast, markChatRead, seedChatUnread } from '../lib/app.svelte';
import { connection } from '../lib/connection.svelte'; import { connection } from '../lib/connection.svelte';
@@ -725,6 +726,7 @@
if (reader && hasAlphabet(v.game.variant)) { if (reader && hasAlphabet(v.game.variant)) {
try { try {
preview = d.evaluateLocal(reader, v.game.variant, board, sub.tiles, v.game.multipleWordsPerTurn); preview = d.evaluateLocal(reader, v.game.variant, board, sub.tiles, v.game.multipleWordsPerTurn);
notePreviewLocal();
return; return;
} catch { } catch {
/* fall through to the network preview */ /* fall through to the network preview */
@@ -736,6 +738,7 @@
previewTimer = setTimeout(async () => { previewTimer = setTimeout(async () => {
try { try {
preview = await gateway.evaluate(id, sub.tiles, variant, ctrl.signal); preview = await gateway.evaluate(id, sub.tiles, variant, ctrl.signal);
notePreviewNetwork();
} catch { } catch {
/* best-effort (or aborted) */ /* best-effort (or aborted) */
} }
+5
View File
@@ -9,6 +9,7 @@ import { GatewayError } from './client';
import { navigate, router } from './router.svelte'; import { navigate, router } from './router.svelte';
import { errorKey, localeFrom, setLocale, t, type Locale } from './i18n/index.svelte'; import { errorKey, localeFrom, setLocale, t, type Locale } from './i18n/index.svelte';
import { languageNeedsServerSync } from './language'; import { languageNeedsServerSync } from './language';
import { startLocalEvalMetrics } from './localeval-metrics';
import { applyReduceMotion, applyTelegramTheme, applyTheme, type ThemePref, type TelegramThemeParams } from './theme'; import { applyReduceMotion, applyTelegramTheme, applyTheme, type ThemePref, type TelegramThemeParams } from './theme';
import { import {
insideTelegram, insideTelegram,
@@ -637,6 +638,10 @@ function applyTelegramChrome(launch: TelegramLaunch): void {
const TELEGRAM_SDK_TIMEOUT_MS = 10000; const TELEGRAM_SDK_TIMEOUT_MS = 10000;
export async function bootstrap(): Promise<void> { export async function bootstrap(): Promise<void> {
// Record the cold start and start the adoption beacon (skipped under the mock harness, which
// has no real backend). Deliberately before any await, so even an early-return launch path
// (e.g. the Telegram launch-error screen) still counts as an app open.
if (import.meta.env.MODE !== 'mock') startLocalEvalMetrics();
const prefs = await loadPrefs(); const prefs = await loadPrefs();
app.theme = prefs.theme ?? 'auto'; app.theme = prefs.theme ?? 'auto';
app.reduceMotion = prefs.reduceMotion ?? false; app.reduceMotion = prefs.reduceMotion ?? false;
+3
View File
@@ -103,6 +103,9 @@ export interface GatewayClient {
* opts.signal aborts a stalled download; opts.reload bypasses the browser HTTP cache (the * opts.signal aborts a stalled download; opts.reload bypasses the browser HTTP cache (the
* debug reset, for testing a cold load). */ * debug reset, for testing a cold load). */
fetchDict(variant: Variant, version: string, opts?: { signal?: AbortSignal; reload?: boolean }): Promise<ArrayBuffer>; fetchDict(variant: Variant, version: string, opts?: { signal?: AbortSignal; reload?: boolean }): Promise<ArrayBuffer>;
/** Post the client's local move-preview adoption telemetry — a small batch of counter deltas,
* session-gated and fire-and-forget (never on the gameplay path). */
reportLocalEval(counts: Record<string, number>): Promise<void>;
// --- draft --- // --- draft ---
/** The player's server-persisted client-side composition (rack order + board tiles), so a /** The player's server-persisted client-side composition (rack order + board tiles), so a
+4
View File
@@ -8,6 +8,7 @@
import { Dawg } from './dawg'; import { Dawg } from './dawg';
import { dictKey, idbGetDawg, idbPutDawg, idbDelDawg, requestPersist } from './store'; import { dictKey, idbGetDawg, idbPutDawg, idbDelDawg, requestPersist } from './store';
import { gateway } from '../gateway'; import { gateway } from '../gateway';
import { noteDictFetched, noteDictCacheHit, noteDictMiss as noteDictMissMetric } from '../localeval-metrics';
import type { Variant } from '../model'; import type { Variant } from '../model';
// Loaded readers by key. A dictionary is immutable, so an instance is reused for // Loaded readers by key. A dictionary is immutable, so an instance is reused for
@@ -31,6 +32,7 @@ let dictDisabled = false;
*/ */
export function noteDictMiss(): void { export function noteDictMiss(): void {
if (!dictDisabled && ++misses >= MISS_LIMIT) dictDisabled = true; if (!dictDisabled && ++misses >= MISS_LIMIT) dictDisabled = true;
noteDictMissMetric();
} }
/** dictLoadingDisabled reports whether the bad-connection breaker has tripped for this /** dictLoadingDisabled reports whether the bad-connection breaker has tripped for this
@@ -73,6 +75,7 @@ async function load(variant: Variant, version: string, key: string, signal?: Abo
if (cached) { if (cached) {
const reader = new Dawg(new Uint8Array(cached)); const reader = new Dawg(new Uint8Array(cached));
instances.set(key, reader); instances.set(key, reader);
noteDictCacheHit();
return reader; return reader;
} }
} catch { } catch {
@@ -101,6 +104,7 @@ async function load(variant: Variant, version: string, key: string, signal?: Abo
return null; // not a dawg — do not cache it return null; // not a dawg — do not cache it
} }
instances.set(key, reader); instances.set(key, reader);
noteDictFetched();
void idbPutDawg(key, buf); void idbPutDawg(key, buf);
void requestPersist(); void requestPersist();
return reader; return reader;
+84
View File
@@ -0,0 +1,84 @@
// Client-side accumulator + beacon for local move-preview adoption telemetry. It counts a
// few events — the app cold start, dictionary loads by result, and move previews by path —
// and posts the running deltas to the edge periodically and when the app is backgrounded,
// so the backend can chart adoption (docs/ARCHITECTURE.md §11). Best-effort throughout: a
// failed beacon keeps its counts to retry, and nothing here throws into the caller. It lives
// outside the lazy dict subsystem so the cold start is recorded even when no game is opened.
import { gateway } from './gateway';
interface Counts {
cold_start: number;
dict_fetched: number;
dict_cache_hit: number;
dict_miss: number;
preview_local: number;
preview_network: number;
}
function zero(): Counts {
return { cold_start: 0, dict_fetched: 0, dict_cache_hit: 0, dict_miss: 0, preview_local: 0, preview_network: 0 };
}
let counts = zero();
let started = false;
function total(c: Counts): number {
return c.cold_start + c.dict_fetched + c.dict_cache_hit + c.dict_miss + c.preview_local + c.preview_network;
}
// flush posts the accumulated deltas and resets them; on failure it folds the batch back in
// so the next flush retries. Best-effort.
async function flush(): Promise<void> {
if (total(counts) === 0) return;
const batch = counts;
counts = zero();
try {
await gateway.reportLocalEval({ ...batch });
} catch {
counts.cold_start += batch.cold_start;
counts.dict_fetched += batch.dict_fetched;
counts.dict_cache_hit += batch.dict_cache_hit;
counts.dict_miss += batch.dict_miss;
counts.preview_local += batch.preview_local;
counts.preview_network += batch.preview_network;
}
}
/**
* startLocalEvalMetrics records the app cold start and begins the periodic beacon. Called once
* at launch (a repeat call is a no-op). It also flushes when the app is backgrounded, so a
* batch is not lost when the tab is closed.
*/
export function startLocalEvalMetrics(): void {
if (started) return;
started = true;
counts.cold_start += 1;
setInterval(() => void flush(), 60_000);
if (typeof document !== 'undefined') {
document.addEventListener('visibilitychange', () => {
if (document.visibilityState === 'hidden') void flush();
});
}
}
/** noteDictFetched: a dictionary was downloaded and cached (an IndexedDB fill). */
export function noteDictFetched(): void {
counts.dict_fetched += 1;
}
/** noteDictCacheHit: a dictionary loaded from the IndexedDB cache (no download). */
export function noteDictCacheHit(): void {
counts.dict_cache_hit += 1;
}
/** noteDictMiss: a warm-up did not produce a usable dictionary (failed or timed out). */
export function noteDictMiss(): void {
counts.dict_miss += 1;
}
/** notePreviewLocal: a move preview was computed on-device. */
export function notePreviewLocal(): void {
counts.preview_local += 1;
}
/** notePreviewNetwork: a move preview fell back to the network. */
export function notePreviewNetwork(): void {
counts.preview_network += 1;
}
+4
View File
@@ -166,6 +166,10 @@ export class MockGateway implements GatewayClient {
throw new Error('fetchDict unsupported in mock'); throw new Error('fetchDict unsupported in mock');
} }
async reportLocalEval(_counts: Record<string, number>): Promise<void> {
// Telemetry is a no-op in mock mode.
}
async gamesList(): Promise<GameList> { async gamesList(): Promise<GameList> {
return { return {
games: [...this.games.values()].map((g) => structuredClone(g.view)), games: [...this.games.values()].map((g) => structuredClone(g.view)),
+12
View File
@@ -80,6 +80,18 @@ export function createTransport(baseUrl: string): GatewayClient {
return res.arrayBuffer(); return res.arrayBuffer();
}, },
async reportLocalEval(counts) {
if (!token) throw new Error('reportLocalEval: no session');
// keepalive so a batch flushed as the app is backgrounded still reaches the edge.
const res = await fetch(`${origin}/metrics/local-eval`, {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify(counts),
keepalive: true,
});
if (!res.ok) throw new Error(`reportLocalEval: HTTP ${res.status}`);
},
async authTelegram(initData) { async authTelegram(initData) {
return codec.decodeSession(await exec('auth.telegram', codec.encodeTelegramLogin(initData, browserOffset()))); return codec.decodeSession(await exec('auth.telegram', codec.encodeTelegramLogin(initData, browserOffset())));
}, },