scrabble-game/backend/cmd/dictgen/main.go

// Command dictgen dumps golden parity vectors from the committed dawg
// dictionaries so the TypeScript dawg reader can be checked byte-for-byte
// against the authoritative Go dafsa reader.
//
// For each *.dawg file it writes, into the output directory:
//
//   - <name>.words.bin — every stored word as alphabet-index bytes, in index
//     order, framed as [1-byte length][length index bytes]. The word at stream
//     position k has IndexOfB == k.
//   - <name>.neg.bin — negative lookups (sequences whose IndexOfB is -1), same
//     framing, to exercise the not-found path at varying depths.
//   - <name>.meta.json — NumAdded/NumNodes/NumEdges plus the alphabet size, for
//     a header-parse sanity cross-check on the TS side.
//
// It is a development tool (not built into any service), analogous to
// cmd/jetgen. Run it from the repository root:
//
//	go run ./backend/cmd/dictgen -dawg-dir ../scrabble-solver/dawg -out <dir>
package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"math/rand"
	"os"
	"path/filepath"
	"sort"
	"strings"

	dawg "github.com/iliadenisov/dafsa"
)

// meta is the per-dictionary sanity payload cross-checked by the TS reader.
type meta struct {
	NumAdded int `json:"numAdded"`
	NumNodes int `json:"numNodes"`
	NumEdges int `json:"numEdges"`
	Alphabet int `json:"alphabet"`
}

func main() {
	dawgDir := flag.String("dawg-dir", "../scrabble-solver/dawg", "directory holding the .dawg files")
	outDir := flag.String("out", "", "output directory for the golden files (required)")
	negCount := flag.Int("neg", 20000, "number of negative lookups to emit per dictionary")
	flag.Parse()

	if *outDir == "" {
		fail("-out is required")
	}
	if err := os.MkdirAll(*outDir, 0o755); err != nil {
		fail("mkdir out: %v", err)
	}

	files, err := filepath.Glob(filepath.Join(*dawgDir, "*.dawg"))
	if err != nil {
		fail("glob: %v", err)
	}
	sort.Strings(files)
	if len(files) == 0 {
		fail("no .dawg files in %s", *dawgDir)
	}

	for _, f := range files {
		if err := process(f, *outDir, *negCount); err != nil {
			fail("%s: %v", filepath.Base(f), err)
		}
	}
}

// process emits the golden files for a single dawg dictionary.
func process(path, outDir string, negCount int) error {
	name := strings.TrimSuffix(filepath.Base(path), ".dawg")

	data, err := os.ReadFile(path)
	if err != nil {
		return err
	}
	finder, err := dawg.Read(bytes.NewReader(data), 0)
	if err != nil {
		return fmt.Errorf("read dawg: %w", err)
	}
	defer finder.Close()

	// Stream every stored word in index order; keep a decimated sample and the
	// maximum alphabet index for negative generation.
	wf, err := os.Create(filepath.Join(outDir, name+".words.bin"))
	if err != nil {
		return err
	}
	bw := bufio.NewWriter(wf)

	var (
		count  int
		maxIx  byte
		sample [][]byte
	)
	finder.EnumerateB(func(index int, word []byte, final bool) int {
		if !final {
			return 0 // Continue
		}
		if index != count {
			panic(fmt.Sprintf("%s: enumerate index gap: got %d want %d", name, index, count))
		}
		writeWord(bw, word)
		for _, b := range word {
			if b > maxIx {
				maxIx = b
			}
		}
		if count%4 == 0 && len(sample) < 60000 {
			sample = append(sample, append([]byte(nil), word...))
		}
		count++
		return 0 // Continue
	})
	if err := bw.Flush(); err != nil {
		return err
	}
	if err := wf.Close(); err != nil {
		return err
	}
	if count != finder.NumAdded() {
		return fmt.Errorf("word count %d != NumAdded %d", count, finder.NumAdded())
	}

	alphabet := int(maxIx) + 1

	// Negatives: mutate sampled real words and keep the ones the reader rejects.
	nf, err := os.Create(filepath.Join(outDir, name+".neg.bin"))
	if err != nil {
		return err
	}
	nbw := bufio.NewWriter(nf)
	rng := rand.New(rand.NewSource(1))
	neg := 0
	for neg < negCount && len(sample) > 0 {
		base := sample[rng.Intn(len(sample))]
		cand := append([]byte(nil), base...)
		switch rng.Intn(3) {
		case 0: // extend by one index
			cand = append(cand, byte(rng.Intn(alphabet)))
		case 1: // flip one index
			if len(cand) > 0 {
				cand[rng.Intn(len(cand))] = byte(rng.Intn(alphabet))
			}
		case 2: // drop the tail and flip the new last index
			if len(cand) > 1 {
				cand = cand[:len(cand)-1]
				cand[len(cand)-1] = byte(rng.Intn(alphabet))
			}
		}
		if finder.IndexOfB(cand) == -1 {
			writeWord(nbw, cand)
			neg++
		}
	}
	if err := nbw.Flush(); err != nil {
		return err
	}
	if err := nf.Close(); err != nil {
		return err
	}

	m := meta{NumAdded: finder.NumAdded(), NumNodes: finder.NumNodes(), NumEdges: finder.NumEdges(), Alphabet: alphabet}
	mb, err := json.MarshalIndent(m, "", "  ")
	if err != nil {
		return err
	}
	if err := os.WriteFile(filepath.Join(outDir, name+".meta.json"), mb, 0o644); err != nil {
		return err
	}

	fmt.Printf("%-12s words=%d negatives=%d alphabet=%d nodes=%d edges=%d\n",
		name, count, neg, alphabet, finder.NumNodes(), finder.NumEdges())
	return nil
}

// writeWord frames one index-byte word as [length][bytes].
func writeWord(w *bufio.Writer, word []byte) {
	if len(word) > 255 {
		panic(fmt.Sprintf("word too long to frame: %d", len(word)))
	}
	w.WriteByte(byte(len(word)))
	w.Write(word)
}

func fail(format string, args ...any) {
	fmt.Fprintf(os.Stderr, "dictgen: "+format+"\n", args...)
	os.Exit(1)
}