// Command dictgen dumps golden parity vectors from the committed dawg // dictionaries so the TypeScript dawg reader can be checked byte-for-byte // against the authoritative Go dafsa reader. // // For each *.dawg file it writes, into the output directory: // // - .words.bin — every stored word as alphabet-index bytes, in index // order, framed as [1-byte length][length index bytes]. The word at stream // position k has IndexOfB == k. // - .neg.bin — negative lookups (sequences whose IndexOfB is -1), same // framing, to exercise the not-found path at varying depths. // - .meta.json — NumAdded/NumNodes/NumEdges plus the alphabet size, for // a header-parse sanity cross-check on the TS side. // // It is a development tool (not built into any service), analogous to // cmd/jetgen. Run it from the repository root: // // go run ./backend/cmd/dictgen -dawg-dir ../scrabble-solver/dawg -out package main import ( "bufio" "bytes" "encoding/json" "flag" "fmt" "math/rand" "os" "path/filepath" "sort" "strings" dawg "github.com/iliadenisov/dafsa" ) // meta is the per-dictionary sanity payload cross-checked by the TS reader. type meta struct { NumAdded int `json:"numAdded"` NumNodes int `json:"numNodes"` NumEdges int `json:"numEdges"` Alphabet int `json:"alphabet"` } func main() { dawgDir := flag.String("dawg-dir", "../scrabble-solver/dawg", "directory holding the .dawg files") outDir := flag.String("out", "", "output directory for the golden files (required)") negCount := flag.Int("neg", 20000, "number of negative lookups to emit per dictionary") flag.Parse() if *outDir == "" { fail("-out is required") } if err := os.MkdirAll(*outDir, 0o755); err != nil { fail("mkdir out: %v", err) } files, err := filepath.Glob(filepath.Join(*dawgDir, "*.dawg")) if err != nil { fail("glob: %v", err) } sort.Strings(files) if len(files) == 0 { fail("no .dawg files in %s", *dawgDir) } for _, f := range files { if err := process(f, *outDir, *negCount); err != nil { fail("%s: %v", filepath.Base(f), err) } } } // process emits the golden files for a single dawg dictionary. func process(path, outDir string, negCount int) error { name := strings.TrimSuffix(filepath.Base(path), ".dawg") data, err := os.ReadFile(path) if err != nil { return err } finder, err := dawg.Read(bytes.NewReader(data), 0) if err != nil { return fmt.Errorf("read dawg: %w", err) } defer finder.Close() // Stream every stored word in index order; keep a decimated sample and the // maximum alphabet index for negative generation. wf, err := os.Create(filepath.Join(outDir, name+".words.bin")) if err != nil { return err } bw := bufio.NewWriter(wf) var ( count int maxIx byte sample [][]byte ) finder.EnumerateB(func(index int, word []byte, final bool) int { if !final { return 0 // Continue } if index != count { panic(fmt.Sprintf("%s: enumerate index gap: got %d want %d", name, index, count)) } writeWord(bw, word) for _, b := range word { if b > maxIx { maxIx = b } } if count%4 == 0 && len(sample) < 60000 { sample = append(sample, append([]byte(nil), word...)) } count++ return 0 // Continue }) if err := bw.Flush(); err != nil { return err } if err := wf.Close(); err != nil { return err } if count != finder.NumAdded() { return fmt.Errorf("word count %d != NumAdded %d", count, finder.NumAdded()) } alphabet := int(maxIx) + 1 // Negatives: mutate sampled real words and keep the ones the reader rejects. nf, err := os.Create(filepath.Join(outDir, name+".neg.bin")) if err != nil { return err } nbw := bufio.NewWriter(nf) rng := rand.New(rand.NewSource(1)) neg := 0 for neg < negCount && len(sample) > 0 { base := sample[rng.Intn(len(sample))] cand := append([]byte(nil), base...) switch rng.Intn(3) { case 0: // extend by one index cand = append(cand, byte(rng.Intn(alphabet))) case 1: // flip one index if len(cand) > 0 { cand[rng.Intn(len(cand))] = byte(rng.Intn(alphabet)) } case 2: // drop the tail and flip the new last index if len(cand) > 1 { cand = cand[:len(cand)-1] cand[len(cand)-1] = byte(rng.Intn(alphabet)) } } if finder.IndexOfB(cand) == -1 { writeWord(nbw, cand) neg++ } } if err := nbw.Flush(); err != nil { return err } if err := nf.Close(); err != nil { return err } m := meta{NumAdded: finder.NumAdded(), NumNodes: finder.NumNodes(), NumEdges: finder.NumEdges(), Alphabet: alphabet} mb, err := json.MarshalIndent(m, "", " ") if err != nil { return err } if err := os.WriteFile(filepath.Join(outDir, name+".meta.json"), mb, 0o644); err != nil { return err } fmt.Printf("%-12s words=%d negatives=%d alphabet=%d nodes=%d edges=%d\n", name, count, neg, alphabet, finder.NumNodes(), finder.NumEdges()) return nil } // writeWord frames one index-byte word as [length][bytes]. func writeWord(w *bufio.Writer, word []byte) { if len(word) > 255 { panic(fmt.Sprintf("word too long to frame: %d", len(word))) } w.WriteByte(byte(len(word))) w.Write(word) } func fail(format string, args ...any) { fmt.Fprintf(os.Stderr, "dictgen: "+format+"\n", args...) os.Exit(1) }