Ilia Denisov
dd61ff1d51
build / dawg (pull_request) Successful in 4m22s
Consolidate the scattered build inputs (dictionaries/english/, dictprep/russian/) into one sources/ tree keyed by the variant labels (scrabble_en/scrabble_ru/ erudit_ru), and move the Russian prep pipeline to tools/. The dawg outputs and their filenames are unchanged — rebuilt byte-identical (en_sowpods/ru_scrabble/ ru_erudit) — so the release artifact and the backend are unaffected. ru_stage2.py OUT_DIR and the ruwords flag defaults are repointed to sources/scrabble_ru/; Makefile / CI / cmd/builddict default / README updated; pipeline intermediates git-ignored. Verified: make dawg byte-identical to the committed baseline, py_compile + go vet of the moved tools. The full Russian regeneration pipeline (pymorphy3/libmorph/orfo PDF) was not run here.
76 lines
2.1 KiB
Go
76 lines
2.1 KiB
Go
// Command builddict converts a word list into a serialized DAWG. By default it reads the
|
|
// English SOWPODS list (Latin alphabet); pass -alphabet russian for the Cyrillic lists.
|
|
package main
|
|
|
|
import (
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"github.com/iliadenisov/alphabet"
|
|
|
|
"gitea.iliadenisov.ru/developer/scrabble-solver/dictdawg"
|
|
"gitea.iliadenisov.ru/developer/scrabble-solver/wordlist"
|
|
)
|
|
|
|
func main() {
|
|
dict := flag.String("dict", "sources/scrabble_en/sowpods.txt", "word list file (one word per line)")
|
|
out := flag.String("out", "testdata", "output directory")
|
|
name := flag.String("name", "sowpods", "base name for the output file")
|
|
minLen := flag.Int("min", 2, "minimum word length")
|
|
maxLen := flag.Int("max", 15, "maximum word length")
|
|
alpha := flag.String("alphabet", "latin", "alphabet: latin (English) or russian")
|
|
flag.Parse()
|
|
|
|
var idx alphabet.Indexer
|
|
switch *alpha {
|
|
case "latin":
|
|
idx = alphabet.Latin()
|
|
case "russian":
|
|
idx = alphabet.Embedded(alphabet.Langs.LangRu)
|
|
default:
|
|
log.Fatalf("unknown -alphabet %q (want latin or russian)", *alpha)
|
|
}
|
|
|
|
t0 := time.Now()
|
|
words, err := wordlist.Read(*dict, idx, *minLen, *maxLen)
|
|
if err != nil {
|
|
log.Fatalf("read %s: %v", *dict, err)
|
|
}
|
|
fmt.Printf("loaded %d words from %s in %s\n", len(words), *dict, time.Since(t0).Round(time.Millisecond))
|
|
|
|
if err := os.MkdirAll(*out, 0o755); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
t := time.Now()
|
|
f, err := dictdawg.Build(idx, words)
|
|
if err != nil {
|
|
log.Fatalf("build dawg: %v", err)
|
|
}
|
|
path := filepath.Join(*out, *name+".dawg")
|
|
if err := dictdawg.Save(f, path); err != nil {
|
|
log.Fatalf("save: %v", err)
|
|
}
|
|
size := int64(0)
|
|
if fi, err := os.Stat(path); err == nil {
|
|
size = fi.Size()
|
|
}
|
|
fmt.Printf("DAWG %d nodes, %s, built+saved in %s -> %s\n",
|
|
f.NumNodes(), humanBytes(size), time.Since(t).Round(time.Millisecond), path)
|
|
}
|
|
|
|
func humanBytes(n int64) string {
|
|
switch {
|
|
case n >= 1<<20:
|
|
return fmt.Sprintf("%.2f MB", float64(n)/(1<<20))
|
|
case n >= 1<<10:
|
|
return fmt.Sprintf("%.1f KB", float64(n)/(1<<10))
|
|
default:
|
|
return fmt.Sprintf("%d B", n)
|
|
}
|
|
}
|