Merge pull request 'feat(ru): drop abbreviations from the Russian noun list' (#6) from feature/drop-abbreviations into master

The list leaked abbreviations two ways: an OpenCorpora noun tagged Abbr without
Fixd (the declinable acronym ОГВЗ) slipped past the Abbr+Fixd seed filter, and the
РАН «(сокр.)» words (вуз, нэп, колхоз, гост, …) were kept on purpose by the old
policy.

Drop every abbreviation instead. A final pass in ru_stage2.py removes a word when
the orthographic note marks it «(сокр.)» (is_abbrev_note) or when every OpenCorpora
noun reading carries Abbr (oc_abbr_only — also the declinable огвз/заз the seed
filter missed). Homographs with a plain-noun reading survive (ваза «ВАЗ», рис).
Lexicalised words are rescued via manual_confirm.txt (колхоз, совхоз, рация,
спецназ, эсминец, дзот, дот, прораб, токамак, тэн, … and под — a real noun OC knows
only as ПОД); the bound prefix гос is vetoed in manual_reject.txt.

108 words removed from ru_scrabble / ru_erudit; en_sowpods unchanged. tools/README
documents the policy and that libmorph is required for a faithful rebuild.

.gitea/workflows/build.yaml

@@ -34,9 +34,9 @@ jobs:
       - name: Build DAWGs
         run: |
           mkdir -p dawg
-          go run ./cmd/builddict -dict dictionaries/english/sowpods.txt -alphabet latin   -name en_sowpods  -out dawg
+          go run ./cmd/builddict -dict sources/scrabble_en/sowpods.txt -alphabet latin   -name en_sowpods  -out dawg
-          go run ./cmd/builddict -dict dictprep/russian/scrabble.txt    -alphabet russian -name ru_scrabble -out dawg
+          go run ./cmd/builddict -dict sources/scrabble_ru/scrabble.txt    -alphabet russian -name ru_scrabble -out dawg
-          go run ./cmd/builddict -dict dictprep/russian/erudit.txt      -alphabet russian -name ru_erudit   -out dawg
+          go run ./cmd/builddict -dict sources/erudit_ru/erudit.txt      -alphabet russian -name ru_erudit   -out dawg
           ls -la dawg/
           for f in en_sowpods ru_scrabble ru_erudit; do
             test -s "dawg/$f.dawg" || { echo "missing dawg/$f.dawg"; exit 1; }

.gitignore

@@ -1,3 +1,14 @@
 # Built DAWGs are release artifacts (published by CI on a vX.Y.Z tag), not committed.
 /dawg/
 /scrabble-dawg-*.tar.gz
+
+# Russian prep-pipeline debug dumps (regenerated locally by tools/, never committed).
+# The pdftotext source of truth (orfo_dict_2025.txt) and the Stage-1 base (all.txt) ARE
+# committed under sources/scrabble_ru/, and the source PDF under tools/ — see tools/README.md.
+/sources/scrabble_ru/undefined.txt
+/sources/scrabble_ru/adjectives.txt
+/sources/scrabble_ru/verbs.txt
+/sources/scrabble_ru/singulars.txt
+/sources/scrabble_ru/fate.tsv
+/tools/libmorph_check
+__pycache__/

Makefile

@@ -8,7 +8,7 @@
 #   ru_erudit.dawg   — Эрудит (the Ё→Е folded + de-duped list, committed as russian/erudit.txt)
 #
 # CI builds the DAWGs as a validation gate; release artifacts are published from this output
-# (see README.md). Regenerate russian/erudit.txt from scrabble.txt with dictprep/fold_yo.py.
+# (see README.md). Regenerate russian/erudit.txt from scrabble.txt with tools/fold_yo.py.
 
 export GOPRIVATE := gitea.iliadenisov.ru/*
 
@@ -21,13 +21,13 @@ BUILDDICT := $(GO) run ./cmd/builddict
 dawg: dawg-en dawg-ru dawg-erudit
 
 dawg-en:
-	$(BUILDDICT) -dict dictionaries/english/sowpods.txt -alphabet latin -name en_sowpods -out $(DAWG_DIR)
+	$(BUILDDICT) -dict sources/scrabble_en/sowpods.txt -alphabet latin -name en_sowpods -out $(DAWG_DIR)
 
 dawg-ru:
-	$(BUILDDICT) -dict dictprep/russian/scrabble.txt -alphabet russian -name ru_scrabble -out $(DAWG_DIR)
+	$(BUILDDICT) -dict sources/scrabble_ru/scrabble.txt -alphabet russian -name ru_scrabble -out $(DAWG_DIR)
 
 dawg-erudit:
-	$(BUILDDICT) -dict dictprep/russian/erudit.txt -alphabet russian -name ru_erudit -out $(DAWG_DIR)
+	$(BUILDDICT) -dict sources/erudit_ru/erudit.txt -alphabet russian -name ru_erudit -out $(DAWG_DIR)
 
 clean-dawg:
 	rm -f $(DAWG_DIR)/*.dawg

README.md

@@ -17,9 +17,9 @@ byte-identical to the solver's committed test fixtures.
 
 | file | variant | source |
 | --- | --- | --- |
-| `en_sowpods.dawg` | English (SOWPODS) | `dictionaries/english/sowpods.txt` |
+| `en_sowpods.dawg` | English (SOWPODS) | `sources/scrabble_en/sowpods.txt` |
-| `ru_scrabble.dawg` | Russian Scrabble | `dictprep/russian/scrabble.txt` |
+| `ru_scrabble.dawg` | Russian Scrabble | `sources/scrabble_ru/scrabble.txt` |
-| `ru_erudit.dawg` | Эрудит | `dictprep/russian/erudit.txt` (Ё→Е folded `scrabble.txt`, via `dictprep/fold_yo.py`) |
+| `ru_erudit.dawg` | Эрудит | `sources/erudit_ru/erudit.txt` (Ё→Е folded `scrabble.txt`, via `tools/fold_yo.py`) |
 
 The CI (`.gitea/workflows/build.yaml`) rebuilds them on every push/PR as a validation gate
 (inlined `go run`, no `make`/`python` needed on the runner). Release artifacts are published per
@@ -30,11 +30,11 @@ a new release, never breaking a running backend).
 
 ## Sources / provenance
 
-- **English:** `dictionaries/english/sowpods.txt`, vendored from
+- **English:** `sources/scrabble_en/sowpods.txt`, vendored from
   [`kamilmielnik/scrabble-dictionaries`](https://github.com/kamilmielnik/scrabble-dictionaries).
-- **Russian:** `dictprep/russian/scrabble.txt`, derived from the Russian academic orthographic
+- **Russian:** `sources/scrabble_ru/scrabble.txt`, derived from the Russian academic orthographic
-  dictionary by the tooling under `dictprep/` (see `dictprep/README.md`); `dictprep/russian/erudit.txt`
+  dictionary by the tooling under `tools/` (see `tools/README.md`); `sources/erudit_ru/erudit.txt`
-  is its Ё→Е folded form (`dictprep/fold_yo.py`). Only the prepared word lists are vendored; the
+  is its Ё→Е folded form (`tools/fold_yo.py`). Only the prepared word lists are vendored; the
   heavy upstream source (the orfo PDF/text) is not.
 
 ## Build
@@ -45,7 +45,7 @@ make dawg     # -> dawg/{en_sowpods,ru_scrabble,ru_erudit}.dawg
 
 Requires Go (module deps fetched with `GOPRIVATE=gitea.iliadenisov.ru/*`, exported by the
 Makefile). No `python` is needed for the build — the Ё→Е fold is committed as `erudit.txt`;
-regenerate it with `python3 dictprep/fold_yo.py dictprep/russian/scrabble.txt > dictprep/russian/erudit.txt`.
+regenerate it with `python3 tools/fold_yo.py sources/scrabble_ru/scrabble.txt > sources/erudit_ru/erudit.txt`.
 
 ## Release
 

cmd/builddict/main.go

@@ -17,7 +17,7 @@ import (
 )
 
 func main() {
-	dict := flag.String("dict", "dictionaries/english/sowpods.txt", "word list file (one word per line)")
+	dict := flag.String("dict", "sources/scrabble_en/sowpods.txt", "word list file (one word per line)")
 	out := flag.String("out", "testdata", "output directory")
 	name := flag.String("name", "sowpods", "base name for the output file")
 	minLen := flag.Int("min", 2, "minimum word length")

sources/erudit_ru/README.md

@@ -0,0 +1,6 @@
+# erudit_ru source
+
+`erudit.txt` — the Эрудит word list: the Ё→Е folded and de-duplicated form of
+[`../scrabble_ru/scrabble.txt`](../scrabble_ru/scrabble.txt), produced by `tools/fold_yo.py`
+(the Эрудит ruleset has no Ё tile and treats Е/Ё as one letter). Built to `dawg/ru_erudit.dawg`
+(`make dawg-erudit`).

sources/scrabble_en/README.md

@@ -0,0 +1,5 @@
+# scrabble_en source
+
+`sowpods.txt` — the English SOWPODS word list, vendored from
+[`kamilmielnik/scrabble-dictionaries`](https://github.com/kamilmielnik/scrabble-dictionaries).
+Built to `dawg/en_sowpods.dawg` (`make dawg-en`).

sources/scrabble_ru/README.md

@@ -0,0 +1,10 @@
+# scrabble_ru source
+
+`scrabble.txt` — Russian Scrabble common nouns (nominative singular), produced by the prep
+pipeline under [`../../tools/`](../../tools/README.md) from the Russian academic orthographic
+dictionary, cross-checked against OpenCorpora and libmorph. `manual_confirm.txt` holds the
+hand-reviewed additions the pipeline merges in. Built to `dawg/ru_scrabble.dawg` (`make dawg-ru`).
+
+The pdftotext source of truth (`orfo_dict_2025.txt`) and the Stage-1 base (`all.txt`) are
+committed here; only the debug dumps (`undefined.txt`, `adjectives.txt`, `verbs.txt`,
+`singulars.txt`, `fate.tsv`) are regenerated locally and git-ignored.

sources/scrabble_ru/manual_confirm.txt

@@ -133,3 +133,31 @@
 флеш
 флешмобер
 шиноремонт
+военкор
+дзот
+дисбат
+дот
+завгар
+иняз
+каско
+колхоз
+линкор
+морпех
+научпоп
+прораб
+рация
+санбат
+совхоз
+сопромат
+спецназ
+спецхран
+старлей
+старпом
+твэл
+теормех
+токамак
+тэн
+чермет
+эсминец
+юннат
+под

sources/scrabble_ru/manual_reject.txt

@@ -0,0 +1,72 @@
+акцизный
+апикальный
+батальонный
+безумный
+благородный
+большой
+брюшнотифозный
+велярный
+верхоконный
+взрывной
+вольный
+головной
+горбатый
+городской
+гос
+грамотный
+гриппозный
+губной
+двугласный
+желтокожий
+живой
+заразный
+зубной
+иноверный
+кандальный
+каторжный
+клеймёный
+конный
+конюшенный
+лабиальный
+лысый
+любопытный
+маленький
+межзубный
+назальный
+несчастненький
+нёбный
+номерной
+носовой
+овинный
+одинокий
+отвальный
+параличный
+плитовой
+поддужный
+полубезработный
+полуротный
+припадочный
+сверхбольшой
+свободный
+сподручный
+старшенький
+стопорный
+стреловой
+судейский
+сыпнотифозный
+тифозный
+толкучий
+увулярный
+уголовный
+услужающий
+участвующий
+фабричный
+фланговый
+холерный
+чановой
+чесоточный
+чумной
+штрафованный
+щелевой
+щелинный
+эскадронный

tools/README.md

@@ -1,14 +1,14 @@
-# Russian word-list preparation (`dictprep`)
+# Russian word-list preparation (`tools`)
 
 Builds the Russian **noun** word list for the Scrabble/Эрудит solver out of the official
 Russian academic **orthographic dictionary**, cross-checked against two independent
 morphological dictionaries.
 
 The goal of the pipeline is a list of **common nouns in the nominative singular**
-(`dictprep/russian/scrabble.txt`), plus an ambiguous tail for manual review.
+(`sources/scrabble_ru/scrabble.txt`), plus an ambiguous tail for manual review.
 
 > This directory is self-contained tooling for *building* the word list. It is not part
-> of the solver library. The committed result lives in `dictprep/russian/`.
+> of the solver library. The committed result lives in `sources/scrabble_ru/`.
 
 ## Source
 
@@ -23,7 +23,7 @@ The PDF is git-ignored (large, third-party); place it here as `orfo_dict_2025.pd
 pdftotext output is committed as `russian/orfo_dict_2025.txt`, so the word list rebuilds
 from the text alone — the binary PDF is needed only to regenerate that text.
 
-## Outputs (`dictprep/russian/`)
+## Outputs (`sources/scrabble_ru/`)
 
 The committed result is **three** files; every other bucket stays in the Stage-2
 process's memory (dump it with `--dump`, query it with `--trace WORD`).
@@ -33,6 +33,7 @@ process's memory (dump it with `--dump`, query it with `--trace WORD`).
 | `orfo_dict_2025.txt` | ✓ | the pdftotext output — the parsed source of truth (the PDF binary is not needed to rebuild). |
 | `all.txt` | ✓ | Stage 1 base: every clean Cyrillic headword/variant; a plural headword with a singular is replaced by that singular. |
 | `manual_confirm.txt` | ✓ | hand-reviewed nouns from the undefined tail; the brain merges them into the result. |
+| `manual_reject.txt` | ✓ | hand-vetoed false nouns (substantivized adjectives a dictionary misreads as nouns, e.g. нёбный); subtracted from the result last. |
 | `scrabble.txt` | ✓ | **Stage 2 result**: common nouns, nominative singular (+ pluralia tantum), length 2–15 — the working dictionary. |
 | `undefined.txt` | — | the ambiguous tail; kept in memory, written only with `--dump`. |
 
@@ -56,28 +57,32 @@ ru-venv/bin/pip install mawo-pymorphy3            # bundles OpenCorpora 2025 (wo
 
 # 4. libmorph — the independent morphological dictionary (Stage 2 cross-check)
 sudo apt-get install -y morphrus morphrus-dev moonycode-dev morphapi-dev
-g++ -std=c++17 -O2 dictprep/libmorph_check.cpp -lmorphrus -lmoonycode -o dictprep/libmorph_check
+g++ -std=c++17 -O2 tools/libmorph_check.cpp -lmorphrus -lmoonycode -o tools/libmorph_check
 ```
 
-If `dictprep/libmorph_check` is absent, Stage 2 still runs — it simply drops libmorph from
+If `tools/libmorph_check` is absent, Stage 2 still runs — it simply drops libmorph from
-the stack and reports `libmorph_helper=MISSING`.
+the stack and reports `libmorph_helper=MISSING`. **Build it before a release rebuild:** the
+committed word lists were produced *with* libmorph, and dropping it shifts the result by a few
+hundred words (libmorph-only nouns disappear, and adjectives/eponyms libmorph would veto —
+неперов, венерин — leak in through the orthographic-note path). A rebuild meant to refresh the
+committed lists is only faithful with the helper present.
 
 ## How to run
 
 ```sh
 # Stage 0 — PDF -> plain text (committed as the source of truth; run once)
-pdftotext dictprep/orfo_dict_2025.pdf dictprep/russian/orfo_dict_2025.txt
+pdftotext tools/orfo_dict_2025.pdf sources/scrabble_ru/orfo_dict_2025.txt
 
-# Stage 1 — build the base word list (Go): dictprep/russian/all.txt + /tmp/ru_*.txt
+# Stage 1 — build the base word list (Go): sources/scrabble_ru/all.txt + /tmp/ru_*.txt
-go run ./dictprep/ruwords
+go run ./tools/ruwords
 
 # Stage 2 — the brain (Python + mawo + libmorph): writes scrabble.txt
-ru-venv/bin/python dictprep/ru_stage2.py
+ru-venv/bin/python tools/ru_stage2.py
 
-# ask how a word did or did not reach the dictionary
+# audit how a single word did or did not reach the dictionary (detailed, per-signal report)
-ru-venv/bin/python dictprep/ru_stage2.py --trace травмпункт
+ru-venv/bin/python tools/ru_stage2.py --trace ндс
 # also write the in-memory buckets (undefined, adjectives, verbs, singulars, fate.tsv)
-ru-venv/bin/python dictprep/ru_stage2.py --dump
+ru-venv/bin/python tools/ru_stage2.py --dump
 ```
 
 `-from`/`-to` (defaulting to 452/168808) bound the column word-list section of
@@ -111,7 +116,9 @@ Each Stage-1 word (length 2–15) is routed by three sources, most authoritative
 
 1. **OpenCorpora** (`words.dawg`, read directly — *not* the predictor): a common-noun
    reading ⇒ keep the OpenCorpora lemma. The full OpenCorpora common-noun lexicon is also
-   added (so nouns absent from the PDF are included).
+   added (so nouns absent from the PDF are included), **minus indeclinable abbreviations**
+   (`Abbr`+`Fixd`: ндс, кпд, чп, …) — OpenCorpora tags those as nouns but they are not
+   Scrabble words. See *Abbreviations, proper nouns & function words* below for the full rule.
 2. **libmorph** (independent dictionary, via `libmorph_check`): a common-noun reading ⇒
    keep the libmorph lemma. The two dictionaries are treated as **complementary** — a noun
    reading in *either* is enough (their disagreements were reviewed and resolved this way,
@@ -125,6 +132,10 @@ Each Stage-1 word (length 2–15) is routed by three sources, most authoritative
    травмпункт`, `регги и реггей`) and one is already a confirmed noun, the other is moved
    from review/undefined into the result as well, propagated transitively through chains.
    The plural-form variants the dictionaries already resolve never reach this step.
+5. **Manual overrides**: `manual_confirm.txt` adds maintainer-approved nouns from the
+   undefined tail (before variant rescue, so they can anchor it); `manual_reject.txt` is
+   subtracted **last**, removing false nouns a dictionary misreads — substantivized
+   adjectives with no nominal use (`нёбный`, `акцизный`, …) — overriding every path above.
 
 The nominative singular always comes from the dictionary that recognised the word, or from
 the orthographic `ед.` note — never from a predictor guess (libmorph and the predictor
@@ -151,6 +162,64 @@ speech. The codes were reverse-engineered (the docs omit the table):
 The analyser instance is requested with the key `libmorph.api.v4:utf-8` so words are
 passed and lemmas returned in UTF-8.
 
+## Abbreviations, proper nouns & function words
+
+**Abbreviations are not Scrabble words and are dropped** (вуз, нэп, гост, жэк, спид, огвз,
+обком, наркомат, роно, тв, …). A final pass in `build` removes every word matched by either
+signal, then re-admits the maintainer's rescues — see `is_abbrev_note`, `oc_abbr_only` and the
+abbreviation subtraction in `ru_stage2.py`:
+
+- **The orthographic note marks it `(сокр.)`** (`is_abbrev_note`): the РАН dictionary's own
+  abbreviation mark, covering both the bare initialisms (вуз, нэп, гост, бтр, кпд) and the
+  Soviet-era compounds (обком, райком, наркомат, госплан, …). This is the authoritative signal.
+- **OpenCorpora reads it only as an abbreviation** (`oc_abbr_only`): *every* common-noun
+  reading carries the `Abbr` grammeme. This catches acronyms РАН does not head at all — the
+  declinable огвз (a full `Abbr` case paradigm, no `Fixd`, which the `Abbr`+`Fixd` seed filter
+  misses), заз, and the indeclinable роно, тв, фио, суперэвм. A homograph that *also* has a
+  plain-noun reading is **not** abbreviation-only and survives — ваза «ВАЗ», рис «РИС».
+
+Still dropped by the seed/loop before this pass: a `CONJ`/`PRCL` **function word** even with a
+noun-like note (**зато**), and an OpenCorpora reading that is an **inflected form** of another
+lemma (**ан** = род. мн. от «ана»; the proper noun «Ан» is capitalised and never reaches
+`all.txt`). Proper nouns (`Name`/`Surn`/`Geox`/`Orgn`/…) are excluded from the seed outright.
+
+**Maintainer overrides decide the borderline cases** (this is policy, not a heuristic):
+
+- `manual_confirm.txt` **rescues** lexicalised words that carry `(сокр.)` or that OpenCorpora
+  only knows as an abbreviation but which read as ordinary nouns — **колхоз, совхоз, рация,
+  спецназ, эсминец, линкор, дзот, дот, прораб, токамак, тэн, …**, and **под** (a real noun,
+  «под печи», that OpenCorpora knows only as the abbreviation ПОД). Edit this list to widen or
+  narrow which abbreviations stay.
+- `manual_reject.txt` **vetoes** the few non-noun leftovers that no signal catches — e.g. the
+  bound prefix **гос** («гос… — первая часть сложных слов», not a standalone word).
+
+> History note: an earlier policy *kept* lexicalised abbreviations (сельпо, вуз, нэп, …) and
+> used `(сокр.)` only as a non-discriminator. The current policy drops all abbreviations and
+> rescues a curated subset, so the dictionary holds words, not acronyms.
+
+### Auditing a word and repeating this analysis
+
+`--trace WORD` prints a per-signal audit — Stage-1/`all.txt` membership, every OpenCorpora
+reading (POS, `Abbr`/`Fixd`, `CONJ`/`PRCL`), libmorph, the РАН note with its `classify`
+verdict, and the final outcome:
+
+```sh
+ru-venv/bin/python tools/ru_stage2.py --trace ндс      # → НЕ в словаре: исключено из посева
+ru-venv/bin/python tools/ru_stage2.py --trace сельпо   # → В СЛОВАРЕ: по помете орфословаря
+```
+
+To find a *whole class* of suspect words (the procedure used to catch the abbreviations above),
+read the signals straight from the dictionaries with the same primitives the pipeline uses:
+
+1. iterate `M._dawg_dict.words_dawg` and collect the lemmas with the grammemes you suspect
+   (e.g. `Abbr`+`Fixd` on the lemma, no clean noun paradigm) — the candidate set;
+2. split it by `w in all.txt` (lowercase РАН headword), by `classify(w, note)` (РАН-note POS),
+   and by an OpenCorpora `CONJ`/`PRCL` reading — these three signals separate real words from
+   abbreviations / proper nouns / function words;
+3. review the candidate list by hand, add the distinguishing grammeme to the filter, re-run,
+   and diff the rebuilt list against the previous one to confirm only the intended words moved
+   (`--trace` each borderline decision).
+
 ## Notes & caveats
 
 - The hard tail (≈ 35 000 Stage-1 words / our candidates) is in **no** morphological

tools/fold_yo.py

@@ -5,7 +5,7 @@ The Эрудит ruleset has no Ё tile and treats Е/Ё as one letter, so its d
 folded before the DAWG is built. Folding merges pairs like ёж/еж, hence the de-dup. Output
 is sorted (Russian order over the 32 folded letters) and LF-separated.
 
-Run:  python3 dictprep/fold_yo.py dictprep/russian/scrabble.txt > /tmp/ru_erudit_words.txt
+Run:  python3 tools/fold_yo.py sources/scrabble_ru/scrabble.txt > /tmp/ru_erudit_words.txt
 """
 import sys
 

tools/ru_stage2.py

@@ -5,10 +5,10 @@ It reads the Stage-1 base word list (built once by ruwords so the heavy PDF is n
 re-parsed) together with the grammatical notes and the singular/variant structure, runs
 the whole noun-selection logic in memory, and writes a minimal result:
 
-    dictprep/russian/scrabble.txt   — the working dictionary (common nouns, nom. sing.)
+    sources/scrabble_ru/scrabble.txt   — the working dictionary (common nouns, nom. sing.)
-    dictprep/russian/undefined.txt  — the ambiguous tail, left for manual review
+    sources/scrabble_ru/undefined.txt  — the ambiguous tail, left for manual review
 
-(dictprep/russian/all.txt is the Stage-1 base.) Every other bucket — adjectives, verbs,
+(sources/scrabble_ru/all.txt is the Stage-1 base.) Every other bucket — adjectives, verbs,
 the merged note-nouns, singulars, variants — stays in memory. Pass --dump to also write
 them; pass --trace WORD to ask how a single word did or did not reach the dictionary.
 
@@ -17,9 +17,9 @@ variants are read from the pdftotext output (slov.txt) and the Stage-1 side file
 expensive PDF parse itself runs only once.
 
 Sources, most authoritative first: OpenCorpora (mawo-pymorphy3), libmorph (libmorph_check),
-and the orthographic dictionary's own notes. See dictprep/README.md.
+and the orthographic dictionary's own notes. See tools/README.md.
 
-Run:  ru-venv/bin/python dictprep/ru_stage2.py [--dump] [--trace WORD]
+Run:  ru-venv/bin/python tools/ru_stage2.py [--dump] [--trace WORD]
 """
 import argparse
 import os
@@ -27,7 +27,9 @@ import re
 import subprocess
 
 HERE = os.path.dirname(os.path.abspath(__file__))
-OUT_DIR = os.path.join(HERE, "russian")
+# The curated Russian word lists live in sources/scrabble_ru/ (this tool sits in tools/);
+# the uncommitted pipeline intermediates (orfo/all/debug) are regenerated alongside them.
+OUT_DIR = os.path.join(HERE, "..", "sources", "scrabble_ru")
 SLOV = os.path.join(OUT_DIR, "orfo_dict_2025.txt")  # committed pdftotext output (source of truth)
 WL_FROM, WL_TO = 452, 168808  # 1-based inclusive bounds of the column word-list section
 OC_CACHE = "/tmp/oc_nouns.txt"
@@ -36,6 +38,8 @@ LIBMORPH_BIN = os.path.join(HERE, "libmorph_check")
 ALPHABET = "абвгдеёжзийклмнопрстуфхцчшщъыьэюя"
 ORDER = {c: i for i, c in enumerate(ALPHABET)}
 PROPER = {"Name", "Surn", "Patr", "Geox", "Orgn", "Trad"}
+ABBR_NOUN = {"Abbr", "Fixd"}      # indeclinable abbreviation marker (НДС, КПД, сельпо, …)
+FUNCTION_POS = {"CONJ", "PRCL"}   # conjunction / particle: a function word, never a Scrabble noun
 LIBMORPH_NOUN_CODES = set(range(7, 22)) | {24}  # 7..21 plus 24 (pluralia tantum)
 ADJ_END = {"ая", "яя", "ое", "ее", "ье", "ья", "ьи"}
 VERB3 = ("ет", "ёт", "ит", "ют", "ут", "ает", "яет", "ует", "уют", "нет", "жет", "чет")
@@ -70,7 +74,10 @@ D = M._dawg_dict
 
 
 def oc_noun_lemmas():
-    """Every common-noun lemma (nom. sing. / pluralia tantum) in OpenCorpora's words.dawg."""
+    """Every common-noun lemma (nom. sing. / pluralia tantum) in OpenCorpora's words.dawg,
+    excluding indeclinable abbreviations (Abbr+Fixd, e.g. НДС, КПД). Such words are admitted
+    only when the orthographic dictionary attests them as nouns (see oc_abbr_fixd_only and the
+    Stage-2 loop); abbreviations carried by OpenCorpora alone (ст, ср, кпд, …) are dropped."""
     gp, pt = D.get_paradigm, D.parse_tag_string
     para0, tagc = {}, {}
 
@@ -102,6 +109,8 @@ def oc_noun_lemmas():
         pre0, suf0, gr0 = g0(pid)
         if (PROPER & gr) or (PROPER & gr0):
             continue
+        if ABBR_NOUN <= gr0:        # indeclinable abbreviation: not seeded; the loop decides via the note
+            continue
         stem = word[len(pre):len(word) - len(suf)] if suf else word[len(pre):]
         out.add(pre0 + stem + suf0)
     return {w for w in out if cyr_ok(w)}
@@ -124,6 +133,75 @@ def oc_status(word):
     return False, True
 
 
+def oc_abbr_fixd_only(word):
+    """True when OpenCorpora's only common-noun reading of word is an indeclinable
+    abbreviation (Abbr+Fixd) — e.g. сельпо, ндс, ан. Such a word is not admitted as a noun on
+    OpenCorpora's say-so; the Stage-2 loop hands it to the orthographic note (and the function-
+    word check) instead, so lexicalised nouns (сельпо, под, ска) survive while bare letter-
+    abbreviations (ндс, кпд) and proper/служебные homographs (ан, зато) do not."""
+    parses = D.get_word_parses(word)
+    if not parses:
+        return False
+    gp, pt = D.get_paradigm, D.parse_tag_string
+    has_noun = has_plain = False
+    for pid, idx in parses:
+        suf, tag, pre = gp(pid, idx)
+        pos, gr = pt(tag)
+        if pos != "NOUN":
+            continue
+        _, tag0, _ = gp(pid, 0)
+        _, gr0 = pt(tag0)
+        if PROPER & gr or PROPER & gr0:
+            continue
+        has_noun = True
+        if not (ABBR_NOUN <= gr0):
+            has_plain = True
+    return has_noun and not has_plain
+
+
+def oc_function_word(word):
+    """True when OpenCorpora reads word as a conjunction or particle — a function word that
+    shares a spelling with an abbreviation/proper noun (ан «ан нет», зато). It is never a
+    Scrabble noun even if the orthographic note looks noun-like (ан → «самолёт Антонова»)."""
+    gp, pt = D.get_paradigm, D.parse_tag_string
+    for pid, idx in (D.get_word_parses(word) or []):
+        if pt(gp(pid, idx)[1])[0] in FUNCTION_POS:
+            return True
+    return False
+
+
+def oc_abbr_only(word):
+    """True when every common-noun reading OpenCorpora gives word is an abbreviation (carries
+    the Abbr grammeme on its paradigm). Generalises oc_abbr_fixd_only past the Fixd
+    (indeclinable) requirement, so it also catches a *declinable* acronym that OpenCorpora
+    inflects through a full Abbr case paradigm without Fixd — огвз, заз — as well as the
+    indeclinable роно, тв, фио. A word with any non-Abbr noun reading (the homographs ваза
+    «ВАЗ», рис «РИС», под «ПОД»/под печи) is not abbreviation-only and is left alone."""
+    parses = D.get_word_parses(word)
+    if not parses:
+        return False
+    gp, pt = D.get_paradigm, D.parse_tag_string
+    has_noun = has_plain = False
+    for pid, idx in parses:
+        suf, tag, pre = gp(pid, idx)
+        pos, gr = pt(tag)
+        if pos != "NOUN":
+            continue
+        _, tag0, _ = gp(pid, 0)
+        _, gr0 = pt(tag0)
+        if PROPER & gr or PROPER & gr0:
+            continue
+        has_noun = True
+        if "Abbr" not in gr0:
+            has_plain = True
+    return has_noun and not has_plain
+
+
+def is_abbrev_note(note):
+    """True when the orthographic note marks an abbreviation («сокр.», e.g. вуз, нэп, колхоз)."""
+    return bool(note) and "сокр" in note
+
+
 def libmorph_analyze(words):
     """Map each word to (known, noun_lemma, codes) per libmorph; noun_lemma is None when it
     is not a common noun there. Empty result if the helper binary is not built."""
@@ -238,9 +316,27 @@ def build():
     scrabble = set(oc)
     adj, verb, amb = [], [], []
     for w in pdf:
+        if oc_abbr_fixd_only(w):
+            # Indeclinable abbreviation in OpenCorpora: do not trust OC's noun verdict. Drop it
+            # if OC also reads it as a function word (ан, зато — служебное/пропер); otherwise let
+            # the orthographic note decide, keeping lexicalised nouns (сельпо, под, ска) and
+            # dropping the rest (кпд, чп, гор, …).
+            if oc_function_word(w):
+                fate[w] = "отброшено: служебное слово (CONJ/PRCL в OpenCorpora)"
+            elif classify(w, hmap.get(w)) == "noun":
+                s = to_singular(w)
+                scrabble.add(s)
+                fate[w] = "scrabble: несклон. аббрев.-сущ. по помете орфословаря" + ("" if s == w else f" → {s}")
+            else:
+                fate[w] = "отброшено: аббревиатура без подтверждающей пометы орфословаря"
+            continue
         oc_noun, oc_known = oc_status(w)
         if oc_noun:
-            fate[w] = "scrabble: сущ. по OpenCorpora"
+            # A common-noun reading already put w in the seed (scrabble = set(oc)) — unless its
+            # lemma is a different word, i.e. w is an inflected form (ан = род. мн. от «ана»);
+            # such a form is not a headword and stays out.
+            fate[w] = ("scrabble: сущ. по OpenCorpora" if w in scrabble
+                       else "отброшено: словоформа OpenCorpora (лемма — другое слово)")
             continue
         lm_known, lm_lemma, _ = lm.get(w, (False, None, frozenset()))
         if lm_lemma is not None:
@@ -289,25 +385,77 @@ def build():
                     fate[y] = f"scrabble: вариант от «{x}» (через «и»)"
                     changed = True
 
-    undefined = [w for w in amb if w not in scrabble]
+    # Abbreviations are not Scrabble words. Two independent signals catch them: the orthographic
+    # note marks «(сокр.: …)» (вуз, нэп, колхоз, гост, …); and OpenCorpora tags acronyms Abbr —
+    # both the indeclinable (роно, тв, фио) and the declinable ones the Abbr+Fixd seed filter
+    # misses (огвз, заз, гулаг). Drop every such word, except the lexicalised ones the maintainer
+    # rescues in manual_confirm.txt (колхоз, совхоз, рация, спецназ, … and под — a real noun
+    # OpenCorpora knows only as the abbreviation ПОД). The homographs that are real words with a
+    # mere abbreviation reading (ваза, рис) survive — oc_abbr_only requires *all* noun readings
+    # to be Abbr. Subtracted before the manual-reject veto.
+    confirm = {w for w in load(os.path.join(OUT_DIR, "manual_confirm.txt")) if cyr_ok(w)}
+    abbrev = {w for w in scrabble if w not in confirm and (is_abbrev_note(hmap.get(w)) or oc_abbr_only(w))}
+    scrabble -= abbrev
+    for w in abbrev:
+        fate[w] = "отброшено: аббревиатура (помета «сокр.» орфословаря или только Abbr-чтения в OpenCorpora)"
+
+    # Manual rejections: words the maintainer vetoed — substantivized-adjective false nouns
+    # a dictionary misreads as nouns (нёбный, акцизный, …). Subtracted last, so it overrides
+    # every admission path above (OC seed / libmorph / note / manual_confirm / variant).
+    reject = {w for w in load(os.path.join(OUT_DIR, "manual_reject.txt")) if cyr_ok(w)}
+    scrabble -= reject
+    for w in reject:
+        fate[w] = "отброшено: ручной запрет (manual_reject.txt)"
+
+    undefined = [w for w in amb if w not in scrabble and w not in reject]
     return {
         "oc": oc, "scrabble": scrabble, "undefined": undefined,
         "adjectives": adj, "verbs": verb, "singulars": ed_nouns,
-        "fate": fate, "all": set(all_words),
+        "fate": fate, "all": set(all_words), "hmap": hmap,
     }
 
 
 def trace(word, r):
+    """A detailed, per-signal audit of how WORD did or did not reach the dictionary: its
+    Stage-1 (all.txt) membership, every OpenCorpora reading, libmorph, the orthographic note
+    with its classify verdict, and the final outcome. Used by `--trace WORD`."""
     w = destress(word)
-    if w in r["fate"]:
+    note = r["hmap"].get(w)
-        return r["fate"][w]
+    parses = D.get_word_parses(w) or []
+    gp, pt = D.get_paradigm, D.parse_tag_string
+    pos = sorted({pt(gp(pid, idx)[1])[0] for pid, idx in parses})
+    oc_noun, _ = oc_status(w)
+
+    out = [f"{word}  →  {'В СЛОВАРЕ' if w in r['scrabble'] else 'НЕ в словаре'}"]
+    out.append(f"  Stage-1 / all.txt (строчный заголовок РАН): {'да' if w in r['all'] else 'нет'}")
+    if parses:
+        kind = ("только несклон. аббрев. (Abbr+Fixd)" if oc_abbr_fixd_only(w) else
+                "обычное сущ." if oc_noun else
+                "не существительное")
+        out.append(f"  OpenCorpora: части речи {pos}; как сущ. — {kind}; "
+                   f"служебное (CONJ/PRCL): {'да' if oc_function_word(w) else 'нет'}")
+    else:
+        out.append("  OpenCorpora: нет в словаре")
+    lm = libmorph_analyze([w]).get(w)
+    if lm:
+        known, lemma, _ = lm
+        out.append("  libmorph: " + (f"сущ. → {lemma}" if lemma else ("известно, не сущ." if known else "нет")))
+    else:
+        out.append("  libmorph: helper отсутствует — в анализе не участвует")
+    out.append("  Орфословарь РАН: " + (f"помета «{note}» → classify={classify(w, note)}"
+                                        if note is not None else "нет заголовка"))
+    reason = r["fate"].get(w)
+    if reason is None:
         if w in r["scrabble"]:
-        return "scrabble: лексикон OpenCorpora" if w in r["oc"] else "scrabble: производная/лемма"
+            reason = "лексикон OpenCorpora (обычное сущ.)"
-    if w not in r["all"]:
+        elif oc_abbr_fixd_only(w) and w not in r["all"]:
-        return "нет в russian_all (не извлечено на Stage 1 — нет в .pdf, либо имя собств./дефис/форма)"
+            reason = "исключено из посева: несклон. аббрев. (Abbr+Fixd), нет в all.txt"
-    if not cyr_ok(w):
+        elif not parses and w not in r["all"]:
-        return "отсеяно: длина или символы вне диапазона (2–15 кириллица)"
+            reason = "нет ни в OpenCorpora, ни в all.txt"
-    return "не определено"
+        else:
+            reason = "не прошло ни один путь отбора"
+    out.append(f"  ИТОГ: {reason}")
+    return "\n".join(out)
 
 
 def main():
@@ -318,11 +466,11 @@ def main():
 
     r = build()
     if args.trace:
-        print(f"{args.trace}: {trace(args.trace, r)}")
+        print(trace(args.trace, r))
         return
 
     write(os.path.join(OUT_DIR, "scrabble.txt"), r["scrabble"])
-    print(f"=> dictprep/russian/scrabble.txt   {len(r['scrabble'])}")
+    print(f"=> sources/scrabble_ru/scrabble.txt   {len(r['scrabble'])}")
     print(f"   undefined kept in memory: {len(set(r['undefined']))} (use --dump to write it)")
     if args.dump:
         write(os.path.join(OUT_DIR, "undefined.txt"), r["undefined"])

tools/ruwords/main.go

@@ -11,10 +11,10 @@
 //
 // It also collects a variant headword joined by "и" when it carries its own grammatical
 // note (e.g. "аблатив, -а и аблятив, -а"). Suffix-singular reconstruction is heuristic;
-// Stage 2 (dictprep/ru_stage2.py) re-checks the words against real dictionaries.
+// Stage 2 (tools/ru_stage2.py) re-checks the words against real dictionaries.
 //
-//	pdftotext dictprep/orfo_dict_2025.pdf /tmp/slov.txt
+//	pdftotext tools/orfo_dict_2025.pdf /tmp/slov.txt
-//	go run ./dictprep/ruwords -in /tmp/slov.txt -from 452 -to 168808 \
+//	go run ./tools/ruwords -in /tmp/slov.txt -from 452 -to 168808 \
 //	    -out russian_all.txt -skip russian_skip.txt
 package main
 
@@ -327,8 +327,8 @@ func writeWords(path string, words []string) error {
 }
 
 func main() {
-	in := flag.String("in", "dictprep/russian/orfo_dict_2025.txt", "plain-text dictionary (pdftotext output)")
+	in := flag.String("in", "sources/scrabble_ru/orfo_dict_2025.txt", "plain-text dictionary (pdftotext output)")
-	out := flag.String("out", "dictprep/russian/all.txt", "output: the base word list (clean headwords + reconstructed singulars + variants)")
+	out := flag.String("out", "sources/scrabble_ru/all.txt", "output: the base word list (clean headwords + reconstructed singulars + variants)")
 	skip := flag.String("skip", "/tmp/ru_skip.txt", "output: every other token, for a later morphology re-check")
 	sings := flag.String("singulars", "/tmp/ru_singulars.txt", "output: singulars reconstructed from \"ед.\" (known nouns)")
 	varsOut := flag.String("variants", "/tmp/ru_variants.txt", "output: variant pairs joined by \"и\" (primary<TAB>variant)")