Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 44713e8a9a | |||
| 74ff3aefb6 | |||
| 1f5e1906ad | |||
| 01821afc9a | |||
| c901fee994 | |||
| e17e945b41 | |||
| b5600771a6 | |||
| 6b8b176f82 | |||
| 5b7a741ec2 |
+3
-5
@@ -2,15 +2,13 @@
|
||||
/dawg/
|
||||
/scrabble-dawg-*.tar.gz
|
||||
|
||||
# Russian prep-pipeline intermediates (regenerated locally by tools/; only the curated
|
||||
# word lists in sources/scrabble_ru/ are committed).
|
||||
/sources/scrabble_ru/orfo_dict_2025.txt
|
||||
/sources/scrabble_ru/all.txt
|
||||
# Russian prep-pipeline debug dumps (regenerated locally by tools/, never committed).
|
||||
# The pdftotext source of truth (orfo_dict_2025.txt) and the Stage-1 base (all.txt) ARE
|
||||
# committed under sources/scrabble_ru/, and the source PDF under tools/ — see tools/README.md.
|
||||
/sources/scrabble_ru/undefined.txt
|
||||
/sources/scrabble_ru/adjectives.txt
|
||||
/sources/scrabble_ru/verbs.txt
|
||||
/sources/scrabble_ru/singulars.txt
|
||||
/sources/scrabble_ru/fate.tsv
|
||||
/tools/libmorph_check
|
||||
/tools/orfo_dict_2025.pdf
|
||||
__pycache__/
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,5 +5,6 @@ pipeline under [`../../tools/`](../../tools/README.md) from the Russian academic
|
||||
dictionary, cross-checked against OpenCorpora and libmorph. `manual_confirm.txt` holds the
|
||||
hand-reviewed additions the pipeline merges in. Built to `dawg/ru_scrabble.dawg` (`make dawg-ru`).
|
||||
|
||||
The pipeline's uncommitted intermediates (`orfo_dict_2025.txt`, `all.txt`, debug dumps) are
|
||||
regenerated here locally and are git-ignored.
|
||||
The pdftotext source of truth (`orfo_dict_2025.txt`) and the Stage-1 base (`all.txt`) are
|
||||
committed here; only the debug dumps (`undefined.txt`, `adjectives.txt`, `verbs.txt`,
|
||||
`singulars.txt`, `fate.tsv`) are regenerated locally and git-ignored.
|
||||
|
||||
+148900
File diff suppressed because it is too large
Load Diff
@@ -133,3 +133,31 @@
|
||||
флеш
|
||||
флешмобер
|
||||
шиноремонт
|
||||
военкор
|
||||
дзот
|
||||
дисбат
|
||||
дот
|
||||
завгар
|
||||
иняз
|
||||
каско
|
||||
колхоз
|
||||
линкор
|
||||
морпех
|
||||
научпоп
|
||||
прораб
|
||||
рация
|
||||
санбат
|
||||
совхоз
|
||||
сопромат
|
||||
спецназ
|
||||
спецхран
|
||||
старлей
|
||||
старпом
|
||||
твэл
|
||||
теормех
|
||||
токамак
|
||||
тэн
|
||||
чермет
|
||||
эсминец
|
||||
юннат
|
||||
под
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
акцизный
|
||||
апикальный
|
||||
батальонный
|
||||
безумный
|
||||
благородный
|
||||
большой
|
||||
брюшнотифозный
|
||||
велярный
|
||||
верхоконный
|
||||
взрывной
|
||||
вольный
|
||||
головной
|
||||
горбатый
|
||||
городской
|
||||
гос
|
||||
грамотный
|
||||
гриппозный
|
||||
губной
|
||||
двугласный
|
||||
желтокожий
|
||||
живой
|
||||
заразный
|
||||
зубной
|
||||
иноверный
|
||||
кандальный
|
||||
каторжный
|
||||
клеймёный
|
||||
конный
|
||||
конюшенный
|
||||
лабиальный
|
||||
лысый
|
||||
любопытный
|
||||
маленький
|
||||
межзубный
|
||||
назальный
|
||||
несчастненький
|
||||
нёбный
|
||||
номерной
|
||||
носовой
|
||||
овинный
|
||||
одинокий
|
||||
отвальный
|
||||
параличный
|
||||
плитовой
|
||||
поддужный
|
||||
полубезработный
|
||||
полуротный
|
||||
припадочный
|
||||
сверхбольшой
|
||||
свободный
|
||||
сподручный
|
||||
старшенький
|
||||
стопорный
|
||||
стреловой
|
||||
судейский
|
||||
сыпнотифозный
|
||||
тифозный
|
||||
толкучий
|
||||
увулярный
|
||||
уголовный
|
||||
услужающий
|
||||
участвующий
|
||||
фабричный
|
||||
фланговый
|
||||
холерный
|
||||
чановой
|
||||
чесоточный
|
||||
чумной
|
||||
штрафованный
|
||||
щелевой
|
||||
щелинный
|
||||
эскадронный
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+73
-4
@@ -33,6 +33,7 @@ process's memory (dump it with `--dump`, query it with `--trace WORD`).
|
||||
| `orfo_dict_2025.txt` | ✓ | the pdftotext output — the parsed source of truth (the PDF binary is not needed to rebuild). |
|
||||
| `all.txt` | ✓ | Stage 1 base: every clean Cyrillic headword/variant; a plural headword with a singular is replaced by that singular. |
|
||||
| `manual_confirm.txt` | ✓ | hand-reviewed nouns from the undefined tail; the brain merges them into the result. |
|
||||
| `manual_reject.txt` | ✓ | hand-vetoed false nouns (substantivized adjectives a dictionary misreads as nouns, e.g. нёбный); subtracted from the result last. |
|
||||
| `scrabble.txt` | ✓ | **Stage 2 result**: common nouns, nominative singular (+ pluralia tantum), length 2–15 — the working dictionary. |
|
||||
| `undefined.txt` | — | the ambiguous tail; kept in memory, written only with `--dump`. |
|
||||
|
||||
@@ -60,7 +61,11 @@ g++ -std=c++17 -O2 tools/libmorph_check.cpp -lmorphrus -lmoonycode -o tools/libm
|
||||
```
|
||||
|
||||
If `tools/libmorph_check` is absent, Stage 2 still runs — it simply drops libmorph from
|
||||
the stack and reports `libmorph_helper=MISSING`.
|
||||
the stack and reports `libmorph_helper=MISSING`. **Build it before a release rebuild:** the
|
||||
committed word lists were produced *with* libmorph, and dropping it shifts the result by a few
|
||||
hundred words (libmorph-only nouns disappear, and adjectives/eponyms libmorph would veto —
|
||||
неперов, венерин — leak in through the orthographic-note path). A rebuild meant to refresh the
|
||||
committed lists is only faithful with the helper present.
|
||||
|
||||
## How to run
|
||||
|
||||
@@ -74,8 +79,8 @@ go run ./tools/ruwords
|
||||
# Stage 2 — the brain (Python + mawo + libmorph): writes scrabble.txt
|
||||
ru-venv/bin/python tools/ru_stage2.py
|
||||
|
||||
# ask how a word did or did not reach the dictionary
|
||||
ru-venv/bin/python tools/ru_stage2.py --trace травмпункт
|
||||
# audit how a single word did or did not reach the dictionary (detailed, per-signal report)
|
||||
ru-venv/bin/python tools/ru_stage2.py --trace ндс
|
||||
# also write the in-memory buckets (undefined, adjectives, verbs, singulars, fate.tsv)
|
||||
ru-venv/bin/python tools/ru_stage2.py --dump
|
||||
```
|
||||
@@ -111,7 +116,9 @@ Each Stage-1 word (length 2–15) is routed by three sources, most authoritative
|
||||
|
||||
1. **OpenCorpora** (`words.dawg`, read directly — *not* the predictor): a common-noun
|
||||
reading ⇒ keep the OpenCorpora lemma. The full OpenCorpora common-noun lexicon is also
|
||||
added (so nouns absent from the PDF are included).
|
||||
added (so nouns absent from the PDF are included), **minus indeclinable abbreviations**
|
||||
(`Abbr`+`Fixd`: ндс, кпд, чп, …) — OpenCorpora tags those as nouns but they are not
|
||||
Scrabble words. See *Abbreviations, proper nouns & function words* below for the full rule.
|
||||
2. **libmorph** (independent dictionary, via `libmorph_check`): a common-noun reading ⇒
|
||||
keep the libmorph lemma. The two dictionaries are treated as **complementary** — a noun
|
||||
reading in *either* is enough (their disagreements were reviewed and resolved this way,
|
||||
@@ -125,6 +132,10 @@ Each Stage-1 word (length 2–15) is routed by three sources, most authoritative
|
||||
травмпункт`, `регги и реггей`) and one is already a confirmed noun, the other is moved
|
||||
from review/undefined into the result as well, propagated transitively through chains.
|
||||
The plural-form variants the dictionaries already resolve never reach this step.
|
||||
5. **Manual overrides**: `manual_confirm.txt` adds maintainer-approved nouns from the
|
||||
undefined tail (before variant rescue, so they can anchor it); `manual_reject.txt` is
|
||||
subtracted **last**, removing false nouns a dictionary misreads — substantivized
|
||||
adjectives with no nominal use (`нёбный`, `акцизный`, …) — overriding every path above.
|
||||
|
||||
The nominative singular always comes from the dictionary that recognised the word, or from
|
||||
the orthographic `ед.` note — never from a predictor guess (libmorph and the predictor
|
||||
@@ -151,6 +162,64 @@ speech. The codes were reverse-engineered (the docs omit the table):
|
||||
The analyser instance is requested with the key `libmorph.api.v4:utf-8` so words are
|
||||
passed and lemmas returned in UTF-8.
|
||||
|
||||
## Abbreviations, proper nouns & function words
|
||||
|
||||
**Abbreviations are not Scrabble words and are dropped** (вуз, нэп, гост, жэк, спид, огвз,
|
||||
обком, наркомат, роно, тв, …). A final pass in `build` removes every word matched by either
|
||||
signal, then re-admits the maintainer's rescues — see `is_abbrev_note`, `oc_abbr_only` and the
|
||||
abbreviation subtraction in `ru_stage2.py`:
|
||||
|
||||
- **The orthographic note marks it `(сокр.)`** (`is_abbrev_note`): the РАН dictionary's own
|
||||
abbreviation mark, covering both the bare initialisms (вуз, нэп, гост, бтр, кпд) and the
|
||||
Soviet-era compounds (обком, райком, наркомат, госплан, …). This is the authoritative signal.
|
||||
- **OpenCorpora reads it only as an abbreviation** (`oc_abbr_only`): *every* common-noun
|
||||
reading carries the `Abbr` grammeme. This catches acronyms РАН does not head at all — the
|
||||
declinable огвз (a full `Abbr` case paradigm, no `Fixd`, which the `Abbr`+`Fixd` seed filter
|
||||
misses), заз, and the indeclinable роно, тв, фио, суперэвм. A homograph that *also* has a
|
||||
plain-noun reading is **not** abbreviation-only and survives — ваза «ВАЗ», рис «РИС».
|
||||
|
||||
Still dropped by the seed/loop before this pass: a `CONJ`/`PRCL` **function word** even with a
|
||||
noun-like note (**зато**), and an OpenCorpora reading that is an **inflected form** of another
|
||||
lemma (**ан** = род. мн. от «ана»; the proper noun «Ан» is capitalised and never reaches
|
||||
`all.txt`). Proper nouns (`Name`/`Surn`/`Geox`/`Orgn`/…) are excluded from the seed outright.
|
||||
|
||||
**Maintainer overrides decide the borderline cases** (this is policy, not a heuristic):
|
||||
|
||||
- `manual_confirm.txt` **rescues** lexicalised words that carry `(сокр.)` or that OpenCorpora
|
||||
only knows as an abbreviation but which read as ordinary nouns — **колхоз, совхоз, рация,
|
||||
спецназ, эсминец, линкор, дзот, дот, прораб, токамак, тэн, …**, and **под** (a real noun,
|
||||
«под печи», that OpenCorpora knows only as the abbreviation ПОД). Edit this list to widen or
|
||||
narrow which abbreviations stay.
|
||||
- `manual_reject.txt` **vetoes** the few non-noun leftovers that no signal catches — e.g. the
|
||||
bound prefix **гос** («гос… — первая часть сложных слов», not a standalone word).
|
||||
|
||||
> History note: an earlier policy *kept* lexicalised abbreviations (сельпо, вуз, нэп, …) and
|
||||
> used `(сокр.)` only as a non-discriminator. The current policy drops all abbreviations and
|
||||
> rescues a curated subset, so the dictionary holds words, not acronyms.
|
||||
|
||||
### Auditing a word and repeating this analysis
|
||||
|
||||
`--trace WORD` prints a per-signal audit — Stage-1/`all.txt` membership, every OpenCorpora
|
||||
reading (POS, `Abbr`/`Fixd`, `CONJ`/`PRCL`), libmorph, the РАН note with its `classify`
|
||||
verdict, and the final outcome:
|
||||
|
||||
```sh
|
||||
ru-venv/bin/python tools/ru_stage2.py --trace ндс # → НЕ в словаре: исключено из посева
|
||||
ru-venv/bin/python tools/ru_stage2.py --trace сельпо # → В СЛОВАРЕ: по помете орфословаря
|
||||
```
|
||||
|
||||
To find a *whole class* of suspect words (the procedure used to catch the abbreviations above),
|
||||
read the signals straight from the dictionaries with the same primitives the pipeline uses:
|
||||
|
||||
1. iterate `M._dawg_dict.words_dawg` and collect the lemmas with the grammemes you suspect
|
||||
(e.g. `Abbr`+`Fixd` on the lemma, no clean noun paradigm) — the candidate set;
|
||||
2. split it by `w in all.txt` (lowercase РАН headword), by `classify(w, note)` (РАН-note POS),
|
||||
and by an OpenCorpora `CONJ`/`PRCL` reading — these three signals separate real words from
|
||||
abbreviations / proper nouns / function words;
|
||||
3. review the candidate list by hand, add the distinguishing grammeme to the filter, re-run,
|
||||
and diff the rebuilt list against the previous one to confirm only the intended words moved
|
||||
(`--trace` each borderline decision).
|
||||
|
||||
## Notes & caveats
|
||||
|
||||
- The hard tail (≈ 35 000 Stage-1 words / our candidates) is in **no** morphological
|
||||
|
||||
Binary file not shown.
+159
-13
@@ -38,6 +38,8 @@ LIBMORPH_BIN = os.path.join(HERE, "libmorph_check")
|
||||
ALPHABET = "абвгдеёжзийклмнопрстуфхцчшщъыьэюя"
|
||||
ORDER = {c: i for i, c in enumerate(ALPHABET)}
|
||||
PROPER = {"Name", "Surn", "Patr", "Geox", "Orgn", "Trad"}
|
||||
ABBR_NOUN = {"Abbr", "Fixd"} # indeclinable abbreviation marker (НДС, КПД, сельпо, …)
|
||||
FUNCTION_POS = {"CONJ", "PRCL"} # conjunction / particle: a function word, never a Scrabble noun
|
||||
LIBMORPH_NOUN_CODES = set(range(7, 22)) | {24} # 7..21 plus 24 (pluralia tantum)
|
||||
ADJ_END = {"ая", "яя", "ое", "ее", "ье", "ья", "ьи"}
|
||||
VERB3 = ("ет", "ёт", "ит", "ют", "ут", "ает", "яет", "ует", "уют", "нет", "жет", "чет")
|
||||
@@ -72,7 +74,10 @@ D = M._dawg_dict
|
||||
|
||||
|
||||
def oc_noun_lemmas():
|
||||
"""Every common-noun lemma (nom. sing. / pluralia tantum) in OpenCorpora's words.dawg."""
|
||||
"""Every common-noun lemma (nom. sing. / pluralia tantum) in OpenCorpora's words.dawg,
|
||||
excluding indeclinable abbreviations (Abbr+Fixd, e.g. НДС, КПД). Such words are admitted
|
||||
only when the orthographic dictionary attests them as nouns (see oc_abbr_fixd_only and the
|
||||
Stage-2 loop); abbreviations carried by OpenCorpora alone (ст, ср, кпд, …) are dropped."""
|
||||
gp, pt = D.get_paradigm, D.parse_tag_string
|
||||
para0, tagc = {}, {}
|
||||
|
||||
@@ -104,6 +109,8 @@ def oc_noun_lemmas():
|
||||
pre0, suf0, gr0 = g0(pid)
|
||||
if (PROPER & gr) or (PROPER & gr0):
|
||||
continue
|
||||
if ABBR_NOUN <= gr0: # indeclinable abbreviation: not seeded; the loop decides via the note
|
||||
continue
|
||||
stem = word[len(pre):len(word) - len(suf)] if suf else word[len(pre):]
|
||||
out.add(pre0 + stem + suf0)
|
||||
return {w for w in out if cyr_ok(w)}
|
||||
@@ -126,6 +133,75 @@ def oc_status(word):
|
||||
return False, True
|
||||
|
||||
|
||||
def oc_abbr_fixd_only(word):
|
||||
"""True when OpenCorpora's only common-noun reading of word is an indeclinable
|
||||
abbreviation (Abbr+Fixd) — e.g. сельпо, ндс, ан. Such a word is not admitted as a noun on
|
||||
OpenCorpora's say-so; the Stage-2 loop hands it to the orthographic note (and the function-
|
||||
word check) instead, so lexicalised nouns (сельпо, под, ска) survive while bare letter-
|
||||
abbreviations (ндс, кпд) and proper/служебные homographs (ан, зато) do not."""
|
||||
parses = D.get_word_parses(word)
|
||||
if not parses:
|
||||
return False
|
||||
gp, pt = D.get_paradigm, D.parse_tag_string
|
||||
has_noun = has_plain = False
|
||||
for pid, idx in parses:
|
||||
suf, tag, pre = gp(pid, idx)
|
||||
pos, gr = pt(tag)
|
||||
if pos != "NOUN":
|
||||
continue
|
||||
_, tag0, _ = gp(pid, 0)
|
||||
_, gr0 = pt(tag0)
|
||||
if PROPER & gr or PROPER & gr0:
|
||||
continue
|
||||
has_noun = True
|
||||
if not (ABBR_NOUN <= gr0):
|
||||
has_plain = True
|
||||
return has_noun and not has_plain
|
||||
|
||||
|
||||
def oc_function_word(word):
|
||||
"""True when OpenCorpora reads word as a conjunction or particle — a function word that
|
||||
shares a spelling with an abbreviation/proper noun (ан «ан нет», зато). It is never a
|
||||
Scrabble noun even if the orthographic note looks noun-like (ан → «самолёт Антонова»)."""
|
||||
gp, pt = D.get_paradigm, D.parse_tag_string
|
||||
for pid, idx in (D.get_word_parses(word) or []):
|
||||
if pt(gp(pid, idx)[1])[0] in FUNCTION_POS:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def oc_abbr_only(word):
|
||||
"""True when every common-noun reading OpenCorpora gives word is an abbreviation (carries
|
||||
the Abbr grammeme on its paradigm). Generalises oc_abbr_fixd_only past the Fixd
|
||||
(indeclinable) requirement, so it also catches a *declinable* acronym that OpenCorpora
|
||||
inflects through a full Abbr case paradigm without Fixd — огвз, заз — as well as the
|
||||
indeclinable роно, тв, фио. A word with any non-Abbr noun reading (the homographs ваза
|
||||
«ВАЗ», рис «РИС», под «ПОД»/под печи) is not abbreviation-only and is left alone."""
|
||||
parses = D.get_word_parses(word)
|
||||
if not parses:
|
||||
return False
|
||||
gp, pt = D.get_paradigm, D.parse_tag_string
|
||||
has_noun = has_plain = False
|
||||
for pid, idx in parses:
|
||||
suf, tag, pre = gp(pid, idx)
|
||||
pos, gr = pt(tag)
|
||||
if pos != "NOUN":
|
||||
continue
|
||||
_, tag0, _ = gp(pid, 0)
|
||||
_, gr0 = pt(tag0)
|
||||
if PROPER & gr or PROPER & gr0:
|
||||
continue
|
||||
has_noun = True
|
||||
if "Abbr" not in gr0:
|
||||
has_plain = True
|
||||
return has_noun and not has_plain
|
||||
|
||||
|
||||
def is_abbrev_note(note):
|
||||
"""True when the orthographic note marks an abbreviation («сокр.», e.g. вуз, нэп, колхоз)."""
|
||||
return bool(note) and "сокр" in note
|
||||
|
||||
|
||||
def libmorph_analyze(words):
|
||||
"""Map each word to (known, noun_lemma, codes) per libmorph; noun_lemma is None when it
|
||||
is not a common noun there. Empty result if the helper binary is not built."""
|
||||
@@ -240,9 +316,27 @@ def build():
|
||||
scrabble = set(oc)
|
||||
adj, verb, amb = [], [], []
|
||||
for w in pdf:
|
||||
if oc_abbr_fixd_only(w):
|
||||
# Indeclinable abbreviation in OpenCorpora: do not trust OC's noun verdict. Drop it
|
||||
# if OC also reads it as a function word (ан, зато — служебное/пропер); otherwise let
|
||||
# the orthographic note decide, keeping lexicalised nouns (сельпо, под, ска) and
|
||||
# dropping the rest (кпд, чп, гор, …).
|
||||
if oc_function_word(w):
|
||||
fate[w] = "отброшено: служебное слово (CONJ/PRCL в OpenCorpora)"
|
||||
elif classify(w, hmap.get(w)) == "noun":
|
||||
s = to_singular(w)
|
||||
scrabble.add(s)
|
||||
fate[w] = "scrabble: несклон. аббрев.-сущ. по помете орфословаря" + ("" if s == w else f" → {s}")
|
||||
else:
|
||||
fate[w] = "отброшено: аббревиатура без подтверждающей пометы орфословаря"
|
||||
continue
|
||||
oc_noun, oc_known = oc_status(w)
|
||||
if oc_noun:
|
||||
fate[w] = "scrabble: сущ. по OpenCorpora"
|
||||
# A common-noun reading already put w in the seed (scrabble = set(oc)) — unless its
|
||||
# lemma is a different word, i.e. w is an inflected form (ан = род. мн. от «ана»);
|
||||
# such a form is not a headword and stays out.
|
||||
fate[w] = ("scrabble: сущ. по OpenCorpora" if w in scrabble
|
||||
else "отброшено: словоформа OpenCorpora (лемма — другое слово)")
|
||||
continue
|
||||
lm_known, lm_lemma, _ = lm.get(w, (False, None, frozenset()))
|
||||
if lm_lemma is not None:
|
||||
@@ -291,25 +385,77 @@ def build():
|
||||
fate[y] = f"scrabble: вариант от «{x}» (через «и»)"
|
||||
changed = True
|
||||
|
||||
undefined = [w for w in amb if w not in scrabble]
|
||||
# Abbreviations are not Scrabble words. Two independent signals catch them: the orthographic
|
||||
# note marks «(сокр.: …)» (вуз, нэп, колхоз, гост, …); and OpenCorpora tags acronyms Abbr —
|
||||
# both the indeclinable (роно, тв, фио) and the declinable ones the Abbr+Fixd seed filter
|
||||
# misses (огвз, заз, гулаг). Drop every such word, except the lexicalised ones the maintainer
|
||||
# rescues in manual_confirm.txt (колхоз, совхоз, рация, спецназ, … and под — a real noun
|
||||
# OpenCorpora knows only as the abbreviation ПОД). The homographs that are real words with a
|
||||
# mere abbreviation reading (ваза, рис) survive — oc_abbr_only requires *all* noun readings
|
||||
# to be Abbr. Subtracted before the manual-reject veto.
|
||||
confirm = {w for w in load(os.path.join(OUT_DIR, "manual_confirm.txt")) if cyr_ok(w)}
|
||||
abbrev = {w for w in scrabble if w not in confirm and (is_abbrev_note(hmap.get(w)) or oc_abbr_only(w))}
|
||||
scrabble -= abbrev
|
||||
for w in abbrev:
|
||||
fate[w] = "отброшено: аббревиатура (помета «сокр.» орфословаря или только Abbr-чтения в OpenCorpora)"
|
||||
|
||||
# Manual rejections: words the maintainer vetoed — substantivized-adjective false nouns
|
||||
# a dictionary misreads as nouns (нёбный, акцизный, …). Subtracted last, so it overrides
|
||||
# every admission path above (OC seed / libmorph / note / manual_confirm / variant).
|
||||
reject = {w for w in load(os.path.join(OUT_DIR, "manual_reject.txt")) if cyr_ok(w)}
|
||||
scrabble -= reject
|
||||
for w in reject:
|
||||
fate[w] = "отброшено: ручной запрет (manual_reject.txt)"
|
||||
|
||||
undefined = [w for w in amb if w not in scrabble and w not in reject]
|
||||
return {
|
||||
"oc": oc, "scrabble": scrabble, "undefined": undefined,
|
||||
"adjectives": adj, "verbs": verb, "singulars": ed_nouns,
|
||||
"fate": fate, "all": set(all_words),
|
||||
"fate": fate, "all": set(all_words), "hmap": hmap,
|
||||
}
|
||||
|
||||
|
||||
def trace(word, r):
|
||||
"""A detailed, per-signal audit of how WORD did or did not reach the dictionary: its
|
||||
Stage-1 (all.txt) membership, every OpenCorpora reading, libmorph, the orthographic note
|
||||
with its classify verdict, and the final outcome. Used by `--trace WORD`."""
|
||||
w = destress(word)
|
||||
if w in r["fate"]:
|
||||
return r["fate"][w]
|
||||
note = r["hmap"].get(w)
|
||||
parses = D.get_word_parses(w) or []
|
||||
gp, pt = D.get_paradigm, D.parse_tag_string
|
||||
pos = sorted({pt(gp(pid, idx)[1])[0] for pid, idx in parses})
|
||||
oc_noun, _ = oc_status(w)
|
||||
|
||||
out = [f"{word} → {'В СЛОВАРЕ' if w in r['scrabble'] else 'НЕ в словаре'}"]
|
||||
out.append(f" Stage-1 / all.txt (строчный заголовок РАН): {'да' if w in r['all'] else 'нет'}")
|
||||
if parses:
|
||||
kind = ("только несклон. аббрев. (Abbr+Fixd)" if oc_abbr_fixd_only(w) else
|
||||
"обычное сущ." if oc_noun else
|
||||
"не существительное")
|
||||
out.append(f" OpenCorpora: части речи {pos}; как сущ. — {kind}; "
|
||||
f"служебное (CONJ/PRCL): {'да' if oc_function_word(w) else 'нет'}")
|
||||
else:
|
||||
out.append(" OpenCorpora: нет в словаре")
|
||||
lm = libmorph_analyze([w]).get(w)
|
||||
if lm:
|
||||
known, lemma, _ = lm
|
||||
out.append(" libmorph: " + (f"сущ. → {lemma}" if lemma else ("известно, не сущ." if known else "нет")))
|
||||
else:
|
||||
out.append(" libmorph: helper отсутствует — в анализе не участвует")
|
||||
out.append(" Орфословарь РАН: " + (f"помета «{note}» → classify={classify(w, note)}"
|
||||
if note is not None else "нет заголовка"))
|
||||
reason = r["fate"].get(w)
|
||||
if reason is None:
|
||||
if w in r["scrabble"]:
|
||||
return "scrabble: лексикон OpenCorpora" if w in r["oc"] else "scrabble: производная/лемма"
|
||||
if w not in r["all"]:
|
||||
return "нет в russian_all (не извлечено на Stage 1 — нет в .pdf, либо имя собств./дефис/форма)"
|
||||
if not cyr_ok(w):
|
||||
return "отсеяно: длина или символы вне диапазона (2–15 кириллица)"
|
||||
return "не определено"
|
||||
reason = "лексикон OpenCorpora (обычное сущ.)"
|
||||
elif oc_abbr_fixd_only(w) and w not in r["all"]:
|
||||
reason = "исключено из посева: несклон. аббрев. (Abbr+Fixd), нет в all.txt"
|
||||
elif not parses and w not in r["all"]:
|
||||
reason = "нет ни в OpenCorpora, ни в all.txt"
|
||||
else:
|
||||
reason = "не прошло ни один путь отбора"
|
||||
out.append(f" ИТОГ: {reason}")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def main():
|
||||
@@ -320,7 +466,7 @@ def main():
|
||||
|
||||
r = build()
|
||||
if args.trace:
|
||||
print(f"{args.trace}: {trace(args.trace, r)}")
|
||||
print(trace(args.trace, r))
|
||||
return
|
||||
|
||||
write(os.path.join(OUT_DIR, "scrabble.txt"), r["scrabble"])
|
||||
|
||||
Reference in New Issue
Block a user