"""segment.py: list the multi-syllable Vietnamese words in a deck's visible text.

A browser breaks Vietnamese at every space, so it splits words like "hồ sơ" or
"đào tạo" across lines. build.mjs binds every word this script returns with
U+00A0, so a line can only break between words, as a Vietnamese typesetter would.

Input: JSON list of text strings on stdin. Output: JSON list of words, each a
space-joined run of 2+ syllables, longest first. Run with the skills venv:
    ~/.claude/skills/.venv/bin/python3 -I scripts/segment.py < texts.json
"""
import json
import re
import sys

from pyvi import ViTokenizer

# A syllable is a run of letters. Digits, codes and punctuation never join a word.
SYLLABLE = re.compile(r"^[^\W\d_]+$", re.UNICODE)

# pyvi sometimes glues a function word onto a real word ("thay thế thì").
# A glued run still cannot split, but it is longer than it needs to be, so trim
# function words off both edges and keep the word inside.
FUNCTION_WORDS = {
    "thì", "là", "mà", "của", "và", "với", "cho", "để", "được", "các", "những",
    "một", "này", "đó", "khi", "nếu", "đã", "sẽ", "đang", "rồi", "cũng", "vào",
    "ra", "ở", "từ", "đến", "theo", "trong", "trên", "dưới", "không", "có",
}


def words_in(text: str) -> set[str]:
    found = set()
    # pyvi joins the syllables of one word with "_".
    for token in ViTokenizer.tokenize(text).split():
        if "_" not in token:
            continue
        parts = token.split("_")
        if not all(SYLLABLE.match(p) for p in parts):
            continue
        while parts and parts[0].lower() in FUNCTION_WORDS:
            parts = parts[1:]
        while parts and parts[-1].lower() in FUNCTION_WORDS:
            parts = parts[:-1]
        if len(parts) >= 2:
            found.add(" ".join(parts))
    return found


DICT_PATH = __import__("pathlib").Path(__file__).resolve().parent.parent / "source" / "vendor" / "Viet74K.txt"


def load_dictionary() -> set[str]:
    """Multi-syllable entries of the Viet74K word list (GPL, see source/vendor/), lowercased."""
    out = set()
    for line in DICT_PATH.read_text(encoding="utf-8").splitlines():
        parts = line.strip().split()
        if 2 <= len(parts) <= 4 and all(SYLLABLE.match(p) for p in parts):
            out.add(" ".join(parts).lower())
    return out


def dictionary_words(text: str, dictionary: set[str]) -> set[str]:
    """Every 2- to 4-syllable run of the text that the dictionary lists as one word.
    Overlapping hits are all kept: gluing two words together never splits one."""
    found = set()
    for run in re.split(r"[^\w\s]|\d|_", text):  # punctuation and digits end a run
        syl = run.split()
        for n in (4, 3, 2):
            for i in range(len(syl) - n + 1):
                cand = syl[i:i + n]
                if all(SYLLABLE.match(s) for s in cand) and " ".join(cand).lower() in dictionary:
                    found.add(" ".join(cand))
    return found


def main() -> None:
    texts = json.load(sys.stdin)
    dictionary = load_dictionary()
    words = set()
    for t in texts:
        t = t.replace(" ", " ").strip()
        if t:
            words |= words_in(t)
            words |= dictionary_words(t, dictionary)
    json.dump(sorted(words, key=lambda w: (-len(w), w)), sys.stdout, ensure_ascii=False)


if __name__ == "__main__":
    main()