syllabreak

13class Syllabreak:
14    """Syllabifies text and detects the most likely rule languages.
15
16    Args:
17        soft_hyphen: Separator inserted at syllable boundaries. Defaults to
18            the Unicode soft hyphen.
19    """
20
21    def __init__(self, soft_hyphen: str = "\u00ad"):
22        self.soft_hyphen = soft_hyphen
23        self.meta_rule = self._load_rules()
24
25    def _load_rules(self) -> MetaRule:
26        rules_file = Path(__file__).parent / "data" / "rules.yaml"
27        with open(rules_file, encoding="utf-8") as f:
28            data = yaml.safe_load(f)
29        rules = [LanguageRule(rule_data) for rule_data in data["rules"]]
30        return MetaRule(rules)
31
32    def detect_language(self, text: str) -> list[str]:
33        """Return matching language codes in descending confidence order.
34
35        Returns an empty list when the text contains no characters known to
36        any loaded rule.
37        """
38        matching_rules = self.meta_rule.find_matches(unicodedata.normalize("NFC", text))
39        return [rule.lang for rule in matching_rules]
40
41    def supported_languages(self) -> list[str]:
42        """Codes of every language the loaded rules cover, in rule-file order."""
43        return [rule.lang for rule in self.meta_rule.rules]
44
45    def _auto_detect_rule(self, text: str) -> LanguageRule | None:
46        matching_rules = self.meta_rule.find_matches(text)
47        return matching_rules[0] if matching_rules else None
48
49    def _get_rule_by_lang(self, lang: str) -> LanguageRule:
50        for rule in self.meta_rule.rules:
51            if rule.lang == lang:
52                return rule
53        raise ValueError(f"Language '{lang}' is not supported")
54
55    def syllabify(self, text: str, lang: str | None = None) -> str:
56        """Syllabify text by inserting soft hyphens at syllable boundaries.
57
58        Args:
59            text: Text to syllabify
60            lang: Optional language code (e.g., 'eng', 'srp-latn'). If not provided, auto-detects.
61
62        Raises:
63            ValueError: If specified language is not supported
64        """
65        if not text:
66            return text
67
68        if lang:
69            rule = self._get_rule_by_lang(lang)
70        else:
71            rule = self._auto_detect_rule(unicodedata.normalize("NFC", text))
72            if not rule:
73                return text
74
75        nfd_text = unicodedata.normalize("NFD", text)
76
77        result = []
78        i = 0
79        while i < len(nfd_text):
80            if not nfd_text[i].isalpha():
81                result.append(nfd_text[i])
82                i += 1
83                continue
84
85            word_start = i
86            while i < len(nfd_text) and rule.is_word_char(nfd_text[i]):
87                i += 1
88
89            word = nfd_text[word_start:i]
90            result.append(WordSyllabifier(word, rule, self.soft_hyphen).syllabify())
91
92        return unicodedata.normalize("NFC", "".join(result))

51class WordSplitter:
52    """Splits text into words per language. Rules loaded from yaml on init."""
53
54    def __init__(self) -> None:
55        self._modes = _load_rules()
56
57    def split(self, text: str, lang: str) -> list[str]:
58        """Return the visible words in text using the language's split mode."""
59        return [text[s:e] for s, e in self.find_ranges(text, lang)]
60
61    def find_ranges(self, text: str, lang: str) -> list[tuple[int, int]]:
62        """Like `split` but returns (start, end) character offsets — needed
63        by clients that highlight or annotate specific word positions in the
64        original text (e.g. iOS lexeme spans), where re-searching the surface
65        form would be ambiguous on repeats ("the cat sat on the mat")."""
66        regex_obj = _CJK_WORD_RE if self._modes.get(lang) == "cjk" else _DEFAULT_WORD_RE
67        return [m.span() for m in regex_obj.finditer(text)]