Chapimenge/amharic-gemination-lexicon

1"""Amharic gemination from HornMorpho analyses.2 3Amharic gemination (consonant doubling) changes meaning but is not written in4Ge'ez script. HornMorpho (Michael Gasser, https://github.com/hltdi/HornMorpho)5computes it while analysing a word, and keeps it in the `seg` field if and only6if you call7 8    hm.anal('a', word, degem=False)9 10With the default degem=True the marks are stripped. With degem=False, the11character `/` stands immediately before every geminated character of the12morphemic segmentation, whatever caused the doubling: the root template, the13passive/reciprocal prefix t- assimilating to the first root consonant, a14suffix such as the auxiliary -all, a lexicon entry, or reduplication.15 16Example segs (degem=False):17 18    ይገኛል  passive imperfect   ----ይ</ገ/ኝ>---ኣ/ል---   ገ (t- prefix), ኛ (root), ል (suffix)19    ክልል   noun                -<ክ/ልል>------          ል (lexicon)20    የሚገኙ  relative            የ--/ም--ይ</ገ/ኝ>ኡ------  ሚ (prefix), ገ (t- prefix), ኙ (root)21 22The seg is morphemic, not a copy of the surface word: vowels merge across23morpheme boundaries (ቀም + ኣል gives ቀማል), palatalisation changes letters24(ን + ኢ gives ኝ), and the glottal letter of a vowel-initial suffix disappears.25This module therefore aligns the seg letters to the surface word by consonant26series and moves each `/` to the surface character it aligned with.27 28Do not read gemination from the `+gemN` feature instead. It only describes the29root template, so it misses every geminate created by a prefix or suffix, and30in reduplicated stems N counts template slots, not root consonants.31 32Usage:33    python gemination.py ይገኛል ክልል መለወጥ          (needs HornMorpho, see INSTALL.md)34 35    >>> from gemination import reading_gemination36    >>> reading_gemination("ይገኛል", {"seg": "----ይ</ገ/ኝ>---ኣ/ል---"})["pos"]37    [1, 2]38 39Positions are 0-based indices into the word's Unicode characters (one Ge'ez40fidel per index).41 42Lexicon v3 rules (on by default; pass rules=False for the v2 behaviour). Each43is backed by more than one independent source, see the dataset card:44  causative_glottal  a causative prefix ስ- or ስተ- on a glottal-initial root45                     holds the next root consonant (ማሳደ፟ግ, አስታው፟ስ);46                     HornMorpho leaves it out of many segs47  relative_m_light   the relative prefix's ም before a subject prefix48                     (የሚ-, እንደሚ-, በሚ-, የምን-, የማይ-) is not held49  relative_exist     with relative የ-, the reading of አለ "exist" (ለ held)50                     is ranked above አለ "say" (ያለው, ያሉት)51"""52import difflib53import re54import sys55 56GEM_PRE = "/"57GEM_MARK = "፟"          # ETHIOPIC COMBINING GEMINATION MARK58BOUNDARY = set("-<>")59GEEZ = re.compile(r"[ሀ-፿]")60GLOTTAL = {(0x12A0 - 0x1200) // 8, (0x12D0 - 0x1200) // 8}      # አ and ዐ series61# palatalised surface letter -> base consonant it comes from62PALATAL = {"ሽ": "ስ", "ች": "ት", "ጭ": "ጥ", "ኝ": "ን", "ዥ": "ዝ", "ጅ": "ድ", "ይ": "ል"}63# letters pronounced the same in Amharic, spelled either way (ሣ/ሳ, ሐ/ሀ, ፀ/ጸ)64HOMOPHONE = {"ሥ": "ስ", "ሕ": "ህ", "ኅ": "ህ", "ፅ": "ጽ", "ኽ": "ህ"}65# labialised series (ቈ ኰ ጐ ኈ ዀ) are the base consonant plus w: ቅ + ዋ gives ቋ66LABIAL_SERIES = {0x1248: 0x1240, 0x12B0: 0x12A8, 0x1310: 0x1308, 0x1288: 0x1280, 0x12C0: 0x12B8}67 68SOURCES = ("root", "t-prefix", "prefix", "suffix", "lexicon", "reduplication")69 70 71def series(ch):72    return (ord(ch) - 0x1200) // 873 74 75def _key(ch):76    """Alignment key: consonant series, with labialised, homophone and palatal77    spellings folded to their base and both glottal series merged."""78    base = 0x1200 + series(ch) * 879    base = LABIAL_SERIES.get(base, base)80    sixth = chr(base + 5)81    sixth = HOMOPHONE.get(sixth, sixth)82    sixth = PALATAL.get(sixth, sixth)83    s = series(sixth)84    return "G" if s in GLOTTAL else s85 86 87def parse_seg(seg):88    """Seg string -> list of (char, geminated, region, source) for its Ge'ez89    characters. region: 'pre' before '<', 'stem' inside <...>, 'suf' after '>'.90 91    source (only for geminated characters) is inferred from the seg alone:92      prefix        '/' before '<' (for example the relative yä-m- in የሚ)93      suffix        '/' after '>' (the auxiliary -all, plural -očč, -nna)94      t-prefix      '/' opens the stem ('</X'), sits on a boundary ('/-X'),95                    or doubles an unassimilated ተ- ('/ተ-'): the passive or96                    reciprocal t- merging into the next consonant97      lexicon       stem without internal boundaries in a noun-shaped seg: an98                    unanalysed dictionary entry that stores the gemination99      reduplication the same consonant series occurs earlier in the same100                    morpheme (frequentative stems such as ገለባ/በጥ). Roots101                    whose second half repeats the first (ግልግል, ስብስብ) also102                    land here.103      root          any other stem geminate: the root's template104    """105    stem = seg[seg.find("<") + 1:seg.find(">")] if "<" in seg and ">" in seg else ""106    lexical = "<" in seg and "-" not in stem and seg[:seg.find("<")].count("-") <= 2107    out, pending, opens, region, last, morph = [], False, False, "pre", "", []108    for i, ch in enumerate(seg):109        if ch == GEM_PRE:110            pending = True111            nxt = seg[i + 1:i + 3]112            opens = last == "<" or nxt[:1] == "-" or nxt == "ተ-"113        elif ch == "<":114            region, morph = "stem", []115        elif ch == ">":116            region = "suf"117        elif ch == "-":118            if region == "stem":119                morph = []120        elif GEEZ.match(ch):121            src = None122            if pending:123                if region == "pre":124                    src = "prefix"125                elif region == "suf":126                    src = "suffix"127                elif opens:128                    src = "t-prefix"129                elif lexical:130                    src = "lexicon"131                elif series(ch) in {series(c) for c in morph}:132                    src = "reduplication"133                else:134                    src = "root"135            out.append((ch, pending, region, src))136            if region == "stem":137                morph.append(ch)138            pending = opens = False139        if ch != "-":140            last = ch141    return out142 143 144def seg_to_positions(word, seg):145    """-> ([(surface index, region, source)], ok). ok is False when a geminated146    seg character could not be aligned to the surface word."""147    segc = parse_seg(seg)148    a = [_key(c) for c, _, _, _ in segc]149    b = [_key(c) for c in word]150    sm = difflib.SequenceMatcher(None, a, b, autojunk=False)151    m = {}152    for tag, i1, i2, j1, j2 in sm.get_opcodes():153        if tag == "equal" or (tag == "replace" and i2 - i1 == j2 - j1):154            for k in range(i2 - i1):155                m[i1 + k] = j1 + k156    pos, ok = {}, True157    for i, (_, gem, region, src) in enumerate(segc):158        if not gem:159            continue160        if i in m:161            pos.setdefault(m[i], (region, src))162        else:163            ok = False164    return sorted((p, r, s) for p, (r, s) in pos.items()), ok165 166 167CAUSATIVE_GLOTTAL = re.compile(r"(ስተ?-ኣ)([ሀ-፿])")168RELATIVE_M = re.compile(r"/ም(?=-+(?:ይ|እን|ት|ኣ))")169 170 171def apply_rules(seg):172    """Apply the v3 seg rules. -> (new seg, names of the rules that fired)."""173    if not seg:174        return seg, []175    fired = []176    new = CAUSATIVE_GLOTTAL.sub(r"\1/\2", seg)177    if new != seg:178        fired.append("causative_glottal")179    if "<" in new:180        pre, rest = new.split("<", 1)181        light = RELATIVE_M.sub("ም", pre)182        if light != pre:183            fired.append("relative_m_light")184            new = light + "<" + rest185    return new, fired186 187 188def relative_exist_first(word, patterns):189    """v3: with relative የ-, put the existence reading of አለ (ለ held) first.190    Bare ያለ is left alone: it can also be privative 'without', which holds nothing."""191    if word == "ያለ":192        return patterns193    for i, p in enumerate(patterns):194        if any("<ኣ/ለ>" in s and s.split("<")[0].replace("-", "") == "የ" for s in p["segs"]):195            if i:196                p["rules"] = sorted(set(p.get("rules", [])) | {"relative_exist"})197                patterns = [p] + patterns[:i] + patterns[i + 1:]198            return patterns199    return patterns200 201 202def native_first(word, patterns, taps):203    """Put a native speaker's tapped positions first. taps: word -> positions."""204    if word not in taps:205        return patterns206    pos = sorted(taps[word])207    for i, p in enumerate(patterns):208        if p["pos"] == pos:209            p["native"] = True210            return [p] + patterns[:i] + patterns[i + 1:]211    return [{"pos": pos, "letters": [word[k] for k in pos], "sources": ["native"] * len(pos), "freq": None,212             "pos_tags": [], "ok": True, "segs": [], "rules": [], "native": True}] + patterns213 214 215def reading_gemination(word, reading, final="suffix", rules=True):216    """Geminated surface positions for one HornMorpho reading (a dict with a217    'seg' from hm.anal(..., degem=False)).218 219    final: what to do with a geminate on the last character.220      "suffix" (default) drop it only if it comes from a suffix and the last221               letter is sixth order, so no vowel follows. In practice this is222               the auxiliary -all of ይገኛል: it is not audible before a pause223               and Armbruster never marks it. Vowel-final suffixes (-allu) and224               stem-final geminates (ይሰ፟ጥ፟) are kept.225      "keep"   keep every geminate.226      "drop"   drop any word-final geminate.227    """228    seg = reading.get("seg") or ""229    used, fired = apply_rules(seg) if rules else (seg, [])230    items, ok = seg_to_positions(word, used)231    last = len(word) - 1232    if final == "suffix" and word:233        sixth = (ord(word[last]) - 0x1200) % 8 == 5234        items = [x for x in items if not (x[0] == last and x[1] == "suf" and sixth)]235    elif final == "drop":236        items = [x for x in items if x[0] != last]237    return {"pos": [p for p, _, _ in items], "letters": [word[p] for p, _, _ in items],238            "sources": [s for _, _, s in items], "ok": ok, "seg": seg, "rules": fired}239 240 241def word_gemination(word, readings, final="suffix", rules=True):242    """Distinct gemination patterns over all readings of a word, most frequent243    root first (HornMorpho's 'freq'). Readings that differ only in grammar but244    geminate the same positions share one pattern."""245    pats = {}246    for r in readings:247        g = reading_gemination(word, r, final, rules)248        f = r.get("freq") or 0249        e = pats.get(tuple(g["pos"]))250        if e is None:251            e = pats[tuple(g["pos"])] = {"pos": g["pos"], "letters": g["letters"], "sources": g["sources"],252                                         "freq": f, "pos_tags": set(), "ok": True, "segs": [], "rules": set()}253        elif f > e["freq"]:254            e["freq"], e["sources"] = f, g["sources"]255        e["pos_tags"].add(str(r.get("pos")))256        e["segs"].append(g["seg"])257        e["ok"] = e["ok"] and g["ok"]258        e["rules"].update(g["rules"])259    out = sorted(pats.values(), key=lambda e: -e["freq"])260    for e in out:261        e["pos_tags"], e["rules"] = sorted(e["pos_tags"]), sorted(e["rules"])262    return relative_exist_first(word, out) if rules else out263 264 265def mark(word, pos):266    """Write the gemination mark U+135F after each geminated character."""267    s = set(pos)268    return "".join(c + (GEM_MARK if i in s else "") for i, c in enumerate(word))269 270 271def analyze(word, final="suffix", rules=True):272    """Run HornMorpho on one word and return its gemination patterns."""273    import hm274    try:275        rs = list(hm.anal("a", word, degem=False))276    except Exception:277        rs = []278    return word_gemination(word, rs, final, rules)279 280 281if __name__ == "__main__":282    for w in sys.argv[1:] or ["ይገኛል", "ክልል", "መለወጥ"]:283        pats = analyze(w)284        if not pats:285            print(f"{w}\tno analysis")286        for p in pats:287            print(f"{w}\t{mark(w, p['pos'])}\tpositions={p['pos']}\tsources={p['sources']}\t"288                  f"pos={','.join(p['pos_tags'])}\troot_freq={p['freq']}\tseg={p['segs'][0]}")289