HebArabNlpProject/Semantic-Retrieval-2nd-place

1"""
2text_utils.py
3Single-source Hebrew normalization & tokenization.
4Controls behavior across all scripts.
5"""
6import re
7import unicodedata
8from typing import List
9
10HEB_PREFIXES = ("ו","ה","ב","ל","כ","מ","ש")
11STOPWORDS = set("""
12    אבל אם או אז אתה את אתם אתן אצל על עד עם אנחנו אני הוא היא הם הן אשר של
13    ולא לא כן כבר כאשר לכן לפני לאחר כדי עוד רק
14    אל זה זו אך כי גם כל כך בלי לפי וכן וכו וכ'
15""".split())
16
17
18# --- Core Function ---
19
20def identity(s: str) -> str:
21    """Does nothing"""
22    return s
23
24def norm_he(s: str) -> str:
25    """Current normalization implementation (bad)"""
26    if not s:
27        return ""
28    s = unicodedata.normalize("NFKC", s)
29    s = re.sub(r"[\u0591-\u05BD\u05BF-\u05C7]", "", s)  # strip nikkud
30    s = (s.replace("״", '"').replace("׳", "'")
31           .replace("”", '"').replace("“", '"')
32           .replace("–", "-").replace("—", "-"))
33    return re.sub(r"\s+", " ", s).strip()
34
35def tok_he(text: str) -> List[str]:
36    """The main tokenizer. It uses the BM25 normalizer internally."""
37    s = norm_bm25(text)  # Use the specific normalizer for BM25
38    toks = re.findall(r"[A-Za-z0-9\u0590-\u05FF]+", s)
39    
40    out: List[str] = []
41    for t in toks:
42        if len(t) > 3 and t[0] in HEB_PREFIXES:
43            out.append(t[1:])  # stripped prefix
44        out.append(t)
45    return [t for t in out if t not in STOPWORDS]
46
47
48# --- Component-Specific Assignments ---
49
50# For now, only BM25 gets real normalization.
51norm_bm25 = norm_he
52
53# For now, E5, Gemma and BGE inputs are passed through unchanged.
54norm_e5_query = identity
55norm_e5_passage = identity
56norm_gemma_query = identity
57norm_gemma_passage = identity
58norm_bge_query = identity
59norm_bge_passage = identity
60
61# --- General Aliases ---
62tokenize = tok_he
63normalize = norm_he # General normalize points to the strong one