NightPrince/Quran-Semantic-Retrieval

1import re2 3 4def remove_tashkeel(text):5    tashkeel_pattern = re.compile(6        r'[\u0617-\u061A\u064B-\u0652]'7    )8    return re.sub(tashkeel_pattern, '', text)9 10 11def normalize_arabic(text):12    text = re.sub("[إأآا]", "ا", text)13    text = re.sub("ى", "ي", text)14    text = re.sub("ؤ", "و", text)15    text = re.sub("ئ", "ي", text)16    text = re.sub("ة", "ه", text)17    return text18 19 20def preprocess_arabic(text):21    text = remove_tashkeel(text)22    text = normalize_arabic(text)23    return text24 25 26def remove_tafsir_stopwords(query):27    # Strip trailing punctuation first28    cleaned = re.sub(r'[\?؟\.،]+$', '', query).strip()29    30    # 1. Strip trailing author names (e.g. "عند الطبري")31    author_pattern = r'\s+(?:عند|حسب|تفسير)\s+(?:الطبري|ابن\s*كثير|القرطبي|السعدي|البغوي|الجلالين|الميسر)$'32    cleaned = re.sub(author_pattern, '', cleaned).strip()33 34    # 2. Main Prefix Stripper35    prefix_pattern = r'^(?:(?:ما\s+)?(?:تفسير|معنى|معني|شرح|مقصود|المقصود|المراد)\s+(?:بـ|في\s+)?(?:قوله\s+تعالى|قوله\s+تعالي|قوله\s+عز\s*وجل|آية|اية|سورة)?\s*)' \36                     r'|^(?:ماذا\s+قال\s+المفسرون\s+في\s+قوله\s+تعالى\s*)' \37                     r'|^(?:كيف\s+فسر\s+.*?\s+آية\s+)' \38                     r'|^(?:أقوال\s+العلماء\s+في\s+تفسير\s*)' \39                     r'|^(?:تفسير\s+(?:آية|اية|سورة)\s*)' \40                     r'|^(?:تفسير\s+)' \41                     r'|^(?:شرح\s+(?:آية|اية)\s*)'42    43    # Needs to be stripped repeatedly to peel layers like "تفسير آية"44    old = ""45    while old != cleaned:46        old = cleaned47        cleaned = re.sub(prefix_pattern, '', cleaned, flags=re.IGNORECASE).strip()48        49    return cleaned50 51 52def bm25_tokenize(text):53    return preprocess_arabic(text).split()54 55 56def preprocess_query(query):57    query = preprocess_arabic(query)58    return bm25_tokenize(query)59