alainkh/arabic-misogyny-detector

1import re
2import json
3import emoji
4import pyarabic.araby as araby
5import tensorflow as tf
6from tensorflow.keras.preprocessing.text import tokenizer_from_json
7from tensorflow.keras.preprocessing.sequence import pad_sequences
8
9def load_tokenizer(path='tokenizer.json'):
10    with open(path, 'r', encoding='utf-8') as f:
11        data = json.load(f)
12        return tokenizer_from_json(data)
13
14TOKENIZER = load_tokenizer()
15
16def data_cleaning(text):
17
18  # For any foreign language or for links
19  text = text.lower()
20
21  # Removing all mentions with @
22  text = re.sub(r"@\w+", '', text)
23
24  # Remove all links
25  text = re.sub(r'https?:\/\/.*[\r\n]*', "", text, flags=re.MULTILINE)
26
27  # These are some issues that couldnt be removed so I had to manually force remove them
28  for char in ["مستخدم@", "#", "…", "RT", "\ufffd"]:
29    text = text.replace(char, "")
30
31  # Convert exclamation point and question mark into words, to give them more weight in the context
32  text = re.sub(r'!+', ' [EXCLAMATION] ', text)
33  text = re.sub(r'\?+', ' [QUESTION] ', text)
34
35  # Removed some useless characters
36  chars_to_remove = r'[!"#$%&\'()*+,-./:;<=>?@\[\\\]^_`{|}~،؛؟ـ٪٫٬«»“”•·…﴾﴿〈〉°±÷ש®™€£¥¢]'
37  text = re.sub(chars_to_remove, ' ', text)
38
39  # I am keeping everything on one line
40  text = re.sub(r'[\r\n]+', ' ', text)
41
42  # Normalizing some letters to unify some words and match more words
43  text = re.sub("[إأآا]", "ا", text)
44  text = re.sub("ى", "ي", text)
45  text = re.sub("ؤ", "ء", text)
46  text = re.sub("ئ", "ء", text)
47  text = re.sub("ة", "ه", text)
48  text = re.sub("گ", "ك", text)
49
50  # Remove extra spaces
51  text = re.sub(r'\s+', ' ', text)
52
53  # This is for underscores inside hashtags, so I am converting the hashtag into words
54  text = text.replace("_", " ")
55
56  # Since I dont have much data, I am converting emojis into text to also give more context to sentences
57  text = emoji.demojize(text, delimiters=(" ", " "))
58
59  # strip tashkeel and tatweel
60  text = araby.strip_tashkeel(text)
61  text = araby.strip_tatweel(text)
62
63  text = text.strip()
64  return text
65
66def prepare_input(text, max_len=48):
67    # Step 1: Clean
68    cleaned_text = data_cleaning(text)
69
70    # Step 2: Tokenize (Convert text to sequence of integers)
71    # Wrap cleaned_text in a list because texts_to_sequences expects a list of strings
72    sequence = TOKENIZER.texts_to_sequences([cleaned_text])
73
74    # Step 3: Pad (Ensure fixed length for the CNN-BiGRU architecture)
75    padded = pad_sequences(sequence, maxlen=max_len)
76
77    return padded
78