Snizhanna/sarcasm_detection

1from nltk.tokenize import TweetTokenizer
2import stanza
3import re
4
5tk = TweetTokenizer()
6uk_nlp = stanza.Pipeline(lang='uk', verbose=False)
7
8def substitute_user_mentions_and_links(text):
9    # Regular expression to match user mentions (e.g., @username)
10    user_mention_pattern = r'@\w+'
11
12    # Regular expression to match links (e.g., http://example.com)
13    link_pattern = r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
14
15    # Substitute user mentions 
16    text = re.sub(user_mention_pattern, '', text)
17
18    # Substitute links 
19    text = re.sub(link_pattern, '', text)
20
21    # Substitute latin chars
22    text = re.sub(r'[a-zA-Z]+', '', text)
23
24    return text.lower()
25
26def remove_some_punc_numbers(text):
27    chars_to_remove = r'[\#\$\%\&\*\+\,\-\/\:\;\<\=\>\@\[\\\]\^\_\{\|\}\~\d\.\–]'
28
29    result = re.sub(chars_to_remove, '', ' '.join(text))
30
31    return result.lower()
32
33pattern = r'\b(\w+)\s*\'\s*(\w+)\b'
34
35# Define a function to join words separated by single quotes
36def join_words(match):
37    return match.group(1) + "'" + match.group(2)
38
39def lemmatize(text):
40    lemmas_st = []
41    for sent in uk_nlp(text).sentences:
42        for word in sent.words:
43            lemmas_st.append(word.lemma)
44    return lemmas_st
45
46def preprocess_text(input_text):
47
48    text_mod = substitute_user_mentions_and_links(input_text)
49    tokenized = tk.tokenize(text_mod)
50    spec_char_remv = remove_some_punc_numbers(tokenized)
51    apostrophe_fixed = re.sub(pattern, join_words, spec_char_remv)
52    spaces_fixed = re.sub(r'\s+', ' ', apostrophe_fixed)
53    lemmatized = lemmatize(spaces_fixed)
54
55    return text_mod, lemmatized
56