alpertml/TopicModelingForSummarization

1import re2from nltk.tokenize import RegexpTokenizer3import spacy4 5def remove_patterns(text):6    """7        Remove punctions, emails, hashtags in given text8    """9 10    if isinstance(text, spacy.tokens.span.Span):11        text = text.text12    # Remove return char13    text = re.sub(r'\n', ' ', text)14    # Remove emails15    text = re.sub(r'\S*@\S*\s?', '', text)16    # Remove hashtags17    text = re.sub(r'#\w+', '', text)18    # Remove punctuation19    text = re.sub(r'[^\w\s]', '', text)20 21    return text22 23def extract_patterns(text):24    """25        Extract punctions, emails, hashtags in given text26    """27    # extract emails28    emails = re.findall(r'\S+@\S+', text)29    # extract hashtags30    hashtags = re.findall(r'#\w+', text)31    # extract punctuation32    punctuation = re.findall(r'[^\w\s]', text)33    34    return punctuation, emails, hashtags35 36def remove_punct_nltk(text):37    tokenizer = RegexpTokenizer(r'\w+')38    tokenizer.tokenize(text)39    return text