Spaces:
Sleeping
Sleeping
| # ## Some functions to clean text | |
| # ### Some other suggested cleaning approaches | |
| # | |
| # #### From here: https://shravan-kuchkula.github.io/topic-modeling/#interactive-plot-showing-results-of-k-means-clustering-lda-topic-modeling-and-sentiment-analysis | |
| # | |
| # - remove_hyphens | |
| # - tokenize_text | |
| # - remove_special_characters | |
| # - convert to lower case | |
| # - remove stopwords | |
| # - lemmatize the token | |
| # - remove short tokens | |
| # - keep only words in wordnet | |
| # - I ADDED ON - creating custom stopwords list | |
| # + | |
| # Create a custom stop words list | |
| import nltk | |
| import re | |
| import string | |
| from nltk.stem import WordNetLemmatizer | |
| from nltk.stem import PorterStemmer | |
| from nltk.corpus import wordnet as wn | |
| from nltk import word_tokenize | |
| # Add calendar months onto stop words | |
| import calendar | |
| from tqdm import tqdm | |
| import gradio as gr | |
| stemmer = PorterStemmer() | |
| nltk.download('stopwords') | |
| nltk.download('wordnet') | |
| #nltk.download('words') | |
| #nltk.download('names') | |
| #nltk.corpus.words.words('en') | |
| #from sklearn.feature_extraction import text | |
| # Adding common names to stopwords | |
| all_names = [x.lower() for x in list(nltk.corpus.names.words())] | |
| # Adding custom words to the stopwords | |
| custom_words = [] | |
| my_stop_words = custom_words | |
| cal_month = (list(calendar.month_name)) | |
| cal_month = [x.lower() for x in cal_month] | |
| # Remove blanks | |
| cal_month = [i for i in cal_month if i] | |
| #print(cal_month) | |
| custom_words.extend(cal_month) | |
| #my_stop_words = frozenset(text.ENGLISH_STOP_WORDS.union(custom_words).union(all_names)) | |
| #custom_stopwords = my_stop_words | |
| # - | |
| # #### Some of my cleaning functions | |
| ''' | |
| # + | |
| # Remove all html elements from the text. Inspired by this: https://stackoverflow.com/questions/9662346/python-code-to-remove-html-tags-from-a-string | |
| def remove_email_start(text): | |
| cleanr = re.compile('.*importance:|.*subject:') | |
| cleantext = re.sub(cleanr, '', text) | |
| return cleantext | |
| def remove_email_end(text): | |
| cleanr = re.compile('kind regards.*|many thanks.*|sincerely.*') | |
| cleantext = re.sub(cleanr, '', text) | |
| return cleantext | |
| def cleanhtml(text): | |
| cleanr = re.compile('<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});|\xa0') | |
| cleantext = re.sub(cleanr, '', text) | |
| return cleantext | |
| ## The above doesn't work when there is no > at the end of the string to match the initial <. Trying this: <[^>]+> but needs work: https://stackoverflow.com/questions/2013124/regex-matching-up-to-the-first-occurrence-of-a-character | |
| # Remove all email addresses and numbers from the text | |
| def cleanemail(text): | |
| cleanr = re.compile('\S*@\S*\s?|\xa0') | |
| cleantext = re.sub(cleanr, '', text) | |
| return cleantext | |
| def cleannum(text): | |
| cleanr = re.compile(r'[0-9]+') | |
| cleantext = re.sub(cleanr, '', text) | |
| return cleantext | |
| def cleanpostcode(text): | |
| cleanr = re.compile(r'(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9][A-Z]{2})|((GIR ?0A{2})\b$)|(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9]{1}?)$)|(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?)\b$)') | |
| cleantext = re.sub(cleanr, '', text) | |
| return cleantext | |
| def cleanwarning(text): | |
| cleanr = re.compile('caution: this email originated from outside of the organization. do not click links or open attachments unless you recognize the sender and know the content is safe.') | |
| cleantext = re.sub(cleanr, '', text) | |
| return cleantext | |
| # - | |
| def initial_clean(texts): | |
| clean_texts = [] | |
| for text in texts: | |
| text = remove_email_start(text) | |
| text = remove_email_end(text) | |
| text = cleanpostcode(text) | |
| text = remove_hyphens(text) | |
| text = cleanhtml(text) | |
| text = cleanemail(text) | |
| #text = cleannum(text) | |
| clean_texts.append(text) | |
| return clean_texts | |
| ''' | |
| # Pre-compiling the regular expressions for efficiency | |
| email_start_pattern = re.compile('.*importance:|.*subject:') | |
| email_end_pattern = re.compile('kind regards.*|many thanks.*|sincerely.*') | |
| html_pattern = re.compile('<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});|\xa0') | |
| email_pattern = re.compile('\S*@\S*\s?') | |
| num_pattern = re.compile(r'[0-9]+') | |
| postcode_pattern = re.compile(r'(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9][A-Z]{2})|((GIR ?0A{2})\b$)|(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9]{1}?)$)|(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?)\b$)') | |
| warning_pattern = re.compile('caution: this email originated from outside of the organization. do not click links or open attachments unless you recognize the sender and know the content is safe.') | |
| nbsp_pattern = re.compile(r' ') | |
| def stem_sentence(sentence): | |
| words = sentence.split() | |
| stemmed_words = [stemmer.stem(word).lower().rstrip("'") for word in words] | |
| return stemmed_words | |
| def stem_sentences(sentences, progress=gr.Progress()): | |
| """Stem each sentence in a list of sentences.""" | |
| stemmed_sentences = [stem_sentence(sentence) for sentence in progress.tqdm(sentences)] | |
| return stemmed_sentences | |
| def get_lemma_text(text): | |
| # Tokenize the input string into words | |
| tokens = word_tokenize(text) | |
| lemmas = [] | |
| for word in tokens: | |
| if len(word) > 3: | |
| lemma = wn.morphy(word) | |
| else: | |
| lemma = None | |
| if lemma is None: | |
| lemmas.append(word) | |
| else: | |
| lemmas.append(lemma) | |
| return lemmas | |
| def get_lemma_tokens(tokens): | |
| # Tokenize the input string into words | |
| lemmas = [] | |
| for word in tokens: | |
| if len(word) > 3: | |
| lemma = wn.morphy(word) | |
| else: | |
| lemma = None | |
| if lemma is None: | |
| lemmas.append(word) | |
| else: | |
| lemmas.append(lemma) | |
| return lemmas | |
| def initial_clean(texts , progress=gr.Progress()): | |
| clean_texts = [] | |
| i = 1 | |
| #progress(0, desc="Cleaning texts") | |
| for text in progress.tqdm(texts, desc = "Cleaning data", unit = "rows"): | |
| #print("Cleaning row: ", i) | |
| text = re.sub(email_start_pattern, '', text) | |
| text = re.sub(email_end_pattern, '', text) | |
| text = re.sub(postcode_pattern, '', text) | |
| text = remove_hyphens(text) | |
| text = re.sub(html_pattern, '', text) | |
| text = re.sub(email_pattern, '', text) | |
| text = re.sub(nbsp_pattern, '', text) | |
| #text = re.sub(warning_pattern, '', text) | |
| #text = stem_sentence(text) | |
| text = get_lemma_text(text) | |
| text = ' '.join(text) | |
| # Uncomment the next line if you want to remove numbers as well | |
| # text = re.sub(num_pattern, '', text) | |
| clean_texts.append(text) | |
| i += 1 | |
| return clean_texts | |
| # Sample execution | |
| #sample_texts = [ | |
| # "Hello, this is a test email. kind regards, John", | |
| # "<div>Email content here</div> many thanks, Jane", | |
| # "caution: this email originated from outside of the organization. do not click links or open attachments unless you recognize the sender and know the content is safe.", | |
| # "[email protected]", | |
| # "Address: 1234 Elm St, AB12 3CD" | |
| #] | |
| #initial_clean(sample_texts) | |
| # + | |
| all_names = [x.lower() for x in list(nltk.corpus.names.words())] | |
| def remove_hyphens(text_text): | |
| return re.sub(r'(\w+)-(\w+)-?(\w)?', r'\1 \2 \3', text_text) | |
| # tokenize text | |
| def tokenize_text(text_text): | |
| TOKEN_PATTERN = r'\s+' | |
| regex_wt = nltk.RegexpTokenizer(pattern=TOKEN_PATTERN, gaps=True) | |
| word_tokens = regex_wt.tokenize(text_text) | |
| return word_tokens | |
| def remove_characters_after_tokenization(tokens): | |
| pattern = re.compile('[{}]'.format(re.escape(string.punctuation))) | |
| filtered_tokens = filter(None, [pattern.sub('', token) for token in tokens]) | |
| return filtered_tokens | |
| def convert_to_lowercase(tokens): | |
| return [token.lower() for token in tokens if token.isalpha()] | |
| def remove_stopwords(tokens, custom_stopwords): | |
| stopword_list = nltk.corpus.stopwords.words('english') | |
| stopword_list += my_stop_words | |
| filtered_tokens = [token for token in tokens if token not in stopword_list] | |
| return filtered_tokens | |
| def remove_names(tokens): | |
| stopword_list = list(nltk.corpus.names.words()) | |
| stopword_list = [x.lower() for x in stopword_list] | |
| filtered_tokens = [token for token in tokens if token not in stopword_list] | |
| return filtered_tokens | |
| def remove_short_tokens(tokens): | |
| return [token for token in tokens if len(token) > 3] | |
| def keep_only_words_in_wordnet(tokens): | |
| return [token for token in tokens if wn.synsets(token)] | |
| def apply_lemmatize(tokens, wnl=WordNetLemmatizer()): | |
| def lem_word(word): | |
| if len(word) > 3: out_word = wnl.lemmatize(word) | |
| else: out_word = word | |
| return out_word | |
| return [lem_word(token) for token in tokens] | |
| # + | |
| ### Do the cleaning | |
| def cleanTexttexts(texts): | |
| clean_texts = [] | |
| for text in texts: | |
| #text = remove_email_start(text) | |
| #text = remove_email_end(text) | |
| text = remove_hyphens(text) | |
| text = cleanhtml(text) | |
| text = cleanemail(text) | |
| text = cleanpostcode(text) | |
| text = cleannum(text) | |
| #text = cleanwarning(text) | |
| text_i = tokenize_text(text) | |
| text_i = remove_characters_after_tokenization(text_i) | |
| #text_i = remove_names(text_i) | |
| text_i = convert_to_lowercase(text_i) | |
| #text_i = remove_stopwords(text_i, my_stop_words) | |
| text_i = get_lemma(text_i) | |
| #text_i = remove_short_tokens(text_i) | |
| text_i = keep_only_words_in_wordnet(text_i) | |
| text_i = apply_lemmatize(text_i) | |
| clean_texts.append(text_i) | |
| return clean_texts | |
| # - | |
| def remove_dups_text(data_samples_ready, data_samples_clean, data_samples): | |
| # Identify duplicates in the data: https://stackoverflow.com/questions/44191465/efficiently-identify-duplicates-in-large-list-500-000 | |
| # Only identifies the second duplicate | |
| seen = set() | |
| dupes = [] | |
| for i, doi in enumerate(data_samples_ready): | |
| if doi not in seen: | |
| seen.add(doi) | |
| else: | |
| dupes.append(i) | |
| #data_samples_ready[dupes[0:]] | |
| # To see a specific duplicated value you know the position of | |
| #matching = [s for s in data_samples_ready if data_samples_ready[83] in s] | |
| #matching | |
| # Remove duplicates only (keep first instance) | |
| #data_samples_ready = list( dict.fromkeys(data_samples_ready) ) # This way would keep one version of the duplicates | |
| ### Remove all duplicates including original instance | |
| # Identify ALL duplicates including initial values | |
| # https://stackoverflow.com/questions/11236006/identify-duplicate-values-in-a-list-in-python | |
| from collections import defaultdict | |
| D = defaultdict(list) | |
| for i,item in enumerate(data_samples_ready): | |
| D[item].append(i) | |
| D = {k:v for k,v in D.items() if len(v)>1} | |
| # https://stackoverflow.com/questions/952914/how-to-make-a-flat-list-out-of-a-list-of-lists | |
| L = list(D.values()) | |
| flat_list_dups = [item for sublist in L for item in sublist] | |
| # https://stackoverflow.com/questions/11303225/how-to-remove-multiple-indexes-from-a-list-at-the-same-time | |
| for index in sorted(flat_list_dups, reverse=True): | |
| del data_samples_ready[index] | |
| del data_samples_clean[index] | |
| del data_samples[index] | |
| # Remove blanks | |
| data_samples_ready = [i for i in data_samples_ready if i] | |
| data_samples_clean = [i for i in data_samples_clean if i] | |
| data_samples = [i for i in data_samples if i] | |
| return data_samples_ready, data_samples_clean, flat_list_dups, data_samples | |