Delete [uma_namboothiripad]assignment_2.py

[uma_namboothiripad]assignment_2.py

@@ -1,226 +0,0 @@
-# -*- coding: utf-8 -*-
-"""[Uma Namboothiripad]Assignment_2.ipynb
-
-Automatically generated by Colaboratory.
-
-Original file is located at
-    https://colab.research.google.com/drive/1_sofOjXRDnId49NOup4sdiVS1E_51T-b
-
-Load the dataset below
-"""
-
-!pip install -U spacy
-#first install the library that would help us use BERT in an easy to use interface
-#https://github.com/UKPLab/sentence-transformers/tree/master/sentence_transformers
-!pip install -U sentence-transformers
-
-"""I was having issues connecting my csv file to the colab notebook, so I ended up connecting this to my drive"""
-
-import spacy
-from spacy.lang.en.stop_words import STOP_WORDS
-from string import punctuation
-from collections import Counter
-from heapq import nlargest
-from google.colab import drive
-drive.mount('/content/drive')
-
-import pandas as pd
-from tqdm import tqdm
-from sentence_transformers import SentenceTransformer, util
-
-! pip install -q kaggle
-
-! pip install lightgbm
-
-"""Setup Kaggle json credentials"""
-
-from google.colab import files
-files.upload()
-
-!mkdir ~/.kaggle/
-!cp kaggle.json ~/.kaggle/
-
-!chmod 600 ~/.kaggle/kaggle.json
-
-!kaggle datasets list
-
-!kaggle datasets download -d hamzafarooq50/hotel-listings-and-reviews/HotelListInBarcelona__en2019100120191005.csv
-
-!ls
-
-!python -m spacy download en_core_web_sm
-
-!kaggle datasets download --force -d hamzafarooq50/hotel-listings-and-reviews/hotelReviewsInBarcelona__en2019100120191005.csv
-
-!ls
-
-nlp = spacy.load("en_core_web_sm")
-
-import re
-
-import nltk
-nltk.download('punkt')
-
-from nltk.tokenize import word_tokenize
-
-nltk.download('stopwords')
-
-from nltk.corpus import stopwords
-
-nltk.download('wordnet')
-nltk.download('omw-1.4')
-from nltk.stem import WordNetLemmatizer
-from nltk.stem import WordNetLemmatizer
-
-import os
-import spacy
-nlp = spacy.load("en_core_web_sm")
-from spacy import displacy
-
-text = """Example text"""
-#text = "I really hope that France does not win the World Cup and Morocco makes it to the finals"
-doc = nlp(text)
-sentence_spans = list(doc.sents)
-displacy.render(doc, jupyter = True, style="ent")
-
-stopwords = list(STOP_WORDS)
-from string import punctuation
-punctuation = punctuation+ '\n'
-
-import pandas as pd
-
-import scipy.spatial
-import pickle as pkl
-
-!pip install -U sentence-transformers
-
-from sentence_transformers import SentenceTransformer
-embedder = SentenceTransformer('all-MiniLM-L6-v2')
-#embedder = SentenceTransformer('bert-base-nli-mean-tokens')
-
-!pip install -U sentence-transformers
-
-from sentence_transformers import SentenceTransformer
-
-embedder = SentenceTransformer('all-MiniLM-L6-v2')
-embedder = SentenceTransformer('bert-base-nli-mean-tokens')
-
-df = pd.read_csv('/content/drive/My Drive/Colab Notebooks/Assignment#2/HotelListInBarcelona__en2019100120191005.csv',sep=",", encoding='cp1252')
-
-!kaggle datasets download --force -d hamzafarooq50/hotel-listings-and-reviews
-
-df.head()
-
-df['hotel_name'].value_counts()
-
-df['hotel_name'].drop_duplicates()
-
-df_combined = df.sort_values(['hotel_name']).groupby('hotel_name', sort=False).hotel_features.apply(''.join).reset_index(name='hotel_features')
-
-df_combined.head().T
-
-import re
-
-df_combined['hotel_features'] = df_combined['hotel_features'].apply(lambda x: re.sub('[^a-zA-z0-9\s]','',x))
-
-def lower_case(input_str):
-    input_str = input_str.lower()
-    return input_str
-
-df_combined['hotel_features']= df_combined['hotel_features'].apply(lambda x: lower_case(x))
-
-df = df_combined
-
-df_sentences = df_combined.set_index("hotel_features")
-df_sentences = df_sentences["hotel_name"].to_dict()
-df_sentences_list = list(df_sentences.keys())
-len(df_sentences_list)
-
-list(df_sentences.keys())[:5]
-
-df_sentences_list = [str(d) for d in tqdm(df_sentences_list)]
-
-# Corpus with example sentences
-corpus = df_sentences_list
-corpus_embeddings = embedder.encode(corpus,show_progress_bar=True)
-
-corpus_embeddings[0]
-
-queries = ['Hotel near tourist locations and with free WIFI',
-           ]
-query_embeddings = embedder.encode(queries,show_progress_bar=True)
-
-import torch
-# Query sentences:
-queries = ['Hotel at least 10 minutes away from sagrada familia'
-           ]
-
-
-# Find the closest 3 sentences of the corpus for each query sentence based on cosine similarity
-top_k = min(3, len(corpus))
-for query in queries:
-    query_embedding = embedder.encode(query, convert_to_tensor=True)
-
-    # We use cosine-similarity and torch.topk to find the highest 5 scores
-    cos_scores = util.pytorch_cos_sim(query_embedding, corpus_embeddings)[0]
-    top_results = torch.topk(cos_scores, k=top_k)
-
-    print("\n\n======================\n\n")
-    print("Query:", query)
-    print("\nTop 3 most similar sentences in corpus:")
-
-    for score, idx in zip(top_results[0], top_results[1]):
-        print("(Score: {:.4f})".format(score))
-        print(corpus[idx], "(Score: {:.4f})".format(score))
-        row_dict = df.loc[df['hotel_features']== corpus[idx]]
-        print("paper_id:  " , row_dict['hotel_name'] , "\n")
-    # for idx, distance in results[0:closest_n]:
-    #     print("Score:   ", "(Score: %.4f)" % (1-distance) , "\n" )
-    #     print("Paragraph:   ", corpus[idx].strip(), "\n" )
-    #     row_dict = df.loc[df['all_review']== corpus[idx]]
-    #     print("paper_id:  " , row_dict['Hotel'] , "\n")
-
-model = SentenceTransformer('sentence-transformers/paraphrase-xlm-r-multilingual-v1')
-embeddings = model.encode(corpus)
-#print(embeddings)
-
-query_embedding.shape
-
-# Query sentences:
-queries = ['Hotel at least 10 minutes away from good food',
-           'quiet'
-           ]
-
-
-# Find the closest 5 sentences of the corpus for each query sentence based on cosine similarity
-top_k = min(5, len(corpus))
-for query in queries:
-
-    query_embedding = model.encode(query, convert_to_tensor=True)
-
-    # We use cosine-similarity and torch.topk to find the highest 5 scores
-    cos_scores = util.pytorch_cos_sim(query_embedding, embeddings)[0]
-    top_results = torch.topk(cos_scores, k=top_k)
-
-    print("\n\n======================\n\n")
-    print("Query:", query)
-    print("\nTop 5 most similar sentences in corpus:")
-
-    for score, idx in zip(top_results[0], top_results[1]):
-        print("(Score: {:.4f})".format(score))
-        print(corpus[idx], "(Score: {:.4f})".format(score))
-        row_dict = df.loc[df['hotel_features']== corpus[idx]]
-        print("paper_id:  " , row_dict['hotel_name'] , "\n")
-
-df
-
-hits = util.semantic_search(query_embedding, embeddings, top_k=5)
-hits = hits[0]      #Get the hits for the first query
-for hit in hits:
-  print (hit)
-  print("(Score: {:.4f})".format(hit['score']))
-  print(corpus[hit['corpus_id']])
-  row_dict = df.loc[df['hotel_features']== corpus[hit['corpus_id']]]
-  print("paper_id:  " , row_dict['hotel_name'] , "\n")
-!pip freeze > requirements.txt
-