london_hotels.py

# -*- coding: utf-8 -*-
"""[Nicole_Lovold_Egar]Assignment 2.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1j5EomK04JF-H3FM_TdzSDk_US55nJd36
"""

import spacy
from spacy.lang.en.stop_words import STOP_WORDS
from string import punctuation
from collections import Counter
from heapq import nlargest

from google.colab import files
files.upload()

!mkdir ~/.kaggle/
!cp kaggle.json ~/.kaggle/

!chmod 600 ~/.kaggle/kaggle.json

!kaggle datasets list

!kaggle datasets download -d hamzafarooq50/hotel-listings-and-reviews/HotelListInLondon__en2019100120191005.csv

!ls

import re

import nltk
nltk.download('punkt')

from nltk.tokenize import word_tokenize

nltk.download('stopwords')

from nltk.corpus import stopwords

nltk.download('wordnet')
nltk.download('omw-1.4')
from nltk.stem import WordNetLemmatizer
from nltk.stem import WordNetLemmatizer

import os
import spacy
nlp = spacy.load("en_core_web_sm")
from spacy import displacy

text = """this is a base text"""

doc = nlp(text)
sentence_spans = list(doc.sents)
displacy.render(doc, jupyter = True, style="ent")

stopwords = list(STOP_WORDS)
from string import punctuation
punctuation = punctuation+ '\n'

import pandas as pd

import scipy.spatial
import pickle as pkl

!pip install -U sentence-transformers

from sentence_transformers import SentenceTransformer

embedder = SentenceTransformer('all-MiniLM-L6-v2')
#embedder = SentenceTransformer('bert-base-nli-mean-tokens')

df = pd.read_csv("/content/HotelListInLondon__en2019100120191005.csv",sep=",", encoding='cp1252')

!kaggle datasets download -d hamzafarooq50/hotel-listings-and-reviews

df.head()

import pandas as pd
from tqdm import tqdm
from sentence_transformers import SentenceTransformer, util

df['hotel_name'].value_counts()

df['hotel_name'].drop_duplicates()

df_combined = df.sort_values(['hotel_name']).groupby('hotel_name', sort=False).hotel_features.apply(''.join).reset_index(name='hotel_features')

df.iloc[6].hotel_features

df_combined.head().T

import re

df_combined['hotel_features'] = df_combined['hotel_features'].apply(lambda x: re.sub('[^a-zA-z0-9\s]','',x))

def lower_case(input_str):
    input_str = input_str.lower()
    return input_str

df_combined['hotel_features']= df_combined['hotel_features'].apply(lambda x: lower_case(x))

df = df_combined

df_sentences = df_combined.set_index("hotel_features")
df_sentences = df_sentences["hotel_name"].to_dict()
df_sentences_list = list(df_sentences.keys())
len(df_sentences_list)

list(df_sentences.keys())[:5]

import pandas as pd
from tqdm import tqdm
from sentence_transformers import SentenceTransformer, util

df_sentences_list = [str(d) for d in tqdm(df_sentences_list)]

# Converts each combined review and creates an embedding for each hotel
corpus = df_sentences_list
corpus_embeddings = embedder.encode(corpus,show_progress_bar=True)

corpus_embeddings[0].shape

queries = ['Hotel near St Pancras Station that have a spa']
query_embeddings = embedder.encode(queries,show_progress_bar=True)

import torch

# Query sentences:
queries = ['Hotel in london close to buckingham palace',
           'Hotel in london less than 200 per night']


# Find the closest 5 sentences of the corpus for each query sentence based on cosine similarity
top_k = min(5, len(corpus))
for query in queries:
    query_embedding = embedder.encode(query, convert_to_tensor=True)

    # We use cosine-similarity and torch.topk to find the highest 5 scores
    cos_scores = util.pytorch_cos_sim(query_embedding, corpus_embeddings)[0]
    top_results = torch.topk(cos_scores, k=top_k)

    print("\n\n======================\n\n")
    print("Query:", query)
    print("\nTop 5 most similar sentences in corpus:")

    for score, idx in zip(top_results[0], top_results[1]):
        print("(Score: {:.4f})".format(score))
        print(corpus[idx], "(Score: {:.4f})".format(score))
        row_dict = df.loc[df['hotel_features']== corpus[idx]]
        print("paper_id:  " , row_dict['hotel_name'] , "\n")
    # for idx, distance in results[0:closest_n]:
    #     print("Score:   ", "(Score: %.4f)" % (1-distance) , "\n" )
    #     print("Paragraph:   ", corpus[idx].strip(), "\n" )
    #     row_dict = df.loc[df['all_review']== corpus[idx]]
    #     print("paper_id:  " , row_dict['Hotel'] , "\n")

model = SentenceTransformer('sentence-transformers/paraphrase-xlm-r-multilingual-v1')
embeddings = model.encode(corpus)
#print(embeddings)

# Query sentences:
queries = ['Hotel near Kings Cross and good restaurants'
           ]

# Find the closest 5 sentences of the corpus for each query sentence based on cosine similarity
top_k = min(5, len(corpus))
for query in queries:
    
    query_embedding = model.encode(query, convert_to_tensor=True)

    # We use cosine-similarity and torch.topk to find the highest 5 scores
    cos_scores = util.pytorch_cos_sim(query_embedding, embeddings)[0]
    top_results = torch.topk(cos_scores, k=top_k)

    print("\n\n======================\n\n")
    print("Query:", query)
    print("\nTop 5 most similar sentences in corpus:")

    for score, idx in zip(top_results[0], top_results[1]):
        print("(Score: {:.4f})".format(score))
        print(corpus[idx], "(Score: {:.4f})".format(score))
        row_dict = df.loc[df['hotel_features']== corpus[idx]]
        print("paper_id:  " , row_dict['hotel_name'] , "\n")

hits = util.semantic_search(query_embedding, embeddings, top_k=5)
hits = hits[0]      #Get the hits for the first query
for hit in hits:
  print (hit)
  print("(Score: {:.4f})".format(hit['score']))
  print(corpus[hit['corpus_id']])
  row_dict = df.loc[df['hotel_features']== corpus[hit['corpus_id']]]
  print("paper_id:  " , row_dict['hotel_name'] , "\n")

query_embedding.shape

df

"""End"""