highlight and review relevance

AirbnbMapVisualiser.py

@@ -2,7 +2,6 @@ import oracledb
 import pandas as pd
 import folium
 from html import escape
-from transformers import AutoTokenizer
 import numpy as np
 from TrafficSpot import TrafficSpotManager
 
@@ -23,13 +22,10 @@ class AirbnbMapVisualiser:
             increment=1,
             getmode=oracledb.SPOOL_ATTRVAL_WAIT
         )
-        self.tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
         self.traffic_manager = TrafficSpotManager(self.connection_params)
 
         try:
-            # Get all neighborhoods first (this is a small query)
             self.neighborhoods = self.get_all_neighborhoods()
-            # Initialize with Sha Tin listings
             self.cached_listings = {}
             self.cached_listings["Sha Tin"] = self.get_neighborhood_listings("Sha Tin")
         except Exception as e:
@@ -40,17 +36,17 @@ class AirbnbMapVisualiser:
     def get_all_neighborhoods(self):
         connection = self.pool.acquire()
         try:
-                cursor = connection.cursor()
-                cursor.prefetchrows = 50
-                cursor.arraysize = 50
-                cursor.execute("""
-                    SELECT DISTINCT NEIGHBOURHOOD 
-                    FROM airbnb_master_data 
-                    WHERE NEIGHBOURHOOD IS NOT NULL 
-                    ORDER BY NEIGHBOURHOOD
-                """)
-                neighborhoods = [row[0] for row in cursor.fetchall()]
-                return neighborhoods
+            cursor = connection.cursor()
+            cursor.prefetchrows = 50
+            cursor.arraysize = 50
+            cursor.execute("""
+                SELECT DISTINCT NEIGHBOURHOOD 
+                FROM airbnb_master_data 
+                WHERE NEIGHBOURHOOD IS NOT NULL 
+                ORDER BY NEIGHBOURHOOD
+            """)
+            neighborhoods = [row[0] for row in cursor.fetchall()]
+            return neighborhoods
         except Exception as e:
             print(f"Database error getting neighborhoods: {str(e)}")
             return []
@@ -58,30 +54,33 @@ class AirbnbMapVisualiser:
             self.pool.release(connection)
 
     def get_neighborhood_listings(self, neighborhood):
-        connection = self.pool.acquire()
         if neighborhood in self.cached_listings:
             return self.cached_listings[neighborhood]
+
+        connection = self.pool.acquire()
         try:
-                cursor = connection.cursor()
-                cursor.prefetchrows = 50
-                cursor.arraysize = 50
-                cursor.execute("""
-                   SELECT m.ID, m.NAME, m.HOST_NAME, m.NEIGHBOURHOOD,
-m.LATITUDE, m.LONGITUDE, m.ROOM_TYPE, m.PRICE,
-COUNT(r.LISTING_ID) as NUMBER_OF_REVIEWS, m.REVIEWS_PER_MONTH,
-m.MINIMUM_NIGHTS, m.AVAILABILITY_365
-FROM airbnb_master_data m
-LEFT JOIN airbnb_reviews_data r ON m.ID = r.LISTING_ID
-WHERE m.LATITUDE IS NOT NULL AND m.LONGITUDE IS NOT NULL AND NEIGHBOURHOOD = :neighborhood AND ROWNUM <= 150
-GROUP BY m.ID, m.NAME, m.HOST_NAME, m.NEIGHBOURHOOD,
-m.LATITUDE, m.LONGITUDE, m.ROOM_TYPE, m.PRICE,
-m.REVIEWS_PER_MONTH, m.MINIMUM_NIGHTS, m.AVAILABILITY_365
-                    
-                """, neighborhood=neighborhood)
-                listings = cursor.fetchall()
-                # Cache the results
-                self.cached_listings[neighborhood] = listings
-                return listings
+            cursor = connection.cursor()
+            cursor.prefetchrows = 50
+            cursor.arraysize = 50
+            cursor.execute("""
+                SELECT m.ID, m.NAME, m.HOST_NAME, m.NEIGHBOURHOOD,
+                       m.LATITUDE, m.LONGITUDE, m.ROOM_TYPE, m.PRICE,
+                       COUNT(r.LISTING_ID) as NUMBER_OF_REVIEWS, m.REVIEWS_PER_MONTH,
+                       m.MINIMUM_NIGHTS, m.AVAILABILITY_365
+                FROM airbnb_master_data m
+                LEFT JOIN airbnb_reviews_data r ON m.ID = r.LISTING_ID
+                WHERE m.LATITUDE IS NOT NULL 
+                AND m.LONGITUDE IS NOT NULL 
+                AND m.NEIGHBOURHOOD = :neighborhood 
+                AND ROWNUM <= 150
+                GROUP BY m.ID, m.NAME, m.HOST_NAME, m.NEIGHBOURHOOD,
+                         m.LATITUDE, m.LONGITUDE, m.ROOM_TYPE, m.PRICE,
+                         m.REVIEWS_PER_MONTH, m.MINIMUM_NIGHTS, m.AVAILABILITY_365
+            """, neighborhood=neighborhood)
+
+            listings = cursor.fetchall()
+            self.cached_listings[neighborhood] = listings
+            return listings
         except Exception as e:
             print(f"Database error: {str(e)}")
             return []
@@ -89,112 +88,150 @@ m.REVIEWS_PER_MONTH, m.MINIMUM_NIGHTS, m.AVAILABILITY_365
             self.pool.release(connection)
 
     def get_listing_reviews(self, listing_id):
+        connection = self.pool.acquire()
         try:
-            with oracledb.connect(**self.connection_params) as conn:
-                cursor = conn.cursor()
-                cursor.execute("""
-                    SELECT REVIEW_DATE, REVIEWER_NAME, 
-                           CASE 
-                               WHEN LENGTH(COMMENTS) > 200 
-                               THEN SUBSTR(COMMENTS, 1, 200) || '...'
-                               ELSE COMMENTS 
-                           END as COMMENTS
-                    FROM AIRBNB_REVIEWS_DATA 
-                    WHERE LISTING_ID = :listing_id AND ROWNUM <= 50
-                    ORDER BY REVIEW_DATE DESC
-                """, listing_id=str(listing_id))
-                reviews = cursor.fetchall()
-
-                formatted_reviews = []
-                for review in reviews:
-                    review_date, reviewer_name, comments = review
-                    formatted_review = (
-                        str(review_date) if review_date else '',
-                        str(reviewer_name) if reviewer_name else '',
-                        str(comments) if comments else ''
-                    )
-                    formatted_reviews.append(formatted_review)
+            cursor = connection.cursor()
+            cursor.execute("""
+                SELECT REVIEW_DATE, REVIEWER_NAME, 
+                       CASE 
+                           WHEN LENGTH(COMMENTS) > 200 
+                           THEN SUBSTR(COMMENTS, 1, 200) || '...'
+                           ELSE COMMENTS 
+                       END as COMMENTS
+                FROM AIRBNB_REVIEWS_DATA 
+                WHERE LISTING_ID = :listing_id 
+                AND ROWNUM <= 50
+                ORDER BY REVIEW_DATE DESC
+            """, listing_id=int(listing_id))
+
+            reviews = cursor.fetchall()
+            formatted_reviews = []
+            for review in reviews:
+                review_date, reviewer_name, comments = review
+                formatted_review = (
+                    str(review_date) if review_date else '',
+                    str(reviewer_name) if reviewer_name else '',
+                    str(comments) if comments else ''
+                )
+                formatted_reviews.append(formatted_review)
+            return formatted_reviews
 
-                return formatted_reviews
-        except oracledb.DatabaseError as e:
-            print(f"Database error fetching reviews: {str(e)}")
-            return []
         except Exception as e:
             print(f"Error fetching reviews: {str(e)}")
             return []
+        finally:
+            self.pool.release(connection)
 
-    def preprocess_text(self, text):
-        if not isinstance(text, str):
-            return ""
-        tokens = self.tokenizer.tokenize(text)
-        cleaned_text = " ".join(tokens)
-        return cleaned_text
-
-    def create_listing_description(self, row):
-        description = f"""{row['name']}"""
-        return self.preprocess_text(description)
-
-    def compute_token_based_similarity(self, text1, text2):
-        tokens1 = set(self.tokenizer.tokenize(text1.lower()))
-        tokens2 = set(self.tokenizer.tokenize(text2.lower()))
-        intersection = len(tokens1.intersection(tokens2))
-        normalization_factor = min(len(tokens1), len(tokens2))
-
-        if normalization_factor == 0:
-            return 0
+    def get_listing_reviews_for_search(self, listing_id):
+        """Get reviews for search analysis"""
+        connection = self.pool.acquire()
+        try:
+            cursor = connection.cursor()
+            cursor.execute("""
+                SELECT COMMENTS
+                FROM AIRBNB_REVIEWS_DATA 
+                WHERE LISTING_ID = :listing_id 
+                AND COMMENTS IS NOT NULL
+                AND ROWNUM <= 10
+                ORDER BY REVIEW_DATE DESC
+            """, listing_id=int(listing_id))
+
+            reviews = cursor.fetchall()
+            return [review[0] for review in reviews if review[0]]
 
-        base_similarity = intersection / normalization_factor
-        boosted_score = 1 / (1 + np.exp(-6 * (base_similarity - 0.5)))
-        return boosted_score
+        except Exception as e:
+            print(f"Error fetching reviews for search: {str(e)}")
+            return []
+        finally:
+            self.pool.release(connection)
 
     def compute_search_scores(self, df, search_query):
-        processed_query = self.preprocess_text(search_query)
-        query_terms = set(processed_query.lower().split())
+        """Compute search scores based on name and review content"""
+        if not search_query:
+            return np.zeros(len(df))
 
+        search_query = search_query.lower()
+        search_terms = search_query.split()
         scores = []
-        for _, row in df.iterrows():
-            listing_desc = self.create_listing_description(row)
-            desc_terms = set(listing_desc.lower().split())
-
-            token_sim = self.compute_token_based_similarity(processed_query, listing_desc)
-
-            exact_match_bonus = 0
-            partial_match_bonus = 0
-
-            for term in query_terms:
-                if term in desc_terms:
-                    exact_match_bonus += 0.2
-
-                for desc_term in desc_terms:
-                    if term in desc_term or desc_term in term:
-                        partial_match_bonus += 0.1
-
-            category_keywords = {
-                'location': ['near', 'close', 'located', 'area', 'district'],
-                'amenities': ['wifi', 'kitchen', 'bathroom', 'furnished'],
-                'type': ['apartment', 'house', 'room', 'studio'],
-                'price': ['cheap', 'affordable', 'expensive', 'luxury']
-            }
-
-            category_bonus = 0
-            for category, keywords in category_keywords.items():
-                if any(keyword in processed_query.lower() for keyword in keywords):
-                    if any(keyword in listing_desc.lower() for keyword in keywords):
-                        category_bonus += 0.15
-
-            final_score = min(1.0, (
-                    token_sim * 0.4 +
-                    exact_match_bonus * 0.3 +
-                    partial_match_bonus * 0.2 +
-                    category_bonus * 0.1
-            ))
-
-            boosted_score = np.power(final_score, 0.7)
-            scores.append(boosted_score)
+
+        for idx, row in df.iterrows():
+            try:
+                name_score = 0
+                review_score = 0
+
+                # Name matching
+                name = str(row['name']).lower()
+
+                # Exact phrase match in name
+                if search_query in name:
+                    name_score += 1.0
+
+                # Individual term matches in name
+                name_term_matches = sum(term in name for term in search_terms)
+                name_score += (name_term_matches / len(search_terms)) * 0.5
+
+                # Get reviews for content matching
+                reviews = self.get_listing_reviews_for_search(row['id'])
+                if reviews:
+                    review_texts = [str(review).lower() for review in reviews]
+
+                    # Search for exact phrase in reviews
+                    phrase_matches = sum(search_query in review for review in review_texts)
+                    if phrase_matches > 0:
+                        review_score += min(phrase_matches * 0.2, 0.6)
+
+                    # Search for individual terms in reviews
+                    term_matches = sum(
+                        sum(term in review for term in search_terms)
+                        for review in review_texts
+                    )
+                    review_score += min(term_matches * 0.1, 0.4)
+
+                # Additional relevance factors
+                boost = 1.0
+
+                # Price relevance
+                if any(word in search_query for word in ['cheap', 'budget', 'affordable']):
+                    if row['price'] < df['price'].mean() * 0.8:
+                        boost += 0.2
+                elif any(word in search_query for word in ['expensive', 'luxury', 'high-end']):
+                    if row['price'] > df['price'].mean() * 1.2:
+                        boost += 0.2
+
+                # Room type relevance
+                room_type = str(row['room_type']).lower()
+                room_type_terms = {
+                    'private': ['private', 'own'],
+                    'shared': ['shared', 'share', 'sharing'],
+                    'entire': ['entire', 'whole', 'full']
+                }
+                for type_key, terms in room_type_terms.items():
+                    if any(term in search_query for term in terms) and type_key in room_type:
+                        boost += 0.2
+                        break
+
+                # Location mentions
+                neighborhood = str(row['neighbourhood']).lower()
+                if neighborhood in search_query:
+                    boost += 0.2
+
+                # Reviews quantity relevance
+                if any(term in search_query for term in ['popular', 'reviewed', 'recommended']):
+                    if row['number_of_reviews'] > df['number_of_reviews'].mean():
+                        boost += 0.2
+
+                # Combine scores with weights
+                final_score = ((name_score * 0.6) + (review_score * 0.4)) * boost
+                scores.append(min(1.0, final_score))
+
+            except Exception as e:
+                print(f"Error computing score for listing {row['id']}: {str(e)}")
+                scores.append(0.0)
 
         return np.array(scores)
 
     def sort_by_relevance(self, df, search_query):
+        """Sort listings by relevance using improved scoring system"""
         if not search_query:
             return df
 
@@ -203,43 +240,58 @@ m.REVIEWS_PER_MONTH, m.MINIMUM_NIGHTS, m.AVAILABILITY_365
         df['relevance_percentage'] = df['relevance_score'] * 100
 
         def get_relevance_description(score):
-            if score >= 70:
+            if score >= 80:
                 return "Perfect match"
-            elif score >= 55:
-                return "Very relevant"
+            elif score >= 60:
+                return "Excellent match"
             elif score >= 40:
-                return "Relevant"
-            elif score >= 25:
-                return "Somewhat relevant"
-            elif score >= 15:
-                return "Slightly relevant"
+                return "Good match"
+            elif score >= 20:
+                return "Partial match"
             else:
-                return "Less relevant"
+                return "Low relevance"
 
         df['relevance_features'] = df['relevance_percentage'].apply(get_relevance_description)
 
         def get_matching_features(row):
-            query_tokens = set(self.tokenizer.tokenize(search_query.lower()))
-            desc_tokens = set(self.tokenizer.tokenize(self.create_listing_description(row).lower()))
-            matching_tokens = query_tokens.intersection(desc_tokens)
-
-            if not matching_tokens:
-                return "No direct matches"
-
-            grouped_matches = []
-            processed_tokens = set()
-
-            for token in matching_tokens:
-                if token in processed_tokens:
-                    continue
-
-                similar_tokens = {t for t in matching_tokens if (token in t or t in token)}
-                processed_tokens.update(similar_tokens)
-
-                if similar_tokens:
-                    grouped_matches.append("/".join(sorted(similar_tokens)))
-
-            return ", ".join(grouped_matches)
+            try:
+                features = []
+                search_terms = search_query.lower().split()
+                name = str(row['name']).lower()
+
+                # Name matches
+                matching_terms = [term for term in search_terms if term in name]
+                if matching_terms:
+                    features.append(f"Name matches: {', '.join(matching_terms)}")
+
+                # Review content matches
+                reviews = self.get_listing_reviews_for_search(row['id'])
+                if reviews:
+                    review_matches = {}
+                    # Initialize count for each search term
+                    for term in search_terms:
+                        review_matches[term] = set()  # Use set to store unique review indices
+
+                    # Count matches in each review
+                    for i, review in enumerate(reviews):
+                        review_text = str(review).lower()
+                        for term in search_terms:
+                            if term in review_text:
+                                review_matches[term].add(i)  # Add review index to set
+
+                    # Format matches for display
+                    formatted_matches = []
+                    for term, matching_indices in review_matches.items():
+                        if matching_indices:  # If there are matches for this term
+                            formatted_matches.append(f"{term} ({len(matching_indices)} reviews)")
+
+                    if formatted_matches:
+                        features.append(f"Matched based on High relevance , Keyword found in Review")
+                return " | ".join(features) if features else "Matched based on Low relevance"
+
+            except Exception as e:
+                print(f"Error in get_matching_features: {str(e)}")
+                return "Unable to determine matches"
 
         df['matching_features'] = df.apply(get_matching_features, axis=1)
         return df.sort_values('relevance_score', ascending=False)
@@ -287,16 +339,20 @@ m.REVIEWS_PER_MONTH, m.MINIMUM_NIGHTS, m.AVAILABILITY_365
             relevance_info = ""
             if search_query and 'relevance_percentage' in row:
                 relevance_info = f"""
-                    <p style='margin: 5px 0;'>
-                        <strong>Relevance:</strong> {row['relevance_percentage']:.0f}%
-                        <br/>
-                        <em>{row['relevance_features']}</em>
-                    </p>
+                    <div class='relevance-info' style='margin: 10px 0; padding: 8px; background-color: #f8f9fa; border-radius: 4px;'>
+                        <p style='margin: 5px 0;'>
+                            <strong>Match Score:</strong> {row['relevance_percentage']:.0f}%
+                            <br/>
+                            <strong>Relevance:</strong> {row['relevance_features']}
+                            <br/>
+                            <strong>Matching Features:</strong> {row['matching_features']}
+                        </p>
+                    </div>
                 """
 
             popup_content = f"""
-                <div style='min-width: 200px; padding: 10px;'>
-                    <h4 style='margin: 0 0 10px 0;'>{escape(str(row['name']))}</h4>
+                <div style='min-width: 280px; max-width: 320px; padding: 15px;'>
+                    <h4 style='margin: 0 0 10px 0; color: #2c3e50;'>{escape(str(row['name']))}</h4>
                     <p style='margin: 5px 0;'><strong>Host:</strong> {escape(str(row['host_name']))}</p>
                     <p style='margin: 5px 0;'><strong>Room Type:</strong> {escape(str(row['room_type']))}</p>
                     <p style='margin: 5px 0;'><strong>Price:</strong> ${row['price']:.0f}</p>
@@ -304,20 +360,18 @@ m.REVIEWS_PER_MONTH, m.MINIMUM_NIGHTS, m.AVAILABILITY_365
                     {relevance_info}
                     <button onclick="streamlit_click('{review_button_key}')" 
                         style="background-color: #4CAF50; color: white; padding: 8px 15px; border: none; 
-                        border-radius: 4px; cursor: pointer; margin-top: 10px;">
+                        border-radius: 4px; cursor: pointer; margin-top: 10px; width: 100%;">
                         View Reviews ({len(reviews)})
                     </button>
                 </div>
             """
 
             marker_color = 'green' if selected_id == row['id'] else 'red'
-
             marker = folium.Marker(
                 location=[row['latitude'], row['longitude']],
                 popup=popup_content,
                 icon=folium.Icon(color=marker_color, icon='home'),
             )
-
             marker.add_to(m)
 
             if selected_id is not None and row['id'] == selected_id:
@@ -325,4 +379,5 @@ m.REVIEWS_PER_MONTH, m.MINIMUM_NIGHTS, m.AVAILABILITY_365
 
         if show_traffic:
             self.traffic_manager.add_spots_to_map(m)
+
         return m, df

app.py

@@ -1,3 +1,4 @@
+import re
 import streamlit as st
 from html import escape
 from streamlit_folium import st_folium, folium_static
@@ -10,34 +11,62 @@ def load_css(css_file):
         st.markdown(f'<style>{f.read()}</style>', unsafe_allow_html=True)
 
 
+def highlight_search_terms(text, search_query):
+    """Highlight search terms in text"""
+    if not search_query:
+        return text
+
+    highlighted_text = text
+    search_terms = search_query.lower().split()
+
+    for term in search_terms:
+        if term.strip():
+            # Case-insensitive replacement with word boundaries
+            pattern = f'(?i)\\b{term}\\b'
+            replacement = f'<span class="highlight">{term}</span>'
+            highlighted_text = re.sub(pattern, replacement, highlighted_text)
+
+    return highlighted_text
+
+
 def render_review_dialog():
-        # Display reviews
-        with st.container():
-            col_title = st.columns([5, 1])
-            with col_title[0]:
-                st.markdown(f"### Reviews for {st.session_state.current_review_listing_name}")
-            reviews = st.session_state.visualizer.get_listing_reviews(st.session_state.current_review_listing)
-            if reviews:
-                for review in reviews:
-                    try:
-                        review_date, reviewer_name, comments = review
-                        st.markdown(f"""
-                                   <div class="review-card">
-                                     <div class="review-header">
-                                       {escape(str(reviewer_name))} - {escape(str(review_date))}
-                                     </div>
-                                     <div class="review-content">
-                                       {escape(str(comments))}
-                                     </div>
-                                   </div>""", unsafe_allow_html=True)
-                    except Exception as e:
-                        st.error(f"Error displaying review: {str(e)}")
-            else:
-                st.info("No reviews available for this listing.")
+    # Display reviews
+    with st.container():
+        col_title = st.columns([5, 1])
+        with col_title[0]:
+            st.markdown(f"### Reviews for {st.session_state.current_review_listing_name}")
+
+        reviews = st.session_state.visualizer.get_listing_reviews(st.session_state.current_review_listing)
+        if reviews:
+            for review in reviews:
+                try:
+                    review_date, reviewer_name, comments = review
+
+                    # Highlight search terms in comments if search query exists
+                    highlighted_comments = highlight_search_terms(
+                        str(comments),
+                        st.session_state.search_query
+                    )
+
+                    st.markdown(f"""
+                        <div class="review-card">
+                            <div class="review-header">
+                                {escape(str(reviewer_name))} - {escape(str(review_date))}
+                            </div>
+                            <div class="review-content">
+                                {highlighted_comments}
+                            </div>
+                        </div>
+                    """, unsafe_allow_html=True)
+                except Exception as e:
+                    st.error(f"Error displaying review: {str(e)}")
+        else:
+            st.info("No reviews available for this listing.")
+
 def main():
     st.set_page_config(
         layout="wide",
-        page_title="HKUST AirBNB+",
+        page_title="HKUST BNB+ | Platform for BNB Matching for HKUST PG Student",
         initial_sidebar_state="expanded"
     )
     load_css('style.css')
@@ -81,13 +110,13 @@ def main():
 
     with st.sidebar:
         st.markdown(
-            '<p class="sidebar-header">HKUST AirBNB+ (External Hall Scheme for Non Local PG Students) <BR/></p>',
+            '<p class="sidebar-header">HKUST BNB+<BR/></p>',
             unsafe_allow_html=True)
 
         search_query = st.text_input(
             "🔍 Search listings",
             value=st.session_state.search_query,
-            placeholder="Try: 'cozy apartment near MTR' or 'quiet room with view'"
+            placeholder="Try: 'cozy , quiet '"
         )
 
         if search_query != st.session_state.search_query:

style.css

@@ -411,3 +411,10 @@
 ::-webkit-scrollbar-thumb:hover {
     background: #555;
 }
+
+.highlight {
+    background-color: #FFEB3B;
+    padding: 0 2px;
+    border-radius: 2px;
+    font-weight: bold;
+}