initial

app.py

@@ -0,0 +1,60 @@
+import streamlit as st
+import pandas as pd
+
+
+st.title('Рекомендация книг на основе пользовательского промта')
+st.subheader('Основная информация')
+
+def load_data(file_path):
+    return pd.read_csv(file_path)
+df = load_data('data/books_data_cleaned.csv')
+
+data = {
+    'Источник': ['chitai-gorod.ru'],
+    'Кол-во книг': ['5000'],
+    'Уникальных авторов': ['2112'],
+    'Жанр': ['Художественная лит-ра'],
+    'Время парсинга': ['77 минут'],
+}
+df1 = pd.DataFrame(data)
+st.dataframe(df1)
+st.subheader('Используемые модели:')
+
+data1 = {
+    'sentence-transformers': ['<a href="https://huggingface.co/sentence-transformers/all-mpnet-base-v2" target="_blank">all-mpnet-base-v2</a>'],
+    'Pre-Trained MS MARCO Models': ['<a href="https://huggingface.co/sentence-transformers/msmarco-roberta-base-v3" target="_blank">msmarco-roberta-base-v3</a>'],
+}
+df2 = pd.DataFrame(data1)
+st.markdown(df2.to_html(escape=False), unsafe_allow_html=True)
+
+
+def show_random_books():
+    sample_df = df.sample(n=10)
+    num_books = len(sample_df)
+    num_rows = (num_books + 1) // 2
+
+    for i in range(num_rows):
+        cols = st.columns(4)  # Создаем 4 колонки
+        for j in range(2):
+            index = i * 2 + j
+            if index < num_books:
+                row = sample_df.iloc[index]
+                if j == 0:
+                    with cols[0]:  # Первая колонка (обложка первой книги)
+                        st.image(row['image_url'], width=200)
+                    with cols[1]:  # Вторая колонка (информация первой книги)
+                        st.subheader(row['title'])
+                        st.write(f"Автор: {row['author']}")
+                        st.write(f"**Ссылка:** [книга]({row['page_url']})")
+                    st.write("---")
+                elif j == 1:
+                    with cols[2]:  # Третья колонка (обложка второй книги)
+                        st.image(row['image_url'], width=200)
+                    with cols[3]:  # Четвертая колонка (информация второй книги)
+                        st.subheader(row['title'])
+                        st.write(f"Автор: {row['author']}")
+                        st.write(f"**Ссылка:** [книга]({row['page_url']})")
+
+
+if st.button('Показать 10 случайных книг'):
+    show_random_books()

data/book_embeddings.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbb86f342a62c07926d39da48daab2d4481d7fcf4437eae254eaa42ef8ac0e91
+size 15283328

data/book_embeddings_ms.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d90b7ba8398f552f593df4688485b51d0aea57e9712a480878f79799d17a59c9
+size 15283328

notebooks/parser.py

@@ -0,0 +1,59 @@
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+
+df = pd.DataFrame(columns=['page_url', 'image_url', 'author', 'title', 'annotation'])
+
+def extract_data_from_page(page_number):
+    url = f'https://www.chitai-gorod.ru/catalog/books/hudozhestvennaya-literatura-110001?page={page_number}'
+    response = requests.get(url)
+    soup = BeautifulSoup(response.content, 'html.parser')
+    books = soup.find_all('article', class_='product-card')
+    
+    data = []
+    for book in books:
+        try:
+            book_url = book.find('a', class_='product-card__picture')['href']
+            title = book.find('div', class_='product-title__head').get_text(strip=True)
+            author = book.find('div', class_='product-title__author').get_text(strip=True)
+            absolute_url = f'https://www.chitai-gorod.ru{book_url}'
+            
+            data.append({'page_url': absolute_url, 'title': title, 'author': author})
+        except Exception as e:
+            print(f"Error processing book: {e}")
+    
+    return data
+
+
+for page in range(2, 201):
+    print(f"Processing page {page}...")
+    page_data = extract_data_from_page(page)
+    df = pd.concat([df, pd.DataFrame(page_data)], ignore_index=True)
+    if len(df) >= 5000:
+        break
+    
+df = df.head(5000)
+
+
+def extract_book_details(book_url):
+    try:
+        response = requests.get(book_url)
+        soup = BeautifulSoup(response.content, 'html.parser')
+        image_tag = soup.find('meta', {'name': 'og:image'})
+        image_url = image_tag['content'] if image_tag else None
+        annotation_tag = soup.find('div', {'itemprop': 'description'})
+        annotation = annotation_tag.get_text(strip=True) if annotation_tag else None
+        
+        return image_url, annotation
+    except Exception as e:
+        print(f"Error extracting details from {book_url}: {e}")
+        return None, None
+
+for idx, row in df.head(5000).iterrows():
+    print(f"Fetching details for {row['page_url']}...")
+    image_url, annotation = extract_book_details(row['page_url'])
+    df.at[idx, 'image_url'] = image_url
+    df.at[idx, 'annotation'] = annotation
+
+
+df.to_csv('books_data_with_details.csv', index=False)

pages/recommendations.py

@@ -0,0 +1,96 @@
+import numpy as np
+import pandas as pd
+import streamlit as st
+import requests
+from sentence_transformers import util
+from sentence_transformers import SentenceTransformer, util
+st.set_page_config(page_title="Custom Button Example", layout="wide")
+
+@st.cache_resource
+def load_model_all_mpnet():
+    return SentenceTransformer('all-mpnet-base-v2')
+model_mp = load_model_all_mpnet()
+@st.cache_data
+def load_embeddings(file_path):
+    return np.load(file_path)
+book_embeddings_mp = load_embeddings('data/book_embeddings.npy')
+@st.cache_data
+def load_data(file_path):
+    return pd.read_csv(file_path)
+df = load_data('data/books_data_cleaned.csv')
+@st.cache_resource
+def load_model_msmarco():
+    return SentenceTransformer('msmarco-roberta-base-v3')
+model_ms = load_model_msmarco()
+@st.cache_data
+def load_embeddings(file_path):
+    return np.load(file_path)
+book_embeddings_ms = load_embeddings('data/book_embeddings_ms.npy')
+
+
+def get_embedding(text, model):
+    text = model.encode(text, convert_to_tensor=True)
+    return text
+
+
+def get_top_10_recommendations(query, model, book_embeddings, top_k):
+    query_embedding = get_embedding(query, model).cpu()
+    similarities = util.pytorch_cos_sim(query_embedding, book_embeddings)[0]
+    top_results = similarities.cpu().numpy().argsort()[::-1][:top_k]
+    top_books = df.iloc[top_results].copy()
+    similarity_scores = similarities.cpu().numpy()[top_results]
+    top_books['similarity_score'] = similarity_scores
+    return top_books
+
+
+st.title('Рекомендации книг')
+
+search = st.radio(
+    "Выберите тип семантического поиска:",
+    [":blue[Симметричный]", ":blue[Асимметричный]"],
+    captions=[
+        "Используем 'all-mpnet-base-v2'",
+        "Используем 'msmarco-roberta-base-v3'",
+    ],
+)
+
+def params(search):
+    if search == ":blue[Симметричный]":
+        text = '''Я ищу книги в жанре фэнтези, которые описывают приключения магов и волшебников, обучающихся в специальных магических школах и сражающихся с темными силами или злыми существами. Особенно интересуют произведения, где главные герои сталкиваются с эпическими испытаниями и развивают свои уникальные способности.'''
+        model = model_mp
+        book_embeddings = book_embeddings_mp
+        return text, model, book_embeddings
+    elif search == ":blue[Асимметричный]":
+        text = '''путешествие во времени'''
+        model = model_ms
+        book_embeddings = book_embeddings_ms
+        return text, model, book_embeddings
+text, model, book_embeddings = params(search)
+
+
+col1, col2 = st.columns([3, 1])
+with col1:
+    query = st.text_area('Введите запрос, чтобы получить рекомендации', f'{text}', height=95)
+with col2:
+    number = st.number_input(
+        "Сколько книг найти?", value=10
+)
+    find_button = st.button('Найти', key='find_button', use_container_width=True)
+
+if find_button and query:
+        top_10_books = get_top_10_recommendations(query, model, book_embeddings, number)
+        for idx, row in top_10_books.iterrows():
+            with st.container():
+                col1, col2 = st.columns([1, 3])
+                
+                with col1:
+                    st.image(row['image_url'], width = 300)
+                with col2:
+                    st.subheader(f"{row['title']}")
+                    st.write(f"**Автор:** {row['author']}")
+                    st.write("---")
+                    st.write(row['annotation'])
+                    st.metric(label="Схожесть", value=f"{row['similarity_score']:.3f}")
+                    st.write(f"**Ссылка:** {row['page_url']}")
+                st.write("---")
+    

requirements.txt

@@ -0,0 +1,5 @@
+beautifulsoup4==4.12.3
+pandas==2.2.2
+Requests==2.32.3
+streamlit==1.37.0
+sentence-transformers==3.0.1