Spaces:
Sleeping
Sleeping
| import spaces | |
| import os | |
| from dotenv import load_dotenv | |
| import re | |
| from urllib.parse import urlparse | |
| import pandas as pd | |
| import unicodedata as uni | |
| import emoji | |
| from langchain_openai import ChatOpenAI | |
| from langchain_community.embeddings import HuggingFaceEmbeddings | |
| from langchain_community.document_loaders import DataFrameLoader | |
| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| from langchain_community.vectorstores import FAISS | |
| from langchain.chains import RetrievalQA | |
| import gradio as gr | |
| import logging | |
| import requests | |
| # Load environment variables | |
| load_dotenv() | |
| # Set command line arguments for Gradio | |
| os.environ["COMMANDLINE_ARGS"] = "--no-gradio-queue" | |
| # Configure logging | |
| logging.basicConfig( | |
| level=logging.DEBUG, | |
| format="%(asctime)s [%(levelname)s] %(message)s", | |
| handlers=[logging.StreamHandler()], | |
| ) | |
| logger = logging.getLogger(__name__) | |
| import http.client | |
| http.client.HTTPConnection.debuglevel = 1 | |
| req_log = logging.getLogger("requests.packages.urllib3") | |
| req_log.setLevel(logging.DEBUG) | |
| req_log.propagate = True | |
| # Constants | |
| LIMIT = 1000 # Limit to 1000 reviews to avoid long processing times | |
| OpenAIModel = "gpt-3.5-turbo" | |
| shop_id = "" | |
| item_id = "" | |
| item = {} | |
| cache_URL = "" | |
| db = None | |
| qa = None | |
| cache = {} | |
| import json | |
| # Function to request product ID from Tokopedia | |
| def request_product_id(shop_domain, product_key): | |
| ENDPOINT = "https://gql.tokopedia.com/graphql/PDPGetLayoutQuery" | |
| payload = { | |
| "operationName": "PDPGetLayoutQuery", | |
| "variables": { | |
| "shopDomain": shop_domain, | |
| "productKey": product_key, | |
| "layoutID": "", | |
| "apiVersion": 1, | |
| "firstTime": True, | |
| }, | |
| } | |
| headers = { | |
| 'authority': 'gql.tokopedia.com', | |
| 'accept': '*/*', | |
| 'accept-language': 'en-US,en;q=0.9', | |
| 'content-type': 'application/json', | |
| 'cookie': '_UUID_NONLOGIN_=e9727c37c5f733a77479185a66e63e4d; _UUID_NONLOGIN_.sig=tkAjvTdngH8Tn2TawWMZs8yir7g; DID=a717cbd11e2c1799009d1f87dd469aee95e922f0f927d3df40966a41e4eec18f634c74b0f2242b80393e711af4bf7119; DID_JS=YTcxN2NiZDExZTJjMTc5OTAwOWQxZjg3ZGQ0NjlhZWU5NWU5MjJmMGY5MjdkM2RmNDA5NjZhNDFlNGVlYzE4ZjYzNGM3NGIwZjIyNDJiODAzOTNlNzExYWY0YmY3MTE547DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=; __auc=f6d4b66f17cefc7db9583cc0ea3; _hjid=52f6214b-1f92-4aac-a3be-adc11e04aafc; cto_bundle=T0m1vF83VlNReTd2VXh6JTJGdGNtNXhZUDZMbkQ3WjZveUxUM1ZVUHdkd3FKcSUyQlNTMUclMkJtZHpDdyUyRllUU0x3ZWlHWHJGT2dIWWQ4WTdqejBxSTNJWFMwMGMlMkZHVXJuUWUyZG9VaDRlblczS0F5TWhJM0YzN2VRdDhwS3UlMkZzV2clMkYxRTlpczRXaWt3Z0xMbWJqbEhtZFg4VlFWV1ZmQSUzRCUzRA; _UUID_CAS_=cc18f322-9a5c-4cf6-9dfd-1270e46f8582; _CASE_=732a6c416c2a3238242a69416c2a3238242a646a642a322a2a242a6b416c2a3238242a6467666f2a322a2a242a64697c2a322a2a242a784b672a322a2a242a7f416c2a32393a3a39383b3f3d242a7b416c2a3239393d3b383d3f3b242a7b5c71786d2a322a3a602a242a7f607b2a322a5373542a7f697a6d60677d7b6d57616c542a32393a3a39383b3f3d24542a7b6d7a7e616b6d577c71786d542a32542a3a60542a24542a57577c71786d6669656d542a32542a5f697a6d60677d7b6d7b542a752473542a7f697a6d60677d7b6d57616c542a323824542a7b6d7a7e616b6d577c71786d542a32542a393d65542a24542a57577c71786d6669656d542a32542a5f697a6d60677d7b6d7b542a75552a75; _gcl_au=1.1.1070690307.1661094594; _jxx=74cb82d0-f38f-11ec-b88d-977b36f46df7; _jx=74cb82d0-f38f-11ec-b88d-977b36f46df7; hfv_banner=true; bm_sz=FEDD193B43C05ABC0ECC7F218FD5E1C0~YAAQrSE1F/1cP+SDAQAA9oK15BF3DdkWy7nnyvTZieay5zJg128l/5uxqPSqkvFazOh4Wv3W/4AUQLS9ZTkA7gC6IWSGdmyUZZDpZneKXHpnw0z91FQk9Ydt+eYC27M4tsYrzfda+aWzsuJrefZsvvOvug/ZvrS4RI1pFjgoeAaotUY3gVVJBEa7KCQis4W/5OO94n03wgyxu7fB1vq8Gve2bXtPMuOP9kc5ShEm/stdSNt9WjiVt4Yvg9TMEDMK/8UqBRsvXbD8YPvbYdWMUw12n/bq+LfTV2EPb8hs1YAkS1U8+IE=~3293746~3355201; _SID_Tokopedia_=699V7myhqHJTekLwfsmffoi8jhxDptrX0TwX7hPKexK0RauqTC_em34ZEmpLo2P4yP7P2bCiEz2ll3qvPtNZtHAc3ocJtX5BLZG8pSe5mP3NYlRhpiclF-cTdKOejSvt; _gid=GA1.2.750563175.1665989199; _jxxs=1665989198-74cb82d0-f38f-11ec-b88d-977b36f46df7; _jxs=1665989198-74cb82d0-f38f-11ec-b88d-977b36f46df7; __asc=c2c83db8183e4b1d556eeadc2a6; NR_SID=NRl9cf5q6os1xrv; AMP_TOKEN=%24NOT_FOUND; ak_bmsc=154ED6F998E215265D990C8CABCF4618~000000000000000000000000000000~YAAQrSE1F3NuP+SDAQAALge35BEb0jR/EDHlGS9fxmk/fhk3+qBfnzY2Jv4HxyBnR6N0LsUayf3aiZXy3bUvITnTC1ZgZmWsLLcdS2aqgATfgZVTpkFvW9tCgDV0L3fpPHYR3sGnhjtTirg4mfbGw+C3xEDjSn14AS7SNnN0aCgzZzX7e229gmZGfuOALcII9lwh7LwBRSSd/BSKyn3Y34PH7qqwOS+F624Xm9fNwbbMGdnadv9PfPteUhSC9U0cqgtDxE7pAjc8vewzIf2xm/cr0+dCN+s7shGPXDk+ELtFGKYgFyz1Sp1aOIs606S4hJpX/sTzXvLceppM0tJ66/vCIzwAJD4id/VwC3N1teNAly0yhb28sTe5lNA5RnTYE27j6cEAGLbKObWIzNyULaf3e2bv0o5utLMRpqGbApAoKvbDgOBPjF+rBxoKSoByJnxlCM98M0G9XAzxpRDnRFLujKTzZ8CKnMk9FrZpdbttHVJT9YTHic7TMHYIOsUgra67HX3PGg==; _abck=DB9B8AC184B53D64511C0CA8E46737CB~0~YAAQrSE1F9ZuP+SDAQAAKxG35AgWg2PHBBvM9/T36YC42lsBvwpdbpPSZABAxneyJL5ZbzE4lx5xlz2XwH9a8MQ5IfXHqhgsra6cBqqSzU8xgcNOFIlp7RMpBNfjV2Cwla2iNAzzdbmskpkIB8HqujKdWibzNMJpXB/YqmiZwj/FLyVR8kUpJo+UG0evJyaNil6vVqoXXUPQmUFSGAoArQTI8WXXlKanMKbaIh8xLxRgv1rt5kKf5/R1m6275w1fQfh83by6VurvHnEd0YDOndLmPJdXI8Piow/tMatTi3FNObNjmHg5CNA63K5yxPtTnJsl3kG1Wexk7cH4FFpG74EMGWHukQZ6IFpeUymcj52FjxWYPwAH0lEKNq/qdOibLir0JybgJeLz8xa1eN2kXLlo06yKOnEkUWDl~-1~-1~-1; _dc_gtm_UA-126956641-6=1; _ga_70947XW48P=GS1.1.1665989198.16.1.1665989428.60.0.0; _ga=GA1.2.426299726.1636110422; _dc_gtm_UA-9801603-6=1; _abck=DB9B8AC184B53D64511C0CA8E46737CB~-1~YAAQrSE1F6clQOSDAQAA8Z3F5AhK11JVGEv0Mwg28CHe7ro9JDkPKhPuivMf6GtvJC4Bk/p7zI4a2xBftcSFG9nLsyh8w7LWvqI9LXyIg8zU4rbbRPpp5yk+oCyh7u0KTSOM4XRJaXk2MKwIG+Irdo5rGB8e0UJy+dr6OsWCBl/bnTIXj2xIvqwKEbLiGyyNX+keTPXnqVhARZ/m0OmEUnreuuXiazWGwjCJMPeMd2H405ipu3hEEJYVDEaxMp+zpT1y3FqjjfgUkSzoISVkh2rF73Cpz4yYNfC0HQeI0E1mcDJjDcxXQjErIOkN1O5bcwK/fWpXcC7r9nWWyDUB8RJanaDewcwGelUaKbA6lOmoJwIuCK7ON8DQzweB4opfl/xTUD7GVBnTyxhavU1zs3G+FDX+9UwPAaOw~0~-1~-1; bm_sz=933FB20E8A08C7F904B2BBEFAF59CF75~YAAQrSE1F8lFP+SDAQAAGuez5BFT+bdQDJxSRM+CqoyWKuJfBc5YLC9LptyhgD3iV0UTDDXYfIRkrJDvV3Uec6IMyRTsdgAjoHmRZ7fcDgjn1ynK05v+6t+cnwthQS1mSNrX6pjpQXQ3GJYjyW4SOG/TxwhZdXe13s/IYVoT8wsqF3jE/zmnc+FRmDrDQRpll4sWG/F/nsWCuBmtRrbB9nuHCuLffgln81YTFV1rWA8koN7HsTzOhv8+t3U1tkERLb1/B4OIaNAiP777rxQXW1gXyC7PafPY98603/oT9yhiNBb1q1U=~4534576~3163705', | |
| 'origin': 'https://www.tokopedia.com', | |
| 'queryhash': 'v1:90338d207352e8b71cf754979b915218;false', | |
| 'referer': 'https://www.tokopedia.com/miniso-official/miniso-sandal-rumah-slipper-wanita-selop-comfortable-nyaman-flip-flop-light-green-39-40?extParam=src%3Dmultiloc%26whid%3D7377294', | |
| 'sec-ch-ua': '"Chromium";v="106", "Google Chrome";v="106", "Not;A=Brand";v="99"', | |
| 'sec-ch-ua-mobile': '?1', | |
| 'sec-ch-ua-platform': '"Android"', | |
| 'sec-fetch-dest': 'empty', | |
| 'sec-fetch-mode': 'cors', | |
| 'sec-fetch-site': 'same-site', | |
| 'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Mobile Safari/537.36', | |
| 'x-device': 'mobile', | |
| 'x-source': 'tokopedia-lite', | |
| 'x-tkpd-akamai': 'pdpGetLayout', | |
| 'x-tkpd-lite-service': 'atreus', | |
| 'x-version': '859a718' | |
| } | |
| response = requests.post(ENDPOINT, headers=headers, json=payload) | |
| response.raise_for_status() | |
| logger.info(response.text) | |
| return response | |
| # payload = { | |
| # "operationName": "PDPGetLayoutQuery", | |
| # "variables": { | |
| # "shopDomain": f"{shop_domain}", | |
| # "productKey": f"{product_key}", | |
| # "apiVersion": 1, | |
| # }, | |
| # "query": """fragment ProductVariant on pdpDataProductVariant { | |
| # errorCode | |
| # parentID | |
| # defaultChild | |
| # children { | |
| # productID | |
| # } | |
| # __typename | |
| # } | |
| # query PDPGetLayoutQuery($shopDomain: String, $productKey: String, $layoutID: String, $apiVersion: Float, $userLocation: pdpUserLocation, $extParam: String, $tokonow: pdpTokoNow, $deviceID: String) { | |
| # pdpGetLayout(shopDomain: $shopDomain, productKey: $productKey, layoutID: $layoutID, apiVersion: $apiVersion, userLocation: $userLocation, extParam: $extParam, tokonow: $tokonow, deviceID: $deviceID) { | |
| # requestID | |
| # name | |
| # pdpSession | |
| # basicInfo { | |
| # id: productID | |
| # } | |
| # components { | |
| # name | |
| # type | |
| # position | |
| # data { | |
| # ...ProductVariant | |
| # __typename | |
| # } | |
| # __typename | |
| # } | |
| # __typename | |
| # } | |
| # } | |
| # """, | |
| # } | |
| # headers = { | |
| # "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36", | |
| # "Referer": "https://www.tokopedia.com", | |
| # "X-TKPD-AKAMAI": "pdpGetLayout", | |
| # } | |
| # try: | |
| # response = requests.post(ENDPOINT, json=payload, headers=headers, timeout=60) | |
| # response.raise_for_status() | |
| # logger.info(f"Request successful. Status code: {response.status_code}") | |
| # return response | |
| # except requests.exceptions.RequestException as e: | |
| # logger.error(f"Request failed: {e}") | |
| # return None | |
| # Function to request product reviews from Tokopedia | |
| def request_product_review(product_id, page=1, limit=20): | |
| ENDPOINT = "https://gql.tokopedia.com/graphql/productReviewList" | |
| payload = { | |
| "operationName": "productReviewList", | |
| "variables": { | |
| "productID": f"{product_id}", | |
| "page": page, | |
| "limit": limit, | |
| "sortBy": "", | |
| "filterBy": "", | |
| }, | |
| "query": """query productReviewList($productID: String!, $page: Int!, $limit: Int!, $sortBy: String, $filterBy: String) { | |
| productrevGetProductReviewList(productID: $productID, page: $page, limit: $limit, sortBy: $sortBy, filterBy: $filterBy) { | |
| productID | |
| list { | |
| id: feedbackID | |
| variantName | |
| message | |
| productRating | |
| reviewCreateTime | |
| reviewCreateTimestamp | |
| isReportable | |
| isAnonymous | |
| reviewResponse { | |
| message | |
| createTime | |
| __typename | |
| } | |
| user { | |
| userID | |
| fullName | |
| image | |
| url | |
| __typename | |
| } | |
| likeDislike { | |
| totalLike | |
| likeStatus | |
| __typename | |
| } | |
| stats { | |
| key | |
| formatted | |
| count | |
| __typename | |
| } | |
| badRatingReasonFmt | |
| __typename | |
| } | |
| shop { | |
| shopID | |
| name | |
| url | |
| image | |
| __typename | |
| } | |
| hasNext | |
| totalReviews | |
| __typename | |
| } | |
| } | |
| """, | |
| } | |
| headers = { | |
| "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36", | |
| "Referer": "https://www.tokopedia.com", | |
| "X-TKPD-AKAMAI": "productReviewList", | |
| } | |
| try: | |
| response = requests.post(ENDPOINT, json=payload, headers=headers, timeout=60) | |
| response.raise_for_status() | |
| logger.info(f"Request successful. Status code: {response.status_code}") | |
| return response | |
| except requests.exceptions.RequestException as e: | |
| logger.error(f"Request failed: {e}") | |
| return None | |
| # Function to scrape reviews for a product | |
| def scrape(product_id, max_reviews=LIMIT): | |
| all_reviews = [] | |
| page = 1 | |
| has_next = True | |
| logger.info("Extracting product reviews...") | |
| while has_next and len(all_reviews) < max_reviews: | |
| response = request_product_review(product_id, page=page) | |
| if not response: | |
| break | |
| data = response.json()["data"]["productrevGetProductReviewList"] | |
| reviews = data["list"] | |
| all_reviews.extend(reviews) | |
| has_next = data["hasNext"] | |
| page += 1 | |
| reviews_df = pd.json_normalize(all_reviews) | |
| reviews_df.rename(columns={"message": "comment"}, inplace=True) | |
| reviews_df = reviews_df[["comment"]] | |
| logger.info(reviews_df.head()) | |
| return reviews_df | |
| # Function to extract product ID from URL | |
| def get_product_id(URL): | |
| parsed_url = urlparse(URL) | |
| *_, shop, product_key = parsed_url.path.split("/") | |
| response = request_product_id(shop, product_key) | |
| if response: | |
| product_id = response.json()["data"]["pdpGetLayout"]["basicInfo"]["id"] | |
| logger.info(f"Product ID: {product_id}") | |
| return product_id | |
| else: | |
| logger.error("Failed to get product ID") | |
| return None | |
| # Function to clean the reviews DataFrame | |
| def clean(df): | |
| df = df.dropna().copy().reset_index(drop=True) # Drop reviews with empty comments | |
| df = df[df["comment"] != ""].reset_index(drop=True) # Remove empty reviews | |
| df["comment"] = df["comment"].apply(lambda x: clean_text(x)) # Clean text | |
| df = df[df["comment"] != ""].reset_index(drop=True) # Remove empty reviews | |
| logger.info("Cleaned reviews DataFrame") | |
| return df | |
| # Function to clean individual text entries | |
| def clean_text(text): | |
| text = uni.normalize("NFKD", text) # Normalize characters | |
| text = emoji.replace_emoji(text, "") # Remove emoji | |
| text = re.sub(r"(\w)\1{2,}", r"\1", text) # Remove repeated characters | |
| text = re.sub(r"[ ]+", " ", text).strip() # Remove extra spaces | |
| return text | |
| # Initialize LLM and embeddings | |
| llm = ChatOpenAI(model=OpenAIModel, temperature=0.1) | |
| embeddings = HuggingFaceEmbeddings(model_name="LazarusNLP/all-indobert-base-v2") | |
| # Function to generate a summary or answer based on reviews | |
| async def generate(URL, query): | |
| global cache_URL, db, qa, cache | |
| if not URL or not query: | |
| return "Input kosong" | |
| try: | |
| product_id = get_product_id(URL) | |
| if not product_id: | |
| return "Gagal mendapatkan product ID" | |
| if URL not in cache: | |
| reviews = scrape(product_id) | |
| if reviews.empty: | |
| return "Tidak ada ulasan ditemukan" | |
| cleaned_reviews = clean(reviews) | |
| loader = DataFrameLoader(cleaned_reviews, page_content_column="comment") | |
| documents = loader.load() | |
| text_splitter = RecursiveCharacterTextSplitter( | |
| chunk_size=1000, chunk_overlap=50 | |
| ) | |
| docs = text_splitter.split_documents(documents) | |
| db = FAISS.from_documents(docs, embeddings) | |
| cache[URL] = (docs, db) | |
| else: | |
| docs, db = cache[URL] | |
| qa = RetrievalQA.from_chain_type(llm=llm, retriever=db.as_retriever()) | |
| res = await qa.ainvoke(query) | |
| return res["result"] | |
| except Exception as e: | |
| logger.error(f"Error in generating response: {e}") | |
| return "Gagal mendapatkan review dari URL" | |
| # Set up Gradio interface | |
| product_box = gr.Textbox(label="URL Produk", placeholder="URL produk dari Tokopedia") | |
| query_box = gr.Textbox( | |
| lines=2, | |
| label="Kueri", | |
| placeholder="Contoh: Apa yang orang katakan tentang kualitas produknya?, Bagaimana pendapat orang yang kurang puas dengan produknya?", | |
| ) | |
| gr.Interface( | |
| fn=generate, | |
| inputs=[product_box, query_box], | |
| outputs=[gr.Textbox(label="Jawaban")], | |
| title="RingkasUlas", | |
| description="Bot percakapan yang bisa meringkas ulasan-ulasan produk di Tokopedia Indonesia (https://tokopedia.com/). Harap bersabar, bot ini dapat memakan waktu agak lama saat mengambil ulasan dari Tokopedia dan menyiapkan jawabannya.", | |
| allow_flagging="never", | |
| ).launch(debug=True) | |