|
import streamlit as st |
|
from transformers import pipeline |
|
import re |
|
import time |
|
|
|
|
|
classifier = pipeline("zero-shot-classification", model="cross-encoder/nli-distilroberta-base") |
|
|
|
|
|
st.title("Text Classification App") |
|
|
|
|
|
operation = st.radio("Choose an operation:", ["Filter Keywords", "Extra & Filter Param (URLs)"]) |
|
|
|
|
|
uploaded_file = st.file_uploader("Upload a text file", type=["txt"]) |
|
|
|
if uploaded_file is not None: |
|
|
|
content = uploaded_file.read().decode("utf-8") |
|
items = [line.strip() for line in content.splitlines() if line.strip()] |
|
|
|
|
|
categories = ["shop", "game", "stream"] |
|
|
|
|
|
shopping_keywords = ["buy", "shop", "purchase", "product", "categories", "supplier", "delivered"] |
|
gaming_keywords = ["play", "game", "player", "level", "score"] |
|
streaming_keywords = ["watch", "stream", "video", "live"] |
|
|
|
|
|
shopping_items = [] |
|
gaming_items = [] |
|
streaming_items = [] |
|
unknown_items = [] |
|
|
|
|
|
param_categories = { |
|
"shop_params": [], |
|
"game_params": [], |
|
"stream_params": [], |
|
"unknown_params": [] |
|
} |
|
|
|
|
|
extensions = {} |
|
|
|
|
|
progress_bar = st.progress(0) |
|
pause_button = st.button("Pause") |
|
stop_button = st.button("Stop") |
|
continue_button = st.button("Continue") |
|
paused = False |
|
stopped = False |
|
current_index = 0 |
|
batch_size = 50 |
|
|
|
|
|
def classify_keywords_batch(items, categories, start_index=0): |
|
global paused, stopped, current_index |
|
total_items = len(items) |
|
for i in range(start_index, total_items, batch_size): |
|
if stopped: |
|
break |
|
if paused: |
|
time.sleep(0.5) |
|
continue |
|
|
|
|
|
batch = items[i:i + batch_size] |
|
for word in batch: |
|
|
|
if word.lower() in shopping_keywords: |
|
shopping_items.append(word) |
|
elif word.lower() in gaming_keywords: |
|
gaming_items.append(word) |
|
elif word.lower() in streaming_keywords: |
|
streaming_items.append(word) |
|
else: |
|
|
|
result = classifier(word, categories) |
|
best_category = result['labels'][0] |
|
score = result['scores'][0] |
|
|
|
if best_category == "shop" and score > 0.5: |
|
shopping_items.append(word) |
|
elif best_category == "game" and score > 0.5: |
|
gaming_items.append(word) |
|
elif best_category == "stream" and score > 0.5: |
|
streaming_items.append(word) |
|
else: |
|
unknown_items.append(word) |
|
|
|
|
|
current_index = i + batch_size |
|
|
|
|
|
progress = min((current_index) / total_items, 1.0) |
|
progress_bar.progress(progress) |
|
|
|
|
|
update_results() |
|
|
|
|
|
time.sleep(0.1) |
|
|
|
|
|
def classify_parameters(items, categories, start_index=0): |
|
global paused, stopped, current_index |
|
total_items = len(items) |
|
for i in range(start_index, total_items, batch_size): |
|
if stopped: |
|
break |
|
if paused: |
|
time.sleep(0.5) |
|
continue |
|
|
|
|
|
batch = items[i:i + batch_size] |
|
for url in batch: |
|
|
|
params = re.findall(r'(\w+)=', url) |
|
for param in params: |
|
|
|
result = classifier(param, categories) |
|
best_category = result['labels'][0] |
|
score = result['scores'][0] |
|
|
|
if best_category == "shop" and score > 0.5: |
|
param_categories["shop_params"].append(param) |
|
elif best_category == "game" and score > 0.5: |
|
param_categories["game_params"].append(param) |
|
elif best_category == "stream" and score > 0.5: |
|
param_categories["stream_params"].append(param) |
|
else: |
|
param_categories["unknown_params"].append(param) |
|
|
|
|
|
page_type_match = re.search(r'(\w+\.[a-z]+)\?', url) |
|
if page_type_match: |
|
page_type = page_type_match.group(1) |
|
if 'full_page_types' not in st.session_state: |
|
st.session_state.full_page_types = set() |
|
st.session_state.full_page_types.add(page_type) |
|
|
|
|
|
extension_match = re.search(r'\.([a-z]+)(\?|$)', url) |
|
if extension_match: |
|
ext = extension_match.group(1) |
|
if ext not in extensions: |
|
extensions[ext] = 0 |
|
extensions[ext] += 1 |
|
|
|
|
|
current_index = i + batch_size |
|
|
|
|
|
progress = min((current_index) / total_items, 1.0) |
|
progress_bar.progress(progress) |
|
|
|
|
|
update_results() |
|
|
|
|
|
time.sleep(0.1) |
|
|
|
|
|
def update_results(): |
|
|
|
st.session_state.shopping_text = "\n".join(shopping_items) |
|
st.session_state.gaming_text = "\n".join(gaming_items) |
|
st.session_state.streaming_text = "\n".join(streaming_items) |
|
st.session_state.unknown_text = "\n".join(unknown_items) |
|
|
|
|
|
st.session_state.shop_params = "\n".join(set(param_categories["shop_params"])) |
|
st.session_state.game_params = "\n".join(set(param_categories["game_params"])) |
|
st.session_state.stream_params = "\n".join(set(param_categories["stream_params"])) |
|
st.session_state.unknown_params = "\n".join(set(param_categories["unknown_params"])) |
|
|
|
|
|
st.session_state.extensions_text = "\n".join(extensions.keys()) |
|
|
|
|
|
st.session_state.full_page_types = "\n".join(st.session_state.full_page_types) |
|
|
|
|
|
def export_results(key, filename): |
|
with open(filename, "w") as f: |
|
f.write(st.session_state[key]) |
|
st.success(f"Results exported to {filename}") |
|
|
|
|
|
if st.button("Start"): |
|
stopped = False |
|
paused = False |
|
current_index = 0 |
|
|
|
if operation == "Filter Keywords": |
|
classify_keywords_batch(items, categories, start_index=current_index) |
|
elif operation == "Extra & Filter Param (URLs)": |
|
classify_parameters(items, categories, start_index=current_index) |
|
|
|
|
|
if pause_button: |
|
paused = True |
|
st.write("Process paused.") |
|
|
|
|
|
if continue_button and paused: |
|
paused = False |
|
st.write("Process resumed.") |
|
if operation == "Filter Keywords": |
|
classify_keywords_batch(items, categories, start_index=current_index) |
|
elif operation == "Extra & Filter Param (URLs)": |
|
classify_parameters(items, categories, start_index=current_index) |
|
|
|
|
|
if stop_button: |
|
stopped = True |
|
st.write("Process stopped.") |
|
|
|
|
|
if operation == "Filter Keywords": |
|
|
|
st.header("Shopping Keywords") |
|
if 'shopping_text' not in st.session_state: |
|
st.session_state.shopping_text = "" |
|
st.text_area("Copy the shopping keywords here:", value=st.session_state.shopping_text, height=200, key="shopping") |
|
st.button("Export Shopping Keywords", on_click=export_results, args=("shopping_text", "shopping_keywords.txt")) |
|
|
|
st.header("Gaming Keywords") |
|
if 'gaming_text' not in st.session_state: |
|
st.session_state.gaming_text = "" |
|
st.text_area("Copy the gaming keywords here:", value=st.session_state.gaming_text, height=200, key="gaming") |
|
st.button("Export Gaming Keywords", on_click=export_results, args=("gaming_text", "gaming_keywords.txt")) |
|
|
|
st.header("Streaming Keywords") |
|
if 'streaming_text' not in st.session_state: |
|
st.session_state.streaming_text = "" |
|
st.text_area("Copy the streaming keywords here:", value=st.session_state.streaming_text, height=200, key="streaming") |
|
st.button("Export Streaming Keywords", on_click=export_results, args=("streaming_text", "streaming_keywords.txt")) |
|
|
|
st.header("Unknown Keywords") |
|
if 'unknown_text' not in st.session_state: |
|
st.session_state.unknown_text = "" |
|
st.text_area("Copy the unknown keywords here:", value=st.session_state.unknown_text, height=200, key="unknown") |
|
st.button("Export Unknown Keywords", on_click=export_results, args=("unknown_text", "unknown_keywords.txt")) |
|
|
|
elif operation == "Extra & Filter Param (URLs)": |
|
|
|
st.header("Shop Parameters") |
|
if 'shop_params' not in st.session_state: |
|
st.session_state.shop_params = "" |
|
st.text_area("Copy the shop parameters here:", value=st.session_state.shop_params, height=200, key="shop_params") |
|
st.button("Export Shop Parameters", on_click=export_results, args=("shop_params", "shop_params.txt")) |
|
|
|
st.header("Game Parameters") |
|
if 'game_params' not in st.session_state: |
|
st.session_state.game_params = "" |
|
st.text_area("Copy the game parameters here:", value=st.session_state.game_params, height=200, key="game_params") |
|
st.button("Export Game Parameters", on_click=export_results, args=("game_params", "game_params.txt")) |
|
|
|
st.header("Stream Parameters") |
|
if 'stream_params' not in st.session_state: |
|
st.session_state.stream_params = "" |
|
st.text_area("Copy the stream parameters here:", value=st.session_state.stream_params, height=200, key="stream_params") |
|
st.button("Export Stream Parameters", on_click=export_results, args=("stream_params", "stream_params.txt")) |
|
|
|
st.header("Unknown Parameters") |
|
if 'unknown_params' not in st.session_state: |
|
st.session_state.unknown_params = "" |
|
st.text_area("Copy the unknown parameters here:", value=st.session_state.unknown_params, height=200, key="unknown_params") |
|
st.button("Export Unknown Parameters", on_click=export_results, args=("unknown_params", "unknown_params.txt")) |
|
|
|
|
|
st.header("Full PageType") |
|
if 'full_page_types' not in st.session_state: |
|
st.session_state.full_page_types = "" |
|
st.text_area("Copy the full page types here:", value=st.session_state.full_page_types, height=200, key="full_page_types") |
|
st.button("Export Full PageTypes", on_click=export_results, args=("full_page_types", "full_page_types.txt")) |
|
|
|
|
|
st.header("File Extensions") |
|
if 'extensions_text' not in st.session_state: |
|
st.session_state.extensions_text = "" |
|
st.text_area("Copy the file extensions here:", value=st.session_state.extensions_text, height=200, key="extensions") |
|
st.button("Export File Extensions", on_click=export_results, args=("extensions_text", "file_extensions.txt")) |
|
|
|
else: |
|
st.warning("Please upload a text file to start classification.") |
