Spaces:
Sleeping
Sleeping
File size: 2,274 Bytes
37ff7dd |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 |
# smart_web_analyzer.py
import requests
from bs4 import BeautifulSoup
from transformers import pipeline
import torch
class WebAnalyzer:
def __init__(self):
self.device = 0 if torch.cuda.is_available() else -1
self.models = {
'summarize': pipeline("summarization", model="facebook/bart-large-cnn"),
'sentiment': pipeline("text-classification",
model="nlptown/bert-base-multilingual-uncased-sentiment"),
'topics': pipeline("zero-shot-classification",
model="facebook/bart-large-mnli")
}
def fetch_content(self, url: str) -> str:
"""Fetch webpage content with custom headers"""
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
response = requests.get(url, headers=headers, timeout=15)
response.raise_for_status()
return response.text
def clean_html(self, html: str) -> str:
"""Basic HTML cleaning preserving all tags"""
soup = BeautifulSoup(html, 'html.parser')
return soup.prettify()
def analyze(self, url: str, modes: list) -> dict:
"""Core analysis pipeline"""
results = {}
try:
html = self.fetch_content(url)
results['clean_text'] = self.clean_html(html)
if 'summarize' in modes:
results['summary'] = self.models['summarize'](html, max_length=150)[0]['summary_text']
if 'sentiment' in modes:
sentiment = self.models['sentiment'](html[:512])[0]
results['sentiment'] = f"{sentiment['label']} ({sentiment['score']:.2f})"
if 'topics' in modes:
topics = self.models['topics'](html[:512],
candidate_labels=["Technology", "AI", "Business",
"Science", "Politics"])
results['topics'] = {topic: score for topic, score
in zip(topics['labels'], topics['scores'])}
except Exception as e:
results['error'] = str(e)
return results |