import gradio as gr
import pandas as pd
import os
import subprocess
import sys
# Install spaCy model
os.system("python -m spacy download en_core_web_sm")
def process_tweets(files, reset_processing=False):
# Save uploaded files
file_paths = []
for file in files:
if file.name.endswith('.csv'):
# Ensure directory exists
os.makedirs("projects_twitter_post", exist_ok=True)
# Save file to the directory
dest_path = f"projects_twitter_post/{os.path.basename(file.name)}"
os.system(f"cp {file.name} {dest_path}")
file_paths.append(dest_path)
if not file_paths:
return "No CSV files uploaded. Please upload CSV files containing tweet data."
# Run the processing script
reset_flag = "--reset" if reset_processing else ""
result = subprocess.run(
f"python process_tweet_huggingface.py {reset_flag}",
shell=True,
capture_output=True,
text=True
)
# Check if output files were created
output_files = []
for file_path in file_paths:
base_name = os.path.basename(file_path).replace('.csv', '')
processed_path = f"projects_twitter_post/{base_name}_processed.csv"
analysis_path = f"projects_twitter_post/{base_name}_analysis.csv"
if os.path.exists(processed_path):
output_files.append(processed_path)
if os.path.exists(analysis_path):
output_files.append(analysis_path)
return_files = [f for f in output_files if os.path.exists(f)]
log_output = result.stdout + "\n" + result.stderr
return log_output, return_files
with gr.Blocks() as demo:
gr.Markdown("# Crypto Tweet Processor")
gr.Markdown("Upload CSV files containing tweet data to process")
with gr.Row():
files_input = gr.File(file_count="multiple", label="Upload CSV Files")
reset_checkbox = gr.Checkbox(label="Reset Processing", value=False)
process_btn = gr.Button("Process Tweets")
output_text = gr.Textbox(label="Processing Log")
output_files = gr.File(label="Processed Files", file_count="multiple")
process_btn.click(
process_tweets,
inputs=[files_input, reset_checkbox],
outputs=[output_text, output_files]
)
# Add the modified processing script code here
with open("process_tweet_huggingface.py", "w") as f:
f.write(
import os
import re
import json
import numpy as np
import torch
import math
import gc
from tqdm import tqdm
from transformers import AutoTokenizer, AutoModelForMaskedLM, pipeline
import spacy
# ==============================================
# COLAB SETUP - Run these cells first in Colab
# ==============================================
# Uncomment and run this cell to mount your Google Drive
"""
from google.colab import drive
drive.mount('/content/drive')
"""
# Uncomment and run this cell to install required packages
"""
!pip install pandas tqdm transformers spacy
!python -m spacy download en_core_web_sm
"""
# Uncomment and run this cell to verify GPU availability
"""
import torch
print(f"GPU available: {torch.cuda.is_available()}")
print(f"GPU device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'None'}")
"""
# ==============================================
# Constants - Update these paths for your setup
# ==============================================
# Update this to your Google Drive path
DRIVE_PATH = "./projects_twitter_post"
OUTPUT_FOLDER = f"{DRIVE_PATH}"
CHECKPOINT_FILE = f"{OUTPUT_FOLDER}/processing_checkpoint.json"
BATCH_SIZE = 500 # Reduced batch size for GPU memory management
# Create output folder if it doesn't exist
if not os.path.exists(OUTPUT_FOLDER):
os.makedirs(OUTPUT_FOLDER)
# ==============================================
# Model Initialization with GPU Acceleration
# ==============================================
print("Loading RoBERTa model...")
model_name = "roberta-base"
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Using device: {device}")
# Initialize with GPU acceleration
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForMaskedLM.from_pretrained(model_name).to(device)
nlp_pipeline = pipeline("fill-mask", model=model_name, device=0 if torch.cuda.is_available() else -1)
# Initialize sentiment analysis pipeline
print("Loading sentiment analysis model...")
try:
# Using a Twitter-specific sentiment model for better results on social media text
sentiment_model = "cardiffnlp/twitter-roberta-base-sentiment"
sentiment_pipeline = pipeline("sentiment-analysis", model=sentiment_model, device=0 if torch.cuda.is_available() else -1)
SENTIMENT_AVAILABLE = True
except Exception as e:
print(f"Error loading sentiment model: {e}")
# Fallback to a simpler sentiment model if the Twitter-specific one fails
try:
sentiment_pipeline = pipeline("sentiment-analysis", device=0 if torch.cuda.is_available() else -1)
SENTIMENT_AVAILABLE = True
except:
print("Sentiment analysis not available. Continuing without sentiment analysis.")
SENTIMENT_AVAILABLE = False
# Try to load spaCy for basic text preprocessing
try:
import spacy
spacy_nlp = spacy.load("en_core_web_sm")
SPACY_AVAILABLE = True
except:
SPACY_AVAILABLE = False
print("SpaCy not available. Using basic text processing instead.")
# Crypto-specific keywords with hierarchical categories
CRYPTO_TAXONOMY = {
"COIN": {
"MAJOR": [
"bitcoin", "ethereum", "btc", "eth", "bnb", "xrp", "sol", "doge",
"cardano", "polkadot", "dot", "avalanche", "avax", "solana", "polygon", "matic"
],
"STABLECOIN": [
"tether", "usdt", "usdc", "busd", "dai", "frax", "tusd", "usdd", "lusd", "gusd", "husd"
],
"ALTCOIN": [
"litecoin", "ltc", "chainlink", "link", "stellar", "xlm", "dogecoin", "shib",
"tron", "trx", "cosmos", "atom", "near", "algo", "fantom", "ftm", "monero", "xmr"
],
"DEFI": [
"uniswap", "uni", "aave", "sushi", "cake", "comp", "maker", "mkr", "curve", "crv",
"yearn", "yfi", "compound", "balancer", "bal", "synthetix", "snx"
],
"UTILITY": [
"filecoin", "fil", "the graph", "grt", "arweave", "ar", "chainlink", "link",
"helium", "hnt", "theta", "icp"
],
"NFT": [
"enjin", "enj", "decentraland", "mana", "sandbox", "sand", "axie", "axs",
"gala", "apecoin", "ape", "flow", "ens", "stepn", "gmt"
]
},
"TECH": {
"CONCEPTS": [
"blockchain", "defi", "nft", "dao", "smart contract", "web3", "dapp", "protocol",
"consensus", "tokenomics", "tokenization"
],
"CHAIN_TYPES": [
"layer1", "layer2", "rollup", "sidechain", "mainnet", "testnet", "devnet",
"pow", "pos", "poh", "pbft", "dpos"
],
"PRIVACY": [
"zk", "zk-rollups", "zero-knowledge", "zkp", "zksnark", "zkstark", "mpc",
"privacy", "private", "anonymous", "confidential", "encrypted"
],
"SECTORS": [
"defi", "cefi", "gamefi", "metaverse", "socialfi", "fintech", "realfi",
"play-to-earn", "move-to-earn", "learn-to-earn", "x-to-earn", "defai", "depin", "desci",
"refi", "did", "dedata", "dedao", "deid", "deai", "degov", "decloud", "dehealth",
"decex", "deinsurance", "deworkplace", "public goods", "zk", "ordinals", "soulbound",
"onchain gaming", "ai agents", "infrastructure", "credentials", "restaking", "modular blockchain",
"liquid staking", "real world assets", "rwa", "synthetic assets", "account abstraction"
]
},
"ACTION": {
"TRADING": [
"buy", "sell", "long", "short", "margin", "leverage", "trade", "swap",
"arbitrage", "dca", "ape", "pump", "dump", "moon", "ath", "atl", "breakout",
"correction", "consolidation", "accumulate", "distribute", "front run", "front runner",
"front running", "mev", "sandwich attack"
],
"DEFI": [
"stake", "yield", "farm", "lend", "borrow", "supply", "withdraw", "claim",
"harvest", "flash loan", "liquidate", "collateralize", "wrap", "unwrap", "bridge",
"provide liquidity", "withdraw liquidity", "impermanent loss"
],
"GOVERNANCE": [
"delegate", "vote", "propose", "governance", "dao", "snapshot", "quorum",
"execution", "timelock", "veto"
],
"NFT": [
"mint", "airdrop", "whitelist", "burn", "floor price", "rarity", "trait", "pfp",
"collection", "secondary", "flip"
],
"DEVELOPMENT": [
"deploy", "audit", "fork", "bootstrap", "initiate", "merge", "split",
"rebase", "optimize", "gas optimization", "implement", "compile"
]
},
"PLATFORM": {
"EXCHANGE": [
"coinbase", "binance", "kraken", "kucoin", "ftx", "okx", "bybit", "bitfinex",
"huobi", "gate", "gemini", "bitstamp", "bittrex", "crypto.com", "cex", "dex"
],
"WALLET": [
"metamask", "phantom", "trust wallet", "ledger", "trezor", "argent", "rainbow",
"wallet", "hot wallet", "cold storage", "hardware wallet", "seed phrase"
],
"NFT_MARKET": [
"opensea", "rarible", "foundation", "superrare", "looksrare", "blur", "magic eden",
"nifty gateway", "zora", "x2y2", "element"
],
"INFRA": [
"alchemy", "infura", "moralis", "quicknode", "ceramic", "arweave", "ipfs",
"node", "rpc", "api", "indexer", "subgraph"
]
},
"NETWORK": {
"LAYER1": [
"ethereum", "bitcoin", "solana", "avalanche", "polygon", "bnb chain", "bsc",
"cardano", "polkadot", "cosmos", "algorand", "tezos", "flow", "near", "tron"
],
"LAYER2": [
"arbitrum", "optimism", "zksync", "starknet", "base", "polygon", "loopring",
"immutablex", "metis", "boba", "aztec", "validium", "zkevm"
],
"INTEROPERABILITY": [
"cosmos", "polkadot", "kusama", "moonbeam", "moonriver", "parachains", "relay chain",
"ibc", "cross-chain", "bridge"
]
},
"EVENTS": {
"MARKET": [
"bull market", "bear market", "bull run", "bear trap", "bull trap", "halving",
"capitulation", "golden cross", "death cross", "breakout", "resistance", "support"
],
"SECURITY": [
"hack", "exploit", "vulnerability", "scam", "phishing", "rug pull", "honeypot",
"flash crash", "attack", "51% attack", "front running", "sandwich attack", "mev extraction"
],
"TOKEN_EVENTS": [
"airdrop", "token unlock", "vesting", "ico", "ido", "ito", "ieo", "fair launch",
"private sale", "seed round", "listing", "delisting"
]
},
"METRICS": {
"FINANCIAL": [
"apy", "apr", "roi", "tvl", "market cap", "mcap", "volume", "liquidity", "supply",
"circulating supply", "total supply", "max supply", "inflation", "deflation",
"volatility", "dominance"
],
"TECHNICAL": [
"gas fee", "gas price", "gas limit", "slippage", "impermanent loss", "yield",
"hashrate", "difficulty", "tps", "latency", "finality", "block time", "block size",
"block reward"
]
},
"COMMUNITY": {
"ROLES": [
"whale", "degen", "anon", "influencer", "kol", "thought leader", "ambassador",
"advocate", "og", "contributor", "dev", "builder", "founder", "investor", "vc",
"angel", "team", "core team", "front runner", "mev bot", "searcher", "validator",
"miner", "node operator", "liquidity provider", "market maker", "arbitrageur"
],
"SLANG": [
"diamond hands", "paper hands", "wagmi", "ngmi", "gm", "gn", "ser", "based",
"crypto twitter", "ct", "alpha", "dyor", "fomo", "fud", "hodl", "rekt"
]
}
}
# ==============================================
# Helper Functions
# ==============================================
def clear_gpu_memory():
"""Clear GPU memory to prevent OOM errors"""
if torch.cuda.is_available():
torch.cuda.empty_cache()
gc.collect()
def load_checkpoint():
"""Load processing checkpoint if it exists"""
if os.path.exists(CHECKPOINT_FILE):
with open(CHECKPOINT_FILE, 'r') as f:
return json.load(f)
return {'last_processed_index': 0}
def save_checkpoint(index):
"""Save the current processing index to a checkpoint file"""
with open(CHECKPOINT_FILE, 'w') as f:
json.dump({'last_processed_index': index}, f)
def identify_crypto_entities(text: str) -> list:
"""
Identify crypto-specific entities in text using the hierarchical taxonomy.
Args:
text (str): Text to analyze
Returns:
list: List of tuples (entity, main_category, sub_category)
"""
if not isinstance(text, str):
return []
text_lower = text.lower()
found_entities = []
# Search for each entity in the taxonomy
for main_cat, subcats in CRYPTO_TAXONOMY.items():
for subcat, terms in subcats.items():
for term in terms:
# Avoid matching partial words (ensure word boundaries)
pattern = r'\b' + re.escape(term) + r'\b'
if re.search(pattern, text_lower):
found_entities.append((term, main_cat, subcat))
return found_entities
def clean_text(text: str) -> str:
"""Clean text while preserving mentions and hashtags"""
if not isinstance(text, str):
return ""
# Remove URLs
text = re.sub(r'http\S+', '', text)
# Remove non-alphanumeric characters (except mentions, hashtags, and spaces)
text = re.sub(r'[^\w\s@#]', ' ', text)
# Remove extra whitespace
text = re.sub(r'\s+', ' ', text).strip()
return text.lower()
def process_nlp_text(text: str) -> str:
"""Process text with advanced NLP (lemmatization, stopword removal)"""
if not isinstance(text, str):
return ""
# Basic cleaning
text = clean_text(text)
if SPACY_AVAILABLE:
# Process with spaCy for advanced NLP
doc = spacy_nlp(text)
# Lemmatize and remove stopwords
processed_tokens = [token.lemma_ for token in doc if not token.is_stop and not token.is_punct]
return " ".join(processed_tokens)
else:
# Fallback to basic cleaning if spaCy is not available
return text
def extract_mentions(text: str) -> list:
"""Extract @mentions from text"""
if not isinstance(text, str):
return []
return re.findall(r'@(\w+)', text)
def extract_hashtags(text: str) -> list:
"""Extract #hashtags from text"""
if not isinstance(text, str):
return []
return re.findall(r'#(\w+)', text)
def extract_urls(text: str) -> list:
"""Extract URLs from text"""
if not isinstance(text, str):
return []
urls = re.findall(r'(https?://\S+)', text)
return urls
def analyze_sentiment(text: str) -> dict:
"""
Analyze the sentiment of a text using the sentiment analysis pipeline.
Args:
text (str): The text to analyze
Returns:
dict: A dictionary containing sentiment label and score
"""
if not SENTIMENT_AVAILABLE or not text.strip():
return {"sentiment": "unknown", "sentiment_score": 0.0, "sentiment_magnitude": 0.0}
try:
# Pre-process the text to improve sentiment analysis accuracy
# Limit text length to avoid errors with very long tweets
truncated_text = text[:512] if len(text) > 512 else text
# Get sentiment prediction
sentiment_result = sentiment_pipeline(truncated_text)[0]
label = sentiment_result['label']
score = sentiment_result['score']
# Map to standardized format (positive, negative, neutral)
sentiment_mapping = {
'LABEL_0': 'negative',
'LABEL_1': 'neutral',
'LABEL_2': 'positive',
'NEGATIVE': 'negative',
'NEUTRAL': 'neutral',
'POSITIVE': 'positive'
}
standardized_sentiment = sentiment_mapping.get(label, label.lower())
# Calculate magnitude (confidence) - useful for filtering high-confidence sentiments
magnitude = abs(score - 0.5) * 2 if standardized_sentiment != 'neutral' else score
return {
"sentiment": standardized_sentiment,
"sentiment_score": score,
"sentiment_magnitude": magnitude
}
except Exception as e:
print(f"Error in sentiment analysis: {e}")
return {"sentiment": "error", "sentiment_score": 0.0, "sentiment_magnitude": 0.0}
def process_with_nlp(text: str) -> dict:
"""
Process text with NLP to extract named entities, key phrases, etc.
Args:
text (str): The text to process
Returns:
dict: A dictionary containing NLP processing results
"""
results = {
"named_entities": [],
"pos_tags": [],
"lemmatized_tokens": [],
"key_phrases": [],
"important_nouns": [],
"sentiment_analysis": {"sentiment": "unknown", "sentiment_score": 0.0, "sentiment_magnitude": 0.0}
}
if not text or text.isspace():
return results
# First, analyze sentiment
results["sentiment_analysis"] = analyze_sentiment(text)
try:
# Use spaCy for advanced NLP if available
if SPACY_AVAILABLE:
doc = spacy_nlp(text)
# Extract named entities (excluding crypto entities which are handled separately)
results["named_entities"] = [(ent.text, ent.label_) for ent in doc.ents]
# Extract POS tags for content words
results["pos_tags"] = [(token.text, token.pos_) for token in doc
if token.pos_ in ["NOUN", "VERB", "ADJ", "ADV"] and not token.is_stop]
# Get lemmatized tokens (normalized words)
results["lemmatized_tokens"] = [token.lemma_ for token in doc
if not token.is_stop and not token.is_punct and token.text.strip()]
# Extract important nouns (potential topics)
results["important_nouns"] = [token.text for token in doc
if token.pos_ == "NOUN" and not token.is_stop]
# Try to extract key phrases using noun chunks
results["key_phrases"] = [chunk.text for chunk in doc.noun_chunks
if len(chunk.text.split()) > 1]
# If key phrases are empty, use RoBERTa to attempt extraction
if not results["key_phrases"] and len(text.split()) > 3:
try:
# Create a masked sentence from the text
words = text.split()
if len(words) > 5:
# Get 3 random positions to mask
import random
positions = sorted(random.sample(range(len(words)), min(3, len(words))))
# Create masked sentences
key_terms = []
for pos in positions:
words_copy = words.copy()
words_copy[pos] = tokenizer.mask_token
masked_text = " ".join(words_copy)
# Get predictions for the masked token
predictions = nlp_pipeline(masked_text, top_k=2)
for prediction in predictions:
key_terms.append(prediction["token_str"].strip())
results["key_phrases"].extend(key_terms)
except Exception as e:
print(f"Error in key phrase extraction: {e}")
# Ensure all results are strings for CSV output
results["named_entities"] = ";".join([f"{ent[0]}:{ent[1]}" for ent in results["named_entities"]])
results["pos_tags"] = ";".join([f"{tag[0]}:{tag[1]}" for tag in results["pos_tags"]])
results["lemmatized_tokens"] = ";".join(results["lemmatized_tokens"])
results["key_phrases"] = ";".join(list(set(results["key_phrases"]))) # Remove duplicates
results["important_nouns"] = ";".join(list(set(results["important_nouns"]))) # Remove duplicates
except Exception as e:
print(f"Error in NLP processing: {e}")
# Clear GPU memory after processing
if (results["named_entities"].count(";") > 100) or (len(text) > 1000):
clear_gpu_memory()
return results
def process_tweet(text: str) -> tuple:
"""
Process a tweet to extract mentions, hashtags, URLs, crypto entities, and perform NLP analysis.
Also performs sentiment analysis.
Args:
text (str): The tweet text to process
Returns:
tuple: A tuple containing mentions, hashtags, URLs, NLP results, and sentiment analysis
"""
if not text or not isinstance(text, str):
return [], [], [], "", "", {}, {"sentiment": "unknown", "sentiment_score": 0.0, "sentiment_magnitude": 0.0}
# Clean the text while preserving mentions and hashtags
cleaned_text = clean_text(text)
# Process text with NLP
processed_text = process_nlp_text(text)
# Extract mentions, hashtags, and URLs
mentions = extract_mentions(text)
hashtags = extract_hashtags(text)
urls = extract_urls(text)
# Identify crypto entities
crypto_entities = identify_crypto_entities(text)
# Process with NLP models
nlp_results = process_with_nlp(text)
# Ensure we have the sentiment analysis results
sentiment_results = nlp_results.pop("sentiment_analysis", {"sentiment": "unknown", "sentiment_score": 0.0, "sentiment_magnitude": 0.0})
# Add crypto entities to the named entities
formatted_crypto_entities = [f"{entity}:{main_cat}.{sub_cat}" for entity, main_cat, sub_cat in crypto_entities]
# If named_entities is a string (joined with semicolons), we need to handle differently
if isinstance(nlp_results.get("named_entities", ""), str):
nlp_results["named_entities"] = nlp_results.get("named_entities", "")
if nlp_results["named_entities"] and formatted_crypto_entities:
nlp_results["named_entities"] += ";" + ";".join(formatted_crypto_entities)
elif formatted_crypto_entities:
nlp_results["named_entities"] = ";".join(formatted_crypto_entities)
return mentions, hashtags, urls, cleaned_text, processed_text, nlp_results, sentiment_results
def process_batch(df_batch):
"""Process a batch of tweets"""
processed_data = []
for idx, row in df_batch.iterrows():
text = row.get('text', '')
# Process the tweet
mentions, hashtags, urls, cleaned_text, processed_text, nlp_results, sentiment_results = process_tweet(text)
# Create a dictionary with the results
result = {
'id': row.get('id', ''),
'original_text': text, # Store the original text
'cleaned_text': cleaned_text,
'nlp_processed_text': processed_text,
'extracted_mentions': ';'.join(mentions),
'extracted_hashtags': ';'.join(hashtags),
'extracted_urls': ';'.join(urls),
'named_entities': nlp_results.get('named_entities', ''),
'pos_tags': nlp_results.get('pos_tags', ''),
'lemmatized_tokens': nlp_results.get('lemmatized_tokens', ''),
'key_phrases': nlp_results.get('key_phrases', ''),
'important_nouns': nlp_results.get('important_nouns', ''),
'sentiment': sentiment_results.get('sentiment', 'unknown'),
'sentiment_score': sentiment_results.get('sentiment_score', 0.0),
'sentiment_magnitude': sentiment_results.get('sentiment_magnitude', 0.0)
}
processed_data.append(result)
return pd.DataFrame(processed_data)
# ==============================================
# Main Processing Function
# ==============================================
def main(reset_checkpoint=False, input_file=None):
"""
Main function to process tweets
Args:
reset_checkpoint (bool): Whether to reset the checkpoint and process all data
input_file (str): Optional specific input file to process, otherwise processes all CSV files
"""
if reset_checkpoint and os.path.exists(CHECKPOINT_FILE):
os.remove(CHECKPOINT_FILE)
print("Checkpoint reset. Will process all data from the beginning.")
# Get list of CSV files to process
if input_file:
# Process a specific file
input_files = [input_file]
else:
# Find all CSV files in the OUTPUT_FOLDER
import glob
input_files = glob.glob(f"{OUTPUT_FOLDER}/*.csv")
# Exclude our output files
input_files = [f for f in input_files if not any(x in f for x in ["_processed.csv", "_analysis.csv"])]
if not input_files:
print(f"No input CSV files found in {OUTPUT_FOLDER}")
return
print(f"Found {len(input_files)} files to process: {[os.path.basename(f) for f in input_files]}")
# Process each file
for input_csv in input_files:
print(f"\nProcessing file: {os.path.basename(input_csv)}")
print("Loading dataset...")
# Check if input file exists
if not os.path.exists(input_csv):
print(f"Input file {input_csv} not found. Skipping.")
continue
# Load the dataset
try:
df = pd.read_csv(input_csv)
print(f"Loaded dataset with {len(df)} records and {len(df.columns)} columns.")
except Exception as e:
print(f"Error loading {input_csv}: {e}")
continue
# Load checkpoint if it exists
checkpoint = load_checkpoint()
start_idx = checkpoint['last_processed_index']
# For simplicity, reset checkpoints between files
start_idx = 0
save_checkpoint(0)
print("\nProcessing tweets...")
print(f"Starting from index {start_idx}")
# Filter to only unprocessed rows
df_to_process = df.iloc[start_idx:]
if len(df_to_process) == 0:
print("No new data to process in this file.")
continue
# Process in batches for memory efficiency
batch_size = BATCH_SIZE
num_batches = math.ceil(len(df_to_process) / batch_size)
print(f"Processing in {num_batches} batches of {batch_size} records each")
processed_batches = []
# Create progress bar
for i in tqdm(range(num_batches)):
batch_start = i * batch_size
batch_end = min((i + 1) * batch_size, len(df_to_process))
# Get current batch
df_batch = df_to_process.iloc[batch_start:batch_end]
# Process the batch
processed_batch = process_batch(df_batch)
processed_batches.append(processed_batch)
# Save checkpoint
save_checkpoint(start_idx + batch_end)
# Save intermediate results every 5 batches to prevent data loss in case of session timeout
if i % 5 == 0 and i > 0:
file_basename = os.path.splitext(os.path.basename(input_csv))[0]
interim_df = pd.concat(processed_batches, ignore_index=True)
interim_file = f"{OUTPUT_FOLDER}/{file_basename}_interim_{i}.csv"
interim_df.to_csv(interim_file, index=False)
print(f"\nSaved interim results to {interim_file}")
# Clear memory
clear_gpu_memory()
# Combine all batches
if processed_batches:
file_basename = os.path.splitext(os.path.basename(input_csv))[0]
final_df = pd.concat(processed_batches, ignore_index=True)
# Calculate statistics columns
final_df["mention_count"] = final_df["extracted_mentions"].str.count(";") + (final_df["extracted_mentions"] != "").astype(int)
final_df["hashtag_count"] = final_df["extracted_hashtags"].str.count(";") + (final_df["extracted_hashtags"] != "").astype(int)
final_df["entity_count"] = final_df["named_entities"].str.count(";") + (final_df["named_entities"] != "").astype(int)
# Save the full processed dataset
output_file = f"{OUTPUT_FOLDER}/{file_basename}_processed.csv"
final_df.to_csv(output_file, index=False)
print(f"Processed data saved to {output_file}")
# Create a lighter version with just the analysis
analysis_columns = [
"id", "original_text", "cleaned_text", "nlp_processed_text",
"extracted_mentions", "extracted_hashtags", "extracted_urls",
"named_entities", "key_phrases", "important_nouns",
"sentiment", "sentiment_score", "sentiment_magnitude",
"mention_count", "hashtag_count", "entity_count"
]
# Ensure all columns exist before subsetting
available_columns = [col for col in analysis_columns if col in final_df.columns]
analysis_df = final_df[available_columns]
analysis_file = f"{OUTPUT_FOLDER}/{file_basename}_analysis.csv"
analysis_df.to_csv(analysis_file, index=False)
print(f"Analysis results saved to {analysis_file}")
# Print statistics
print(f"\nAnalysis completed successfully!")
print(f"Total records: {len(final_df)}")
print(f"Tweets with Mentions: {(final_df['extracted_mentions'] != '').sum()}")
print(f"Tweets with Hashtags: {(final_df['extracted_hashtags'] != '').sum()}")
print(f"Tweets with Named Entities: {(final_df['named_entities'] != '').sum()}")
# Print sentiment statistics
sentiment_counts = final_df['sentiment'].value_counts()
print("\nSentiment Distribution:")
for sentiment, count in sentiment_counts.items():
percentage = (count / len(final_df)) * 100
print(f" {sentiment}: {count} tweets ({percentage:.1f}%)")
# Get average sentiment scores
avg_score = final_df['sentiment_score'].mean()
avg_magnitude = final_df['sentiment_magnitude'].mean()
print(f"\nAverage sentiment score: {avg_score:.3f}")
print(f"Average sentiment magnitude: {avg_magnitude:.3f}")
# Get top entities by sentiment
positive_entities = []
for idx, row in final_df[final_df['sentiment'] == 'positive'].iterrows():
entities = row['named_entities'].split(';') if isinstance(row['named_entities'], str) and row['named_entities'] else []
for entity in entities:
if entity and ':' in entity:
entity_name = entity.split(':')[0]
positive_entities.append(entity_name)
# Get the most common positive entities
from collections import Counter
top_positive = Counter(positive_entities).most_common(5)
if top_positive:
print("\nTop entities with positive sentiment:")
for entity, count in top_positive:
print(f" {entity}: {count} mentions")
# Print sample results
print("\nSample of processing results:")
for i, row in analysis_df.head(3).iterrows():
print(f"\nOriginal Text: {row['original_text']}")
print(f"Cleaned Text: {row['cleaned_text']}")
print(f"NLP Processed Text: {row['nlp_processed_text']}")
print(f"Mentions: {row['extracted_mentions']}")
print(f"Hashtags: {row['extracted_hashtags']}")
print(f"Named Entities: {row['named_entities']}")
print(f"Key Phrases: {row['key_phrases']}")
print(f"Sentiment: {row['sentiment']} (Score: {row['sentiment_score']:.3f}, Magnitude: {row['sentiment_magnitude']:.3f})")
print("-" * 80)
# Delete interim files
import glob
interim_files = glob.glob(f"{OUTPUT_FOLDER}/{file_basename}_interim_*.csv")
for f in interim_files:
try:
os.remove(f)
print(f"Deleted interim file: {os.path.basename(f)}")
except:
pass
# Clear memory after processing each file
clear_gpu_memory()
else:
print("No data processed for this file.")
# Clean up checkpoint file after successful processing
if os.path.exists(CHECKPOINT_FILE):
os.remove(CHECKPOINT_FILE)
print("\nAll files processed successfully!")
# ==============================================
# Colab Usage Example
# ==============================================
"""
# EXAMPLE USAGE IN COLAB:
# 1. Install packages and mount drive
from google.colab import drive
drive.mount('/content/drive')
# 2. Process one specific file
input_file = "/content/drive/MyDrive/projects_twitter_post/zilliqa.csv"
main(reset_checkpoint=True, input_file=input_file)
# 3. Process all files
main(reset_checkpoint=True)
"""
if __name__ == "__main__":
import sys
# Check if --reset flag is provided
reset_checkpoint = "--reset" in sys.argv
# Check if --file flag is provided
input_file = None
if "--file" in sys.argv:
try:
input_file = sys.argv[sys.argv.index("--file") + 1]
except IndexError:
print("Error: --file flag requires a filename argument")
sys.exit(1)
# Run the main function
main(reset_checkpoint=reset_checkpoint, input_file=input_file) )
demo.launch()