app.py

   import gradio as gr
   import pandas as pd
   import os
   import subprocess
   import sys
   
   # Install spaCy model
   os.system("python -m spacy download en_core_web_sm")
   
   def process_tweets(files, reset_processing=False):
       # Save uploaded files
       file_paths = []
       for file in files:
           if file.name.endswith('.csv'):
               # Ensure directory exists
               os.makedirs("projects_twitter_post", exist_ok=True)
               
               # Save file to the directory
               dest_path = f"projects_twitter_post/{os.path.basename(file.name)}"
               os.system(f"cp {file.name} {dest_path}")
               file_paths.append(dest_path)
       
       if not file_paths:
           return "No CSV files uploaded. Please upload CSV files containing tweet data."
       
       # Run the processing script
       reset_flag = "--reset" if reset_processing else ""
       result = subprocess.run(
           f"python process_tweet_huggingface.py {reset_flag}",
           shell=True,
           capture_output=True,
           text=True
       )
       
       # Check if output files were created
       output_files = []
       for file_path in file_paths:
           base_name = os.path.basename(file_path).replace('.csv', '')
           processed_path = f"projects_twitter_post/{base_name}_processed.csv"
           analysis_path = f"projects_twitter_post/{base_name}_analysis.csv"
           
           if os.path.exists(processed_path):
               output_files.append(processed_path)
           if os.path.exists(analysis_path):
               output_files.append(analysis_path)
       
       return_files = [f for f in output_files if os.path.exists(f)]
       
       log_output = result.stdout + "\n" + result.stderr
       
       return log_output, return_files
   
   with gr.Blocks() as demo:
       gr.Markdown("# Crypto Tweet Processor")
       gr.Markdown("Upload CSV files containing tweet data to process")
       
       with gr.Row():
           files_input = gr.File(file_count="multiple", label="Upload CSV Files")
           reset_checkbox = gr.Checkbox(label="Reset Processing", value=False)
       
       process_btn = gr.Button("Process Tweets")
       
       output_text = gr.Textbox(label="Processing Log")
       output_files = gr.File(label="Processed Files", file_count="multiple")
       
       process_btn.click(
           process_tweets,
           inputs=[files_input, reset_checkbox],
           outputs=[output_text, output_files]
       )
   
   # Add the modified processing script code here
   with open("process_tweet_huggingface.py", "w") as f:
       f.write(#!/usr/bin/env python3
"""
Tweet Processing Script for Google Colab - Enhanced with NLP and Sentiment Analysis
This version is optimized for Google Colab with GPU acceleration and Google Drive integration.
"""

import os
import re
import json
import pandas as pd
import numpy as np
import torch
import math
import gc
from tqdm import tqdm
from transformers import AutoTokenizer, AutoModelForMaskedLM, pipeline
import spacy

# ==============================================
# COLAB SETUP - Run these cells first in Colab
# ==============================================

# Uncomment and run this cell to mount your Google Drive
"""
from google.colab import drive
drive.mount('/content/drive')
"""

# Uncomment and run this cell to install required packages
"""
!pip install pandas tqdm transformers spacy
!python -m spacy download en_core_web_sm
"""

# Uncomment and run this cell to verify GPU availability
"""
import torch
print(f"GPU available: {torch.cuda.is_available()}")
print(f"GPU device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'None'}")
"""

# ==============================================
# Constants - Update these paths for your setup
# ==============================================

# Update this to your Google Drive path
DRIVE_PATH = "./projects_twitter_post"
OUTPUT_FOLDER = f"{DRIVE_PATH}"
CHECKPOINT_FILE = f"{OUTPUT_FOLDER}/processing_checkpoint.json"
BATCH_SIZE = 500  # Reduced batch size for GPU memory management

# Create output folder if it doesn't exist
if not os.path.exists(OUTPUT_FOLDER):
    os.makedirs(OUTPUT_FOLDER)

# ==============================================
# Model Initialization with GPU Acceleration
# ==============================================

print("Loading RoBERTa model...")
model_name = "roberta-base"
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Using device: {device}")

# Initialize with GPU acceleration
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForMaskedLM.from_pretrained(model_name).to(device)
nlp_pipeline = pipeline("fill-mask", model=model_name, device=0 if torch.cuda.is_available() else -1)

# Initialize sentiment analysis pipeline
print("Loading sentiment analysis model...")
try:
    # Using a Twitter-specific sentiment model for better results on social media text
    sentiment_model = "cardiffnlp/twitter-roberta-base-sentiment"
    sentiment_pipeline = pipeline("sentiment-analysis", model=sentiment_model, device=0 if torch.cuda.is_available() else -1)
    SENTIMENT_AVAILABLE = True
except Exception as e:
    print(f"Error loading sentiment model: {e}")
    # Fallback to a simpler sentiment model if the Twitter-specific one fails
    try:
        sentiment_pipeline = pipeline("sentiment-analysis", device=0 if torch.cuda.is_available() else -1)
        SENTIMENT_AVAILABLE = True
    except:
        print("Sentiment analysis not available. Continuing without sentiment analysis.")
        SENTIMENT_AVAILABLE = False

# Try to load spaCy for basic text preprocessing
try:
    import spacy
    spacy_nlp = spacy.load("en_core_web_sm")
    SPACY_AVAILABLE = True
except:
    SPACY_AVAILABLE = False
    print("SpaCy not available. Using basic text processing instead.")

# Crypto-specific keywords with hierarchical categories
CRYPTO_TAXONOMY = {
    "COIN": {
        "MAJOR": [
            "bitcoin", "ethereum", "btc", "eth", "bnb", "xrp", "sol", "doge",
            "cardano", "polkadot", "dot", "avalanche", "avax", "solana", "polygon", "matic"
        ],
        "STABLECOIN": [
            "tether", "usdt", "usdc", "busd", "dai", "frax", "tusd", "usdd", "lusd", "gusd", "husd"
        ],
        "ALTCOIN": [
            "litecoin", "ltc", "chainlink", "link", "stellar", "xlm", "dogecoin", "shib",
            "tron", "trx", "cosmos", "atom", "near", "algo", "fantom", "ftm", "monero", "xmr"
        ],
        "DEFI": [
            "uniswap", "uni", "aave", "sushi", "cake", "comp", "maker", "mkr", "curve", "crv",
            "yearn", "yfi", "compound", "balancer", "bal", "synthetix", "snx"
        ],
        "UTILITY": [
            "filecoin", "fil", "the graph", "grt", "arweave", "ar", "chainlink", "link", 
            "helium", "hnt", "theta", "icp"
        ],
        "NFT": [
            "enjin", "enj", "decentraland", "mana", "sandbox", "sand", "axie", "axs",
            "gala", "apecoin", "ape", "flow", "ens", "stepn", "gmt"
        ]
    },
    
    "TECH": {
        "CONCEPTS": [
            "blockchain", "defi", "nft", "dao", "smart contract", "web3", "dapp", "protocol",
            "consensus", "tokenomics", "tokenization"
        ],
        "CHAIN_TYPES": [
            "layer1", "layer2", "rollup", "sidechain", "mainnet", "testnet", "devnet",
            "pow", "pos", "poh", "pbft", "dpos"
        ],
        "PRIVACY": [
            "zk", "zk-rollups", "zero-knowledge", "zkp", "zksnark", "zkstark", "mpc",
            "privacy", "private", "anonymous", "confidential", "encrypted"
        ],
        "SECTORS": [
            "defi", "cefi", "gamefi", "metaverse", "socialfi", "fintech", "realfi",
            "play-to-earn", "move-to-earn", "learn-to-earn", "x-to-earn", "defai", "depin", "desci",
            "refi", "did", "dedata", "dedao", "deid", "deai", "degov", "decloud", "dehealth", 
            "decex", "deinsurance", "deworkplace", "public goods", "zk", "ordinals", "soulbound",
            "onchain gaming", "ai agents", "infrastructure", "credentials", "restaking", "modular blockchain",
            "liquid staking", "real world assets", "rwa", "synthetic assets", "account abstraction"
        ]
    },
    
    "ACTION": {
        "TRADING": [
            "buy", "sell", "long", "short", "margin", "leverage", "trade", "swap",
            "arbitrage", "dca", "ape", "pump", "dump", "moon", "ath", "atl", "breakout",
            "correction", "consolidation", "accumulate", "distribute", "front run", "front runner", 
            "front running", "mev", "sandwich attack"
        ],
        "DEFI": [
            "stake", "yield", "farm", "lend", "borrow", "supply", "withdraw", "claim",
            "harvest", "flash loan", "liquidate", "collateralize", "wrap", "unwrap", "bridge",
            "provide liquidity", "withdraw liquidity", "impermanent loss"
        ],
        "GOVERNANCE": [
            "delegate", "vote", "propose", "governance", "dao", "snapshot", "quorum",
            "execution", "timelock", "veto"
        ],
        "NFT": [
            "mint", "airdrop", "whitelist", "burn", "floor price", "rarity", "trait", "pfp",
            "collection", "secondary", "flip"
        ],
        "DEVELOPMENT": [
            "deploy", "audit", "fork", "bootstrap", "initiate", "merge", "split",
            "rebase", "optimize", "gas optimization", "implement", "compile"
        ]
    },
    
    "PLATFORM": {
        "EXCHANGE": [
            "coinbase", "binance", "kraken", "kucoin", "ftx", "okx", "bybit", "bitfinex", 
            "huobi", "gate", "gemini", "bitstamp", "bittrex", "crypto.com", "cex", "dex"
        ],
        "WALLET": [
            "metamask", "phantom", "trust wallet", "ledger", "trezor", "argent", "rainbow",
            "wallet", "hot wallet", "cold storage", "hardware wallet", "seed phrase"
        ],
        "NFT_MARKET": [
            "opensea", "rarible", "foundation", "superrare", "looksrare", "blur", "magic eden",
            "nifty gateway", "zora", "x2y2", "element"
        ],
        "INFRA": [
            "alchemy", "infura", "moralis", "quicknode", "ceramic", "arweave", "ipfs",
            "node", "rpc", "api", "indexer", "subgraph"
        ]
    },
    
    "NETWORK": {
        "LAYER1": [
            "ethereum", "bitcoin", "solana", "avalanche", "polygon", "bnb chain", "bsc",
            "cardano", "polkadot", "cosmos", "algorand", "tezos", "flow", "near", "tron"
        ],
        "LAYER2": [
            "arbitrum", "optimism", "zksync", "starknet", "base", "polygon", "loopring",
            "immutablex", "metis", "boba", "aztec", "validium", "zkevm"
        ],
        "INTEROPERABILITY": [
            "cosmos", "polkadot", "kusama", "moonbeam", "moonriver", "parachains", "relay chain",
            "ibc", "cross-chain", "bridge"
        ]
    },
    
    "EVENTS": {
        "MARKET": [
            "bull market", "bear market", "bull run", "bear trap", "bull trap", "halving",
            "capitulation", "golden cross", "death cross", "breakout", "resistance", "support"
        ],
        "SECURITY": [
            "hack", "exploit", "vulnerability", "scam", "phishing", "rug pull", "honeypot",
            "flash crash", "attack", "51% attack", "front running", "sandwich attack", "mev extraction"
        ],
        "TOKEN_EVENTS": [
            "airdrop", "token unlock", "vesting", "ico", "ido", "ito", "ieo", "fair launch",
            "private sale", "seed round", "listing", "delisting"
        ]
    },
    
    "METRICS": {
        "FINANCIAL": [
            "apy", "apr", "roi", "tvl", "market cap", "mcap", "volume", "liquidity", "supply",
            "circulating supply", "total supply", "max supply", "inflation", "deflation",
            "volatility", "dominance"
        ],
        "TECHNICAL": [
            "gas fee", "gas price", "gas limit", "slippage", "impermanent loss", "yield",
            "hashrate", "difficulty", "tps", "latency", "finality", "block time", "block size",
            "block reward"
        ]
    },
    
    "COMMUNITY": {
        "ROLES": [
            "whale", "degen", "anon", "influencer", "kol", "thought leader", "ambassador",
            "advocate", "og", "contributor", "dev", "builder", "founder", "investor", "vc",
            "angel", "team", "core team", "front runner", "mev bot", "searcher", "validator",
            "miner", "node operator", "liquidity provider", "market maker", "arbitrageur"
        ],
        "SLANG": [
            "diamond hands", "paper hands", "wagmi", "ngmi", "gm", "gn", "ser", "based",
            "crypto twitter", "ct", "alpha", "dyor", "fomo", "fud", "hodl", "rekt"
        ]
    }
}

# ==============================================
# Helper Functions
# ==============================================

def clear_gpu_memory():
    """Clear GPU memory to prevent OOM errors"""
    if torch.cuda.is_available():
        torch.cuda.empty_cache()
    gc.collect()

def load_checkpoint():
    """Load processing checkpoint if it exists"""
    if os.path.exists(CHECKPOINT_FILE):
        with open(CHECKPOINT_FILE, 'r') as f:
            return json.load(f)
    return {'last_processed_index': 0}

def save_checkpoint(index):
    """Save the current processing index to a checkpoint file"""
    with open(CHECKPOINT_FILE, 'w') as f:
        json.dump({'last_processed_index': index}, f)

def identify_crypto_entities(text: str) -> list:
    """
    Identify crypto-specific entities in text using the hierarchical taxonomy.
    
    Args:
        text (str): Text to analyze
    
    Returns:
        list: List of tuples (entity, main_category, sub_category)
    """
    if not isinstance(text, str):
        return []
    
    text_lower = text.lower()
    found_entities = []
    
    # Search for each entity in the taxonomy
    for main_cat, subcats in CRYPTO_TAXONOMY.items():
        for subcat, terms in subcats.items():
            for term in terms:
                # Avoid matching partial words (ensure word boundaries)
                pattern = r'\b' + re.escape(term) + r'\b'
                if re.search(pattern, text_lower):
                    found_entities.append((term, main_cat, subcat))
    
    return found_entities

def clean_text(text: str) -> str:
    """Clean text while preserving mentions and hashtags"""
    if not isinstance(text, str):
        return ""
    
    # Remove URLs
    text = re.sub(r'http\S+', '', text)
    
    # Remove non-alphanumeric characters (except mentions, hashtags, and spaces)
    text = re.sub(r'[^\w\s@#]', ' ', text)
    
    # Remove extra whitespace
    text = re.sub(r'\s+', ' ', text).strip()
    
    return text.lower()

def process_nlp_text(text: str) -> str:
    """Process text with advanced NLP (lemmatization, stopword removal)"""
    if not isinstance(text, str):
        return ""
    
    # Basic cleaning
    text = clean_text(text)
    
    if SPACY_AVAILABLE:
        # Process with spaCy for advanced NLP
        doc = spacy_nlp(text)
        
        # Lemmatize and remove stopwords
        processed_tokens = [token.lemma_ for token in doc if not token.is_stop and not token.is_punct]
        
        return " ".join(processed_tokens)
    else:
        # Fallback to basic cleaning if spaCy is not available
        return text

def extract_mentions(text: str) -> list:
    """Extract @mentions from text"""
    if not isinstance(text, str):
        return []
    return re.findall(r'@(\w+)', text)

def extract_hashtags(text: str) -> list:
    """Extract #hashtags from text"""
    if not isinstance(text, str):
        return []
    return re.findall(r'#(\w+)', text)

def extract_urls(text: str) -> list:
    """Extract URLs from text"""
    if not isinstance(text, str):
        return []
    urls = re.findall(r'(https?://\S+)', text)
    return urls

def analyze_sentiment(text: str) -> dict:
    """
    Analyze the sentiment of a text using the sentiment analysis pipeline.
    
    Args:
        text (str): The text to analyze
        
    Returns:
        dict: A dictionary containing sentiment label and score
    """
    if not SENTIMENT_AVAILABLE or not text.strip():
        return {"sentiment": "unknown", "sentiment_score": 0.0, "sentiment_magnitude": 0.0}
    
    try:
        # Pre-process the text to improve sentiment analysis accuracy
        # Limit text length to avoid errors with very long tweets
        truncated_text = text[:512] if len(text) > 512 else text
        
        # Get sentiment prediction
        sentiment_result = sentiment_pipeline(truncated_text)[0]
        label = sentiment_result['label']
        score = sentiment_result['score']
        
        # Map to standardized format (positive, negative, neutral)
        sentiment_mapping = {
            'LABEL_0': 'negative',
            'LABEL_1': 'neutral',
            'LABEL_2': 'positive',
            'NEGATIVE': 'negative',
            'NEUTRAL': 'neutral',
            'POSITIVE': 'positive'
        }
        
        standardized_sentiment = sentiment_mapping.get(label, label.lower())
        
        # Calculate magnitude (confidence) - useful for filtering high-confidence sentiments
        magnitude = abs(score - 0.5) * 2 if standardized_sentiment != 'neutral' else score
        
        return {
            "sentiment": standardized_sentiment,
            "sentiment_score": score,
            "sentiment_magnitude": magnitude
        }
    except Exception as e:
        print(f"Error in sentiment analysis: {e}")
        return {"sentiment": "error", "sentiment_score": 0.0, "sentiment_magnitude": 0.0}

def process_with_nlp(text: str) -> dict:
    """
    Process text with NLP to extract named entities, key phrases, etc.
    
    Args:
        text (str): The text to process
        
    Returns:
        dict: A dictionary containing NLP processing results
    """
    results = {
        "named_entities": [],
        "pos_tags": [],
        "lemmatized_tokens": [],
        "key_phrases": [],
        "important_nouns": [],
        "sentiment_analysis": {"sentiment": "unknown", "sentiment_score": 0.0, "sentiment_magnitude": 0.0}
    }
    
    if not text or text.isspace():
        return results
    
    # First, analyze sentiment
    results["sentiment_analysis"] = analyze_sentiment(text)
    
    try:
        # Use spaCy for advanced NLP if available
        if SPACY_AVAILABLE:
            doc = spacy_nlp(text)
            
            # Extract named entities (excluding crypto entities which are handled separately)
            results["named_entities"] = [(ent.text, ent.label_) for ent in doc.ents]
            
            # Extract POS tags for content words
            results["pos_tags"] = [(token.text, token.pos_) for token in doc 
                                   if token.pos_ in ["NOUN", "VERB", "ADJ", "ADV"] and not token.is_stop]
            
            # Get lemmatized tokens (normalized words)
            results["lemmatized_tokens"] = [token.lemma_ for token in doc 
                                           if not token.is_stop and not token.is_punct and token.text.strip()]
            
            # Extract important nouns (potential topics)
            results["important_nouns"] = [token.text for token in doc 
                                         if token.pos_ == "NOUN" and not token.is_stop]
            
            # Try to extract key phrases using noun chunks
            results["key_phrases"] = [chunk.text for chunk in doc.noun_chunks 
                                     if len(chunk.text.split()) > 1]
        
        # If key phrases are empty, use RoBERTa to attempt extraction
        if not results["key_phrases"] and len(text.split()) > 3:
            try:
                # Create a masked sentence from the text
                words = text.split()
                if len(words) > 5:
                    # Get 3 random positions to mask
                    import random
                    positions = sorted(random.sample(range(len(words)), min(3, len(words))))
                    
                    # Create masked sentences
                    key_terms = []
                    for pos in positions:
                        words_copy = words.copy()
                        words_copy[pos] = tokenizer.mask_token
                        masked_text = " ".join(words_copy)
                        
                        # Get predictions for the masked token
                        predictions = nlp_pipeline(masked_text, top_k=2)
                        for prediction in predictions:
                            key_terms.append(prediction["token_str"].strip())
                    
                    results["key_phrases"].extend(key_terms)
            except Exception as e:
                print(f"Error in key phrase extraction: {e}")
                
        # Ensure all results are strings for CSV output
        results["named_entities"] = ";".join([f"{ent[0]}:{ent[1]}" for ent in results["named_entities"]])
        results["pos_tags"] = ";".join([f"{tag[0]}:{tag[1]}" for tag in results["pos_tags"]])
        results["lemmatized_tokens"] = ";".join(results["lemmatized_tokens"])
        results["key_phrases"] = ";".join(list(set(results["key_phrases"])))  # Remove duplicates
        results["important_nouns"] = ";".join(list(set(results["important_nouns"])))  # Remove duplicates
            
    except Exception as e:
        print(f"Error in NLP processing: {e}")
    
    # Clear GPU memory after processing
    if (results["named_entities"].count(";") > 100) or (len(text) > 1000):
        clear_gpu_memory()
        
    return results

def process_tweet(text: str) -> tuple:
    """
    Process a tweet to extract mentions, hashtags, URLs, crypto entities, and perform NLP analysis.
    Also performs sentiment analysis.
    
    Args:
        text (str): The tweet text to process
        
    Returns:
        tuple: A tuple containing mentions, hashtags, URLs, NLP results, and sentiment analysis
    """
    if not text or not isinstance(text, str):
        return [], [], [], "", "", {}, {"sentiment": "unknown", "sentiment_score": 0.0, "sentiment_magnitude": 0.0}
    
    # Clean the text while preserving mentions and hashtags
    cleaned_text = clean_text(text)
    
    # Process text with NLP
    processed_text = process_nlp_text(text)
    
    # Extract mentions, hashtags, and URLs
    mentions = extract_mentions(text)
    hashtags = extract_hashtags(text)
    urls = extract_urls(text)
    
    # Identify crypto entities
    crypto_entities = identify_crypto_entities(text)
    
    # Process with NLP models
    nlp_results = process_with_nlp(text)
    
    # Ensure we have the sentiment analysis results
    sentiment_results = nlp_results.pop("sentiment_analysis", {"sentiment": "unknown", "sentiment_score": 0.0, "sentiment_magnitude": 0.0})
    
    # Add crypto entities to the named entities
    formatted_crypto_entities = [f"{entity}:{main_cat}.{sub_cat}" for entity, main_cat, sub_cat in crypto_entities]
    
    # If named_entities is a string (joined with semicolons), we need to handle differently
    if isinstance(nlp_results.get("named_entities", ""), str):
        nlp_results["named_entities"] = nlp_results.get("named_entities", "")
        if nlp_results["named_entities"] and formatted_crypto_entities:
            nlp_results["named_entities"] += ";" + ";".join(formatted_crypto_entities)
        elif formatted_crypto_entities:
            nlp_results["named_entities"] = ";".join(formatted_crypto_entities)
    
    return mentions, hashtags, urls, cleaned_text, processed_text, nlp_results, sentiment_results

def process_batch(df_batch):
    """Process a batch of tweets"""
    processed_data = []
    
    for idx, row in df_batch.iterrows():
        text = row.get('text', '')
        
        # Process the tweet
        mentions, hashtags, urls, cleaned_text, processed_text, nlp_results, sentiment_results = process_tweet(text)
        
        # Create a dictionary with the results
        result = {
            'id': row.get('id', ''),
            'original_text': text,  # Store the original text
            'cleaned_text': cleaned_text,
            'nlp_processed_text': processed_text,
            'extracted_mentions': ';'.join(mentions),
            'extracted_hashtags': ';'.join(hashtags),
            'extracted_urls': ';'.join(urls),
            'named_entities': nlp_results.get('named_entities', ''),
            'pos_tags': nlp_results.get('pos_tags', ''),
            'lemmatized_tokens': nlp_results.get('lemmatized_tokens', ''),
            'key_phrases': nlp_results.get('key_phrases', ''),
            'important_nouns': nlp_results.get('important_nouns', ''),
            'sentiment': sentiment_results.get('sentiment', 'unknown'),
            'sentiment_score': sentiment_results.get('sentiment_score', 0.0),
            'sentiment_magnitude': sentiment_results.get('sentiment_magnitude', 0.0)
        }
        
        processed_data.append(result)
    
    return pd.DataFrame(processed_data)

# ==============================================
# Main Processing Function
# ==============================================

def main(reset_checkpoint=False, input_file=None):
    """
    Main function to process tweets
    
    Args:
        reset_checkpoint (bool): Whether to reset the checkpoint and process all data
        input_file (str): Optional specific input file to process, otherwise processes all CSV files
    """
    if reset_checkpoint and os.path.exists(CHECKPOINT_FILE):
        os.remove(CHECKPOINT_FILE)
        print("Checkpoint reset. Will process all data from the beginning.")
    
    # Get list of CSV files to process
    if input_file:
        # Process a specific file
        input_files = [input_file]
    else:
        # Find all CSV files in the OUTPUT_FOLDER
        import glob
        input_files = glob.glob(f"{OUTPUT_FOLDER}/*.csv")
        
        # Exclude our output files
        input_files = [f for f in input_files if not any(x in f for x in ["_processed.csv", "_analysis.csv"])]
    
    if not input_files:
        print(f"No input CSV files found in {OUTPUT_FOLDER}")
        return
    
    print(f"Found {len(input_files)} files to process: {[os.path.basename(f) for f in input_files]}")
    
    # Process each file
    for input_csv in input_files:
        print(f"\nProcessing file: {os.path.basename(input_csv)}")
        
        print("Loading dataset...")
        # Check if input file exists
        if not os.path.exists(input_csv):
            print(f"Input file {input_csv} not found. Skipping.")
            continue
        
        # Load the dataset
        try:
            df = pd.read_csv(input_csv)
            print(f"Loaded dataset with {len(df)} records and {len(df.columns)} columns.")
        except Exception as e:
            print(f"Error loading {input_csv}: {e}")
            continue
        
        # Load checkpoint if it exists
        checkpoint = load_checkpoint()
        start_idx = checkpoint['last_processed_index']
        
        # For simplicity, reset checkpoints between files
        start_idx = 0
        save_checkpoint(0)
        
        print("\nProcessing tweets...")
        print(f"Starting from index {start_idx}")
        
        # Filter to only unprocessed rows
        df_to_process = df.iloc[start_idx:]
        
        if len(df_to_process) == 0:
            print("No new data to process in this file.")
            continue
        
        # Process in batches for memory efficiency
        batch_size = BATCH_SIZE
        num_batches = math.ceil(len(df_to_process) / batch_size)
        print(f"Processing in {num_batches} batches of {batch_size} records each")
        
        processed_batches = []
        
        # Create progress bar
        for i in tqdm(range(num_batches)):
            batch_start = i * batch_size
            batch_end = min((i + 1) * batch_size, len(df_to_process))
            
            # Get current batch
            df_batch = df_to_process.iloc[batch_start:batch_end]
            
            # Process the batch
            processed_batch = process_batch(df_batch)
            processed_batches.append(processed_batch)
            
            # Save checkpoint
            save_checkpoint(start_idx + batch_end)
            
            # Save intermediate results every 5 batches to prevent data loss in case of session timeout
            if i % 5 == 0 and i > 0:
                file_basename = os.path.splitext(os.path.basename(input_csv))[0]
                interim_df = pd.concat(processed_batches, ignore_index=True)
                interim_file = f"{OUTPUT_FOLDER}/{file_basename}_interim_{i}.csv"
                interim_df.to_csv(interim_file, index=False)
                print(f"\nSaved interim results to {interim_file}")
                
                # Clear memory
                clear_gpu_memory()
        
        # Combine all batches
        if processed_batches:
            file_basename = os.path.splitext(os.path.basename(input_csv))[0]
            
            final_df = pd.concat(processed_batches, ignore_index=True)
            
            # Calculate statistics columns
            final_df["mention_count"] = final_df["extracted_mentions"].str.count(";") + (final_df["extracted_mentions"] != "").astype(int)
            final_df["hashtag_count"] = final_df["extracted_hashtags"].str.count(";") + (final_df["extracted_hashtags"] != "").astype(int)
            final_df["entity_count"] = final_df["named_entities"].str.count(";") + (final_df["named_entities"] != "").astype(int)
            
            # Save the full processed dataset
            output_file = f"{OUTPUT_FOLDER}/{file_basename}_processed.csv"
            final_df.to_csv(output_file, index=False)
            print(f"Processed data saved to {output_file}")
            
            # Create a lighter version with just the analysis
            analysis_columns = [
                "id", "original_text", "cleaned_text", "nlp_processed_text", 
                "extracted_mentions", "extracted_hashtags", "extracted_urls",
                "named_entities", "key_phrases", "important_nouns",
                "sentiment", "sentiment_score", "sentiment_magnitude",
                "mention_count", "hashtag_count", "entity_count"
            ]
            
            # Ensure all columns exist before subsetting
            available_columns = [col for col in analysis_columns if col in final_df.columns]
            analysis_df = final_df[available_columns]
            analysis_file = f"{OUTPUT_FOLDER}/{file_basename}_analysis.csv"
            analysis_df.to_csv(analysis_file, index=False)
            print(f"Analysis results saved to {analysis_file}")
            
            # Print statistics
            print(f"\nAnalysis completed successfully!")
            print(f"Total records: {len(final_df)}")
            print(f"Tweets with Mentions: {(final_df['extracted_mentions'] != '').sum()}")
            print(f"Tweets with Hashtags: {(final_df['extracted_hashtags'] != '').sum()}")
            print(f"Tweets with Named Entities: {(final_df['named_entities'] != '').sum()}")
            
            # Print sentiment statistics
            sentiment_counts = final_df['sentiment'].value_counts()
            print("\nSentiment Distribution:")
            for sentiment, count in sentiment_counts.items():
                percentage = (count / len(final_df)) * 100
                print(f"  {sentiment}: {count} tweets ({percentage:.1f}%)")
            
            # Get average sentiment scores
            avg_score = final_df['sentiment_score'].mean()
            avg_magnitude = final_df['sentiment_magnitude'].mean()
            print(f"\nAverage sentiment score: {avg_score:.3f}")
            print(f"Average sentiment magnitude: {avg_magnitude:.3f}")
            
            # Get top entities by sentiment
            positive_entities = []
            for idx, row in final_df[final_df['sentiment'] == 'positive'].iterrows():
                entities = row['named_entities'].split(';') if isinstance(row['named_entities'], str) and row['named_entities'] else []
                for entity in entities:
                    if entity and ':' in entity:
                        entity_name = entity.split(':')[0]
                        positive_entities.append(entity_name)
            
            # Get the most common positive entities
            from collections import Counter
            top_positive = Counter(positive_entities).most_common(5)
            if top_positive:
                print("\nTop entities with positive sentiment:")
                for entity, count in top_positive:
                    print(f"  {entity}: {count} mentions")
            
            # Print sample results
            print("\nSample of processing results:")
            for i, row in analysis_df.head(3).iterrows():
                print(f"\nOriginal Text: {row['original_text']}")
                print(f"Cleaned Text: {row['cleaned_text']}")
                print(f"NLP Processed Text: {row['nlp_processed_text']}")
                print(f"Mentions: {row['extracted_mentions']}")
                print(f"Hashtags: {row['extracted_hashtags']}")
                print(f"Named Entities: {row['named_entities']}")
                print(f"Key Phrases: {row['key_phrases']}")
                print(f"Sentiment: {row['sentiment']} (Score: {row['sentiment_score']:.3f}, Magnitude: {row['sentiment_magnitude']:.3f})")
                print("-" * 80)
            
            # Delete interim files
            import glob
            interim_files = glob.glob(f"{OUTPUT_FOLDER}/{file_basename}_interim_*.csv")
            for f in interim_files:
                try:
                    os.remove(f)
                    print(f"Deleted interim file: {os.path.basename(f)}")
                except:
                    pass
                    
            # Clear memory after processing each file
            clear_gpu_memory()
        else:
            print("No data processed for this file.")
    
    # Clean up checkpoint file after successful processing
    if os.path.exists(CHECKPOINT_FILE):
        os.remove(CHECKPOINT_FILE)
    print("\nAll files processed successfully!")

# ==============================================
# Colab Usage Example
# ==============================================

"""
# EXAMPLE USAGE IN COLAB:

# 1. Install packages and mount drive
from google.colab import drive
drive.mount('/content/drive')

# 2. Process one specific file
input_file = "/content/drive/MyDrive/projects_twitter_post/zilliqa.csv"
main(reset_checkpoint=True, input_file=input_file)

# 3. Process all files
main(reset_checkpoint=True)
"""

if __name__ == "__main__":
    import sys
    
    # Check if --reset flag is provided
    reset_checkpoint = "--reset" in sys.argv
    
    # Check if --file flag is provided
    input_file = None
    if "--file" in sys.argv:
        try:
            input_file = sys.argv[sys.argv.index("--file") + 1]
        except IndexError:
            print("Error: --file flag requires a filename argument")
            sys.exit(1)
    
    # Run the main function
    main(reset_checkpoint=reset_checkpoint, input_file=input_file) )
   
   demo.launch()