Delete utils.py

utils.py

@@ -1,462 +0,0 @@
-"""
-Utility functions for AI Dataset Studio
-Common helpers for text processing, validation, and data manipulation
-"""
-
-import re
-import hashlib
-import json
-import csv
-import io
-from typing import List, Dict, Any, Optional, Tuple, Union
-from urllib.parse import urlparse, urljoin
-from datetime import datetime
-import logging
-
-logger = logging.getLogger(__name__)
-
-def clean_text(text: str, aggressive: bool = False) -> str:
-    """
-    Clean text content with various strategies
-    
-    Args:
-        text: Input text to clean
-        aggressive: Whether to apply aggressive cleaning
-    
-    Returns:
-        Cleaned text
-    """
-    if not text:
-        return ""
-    
-    # Basic cleaning
-    text = text.strip()
-    
-    # Remove excessive whitespace
-    text = re.sub(r'\s+', ' ', text)
-    
-    # Remove URLs if aggressive
-    if aggressive:
-        text = re.sub(r'http\S+|www\.\S+', '', text)
-        text = re.sub(r'\S+@\S+', '', text)  # Email addresses
-    
-    # Fix common encoding issues
-    text = text.replace('’', "'")
-    text = text.replace('“', '"')
-    text = text.replace('â€', '"')
-    text = text.replace('â€"', '—')
-    
-    # Remove excessive punctuation
-    text = re.sub(r'[!?]{3,}', '!!!', text)
-    text = re.sub(r'\.{4,}', '...', text)
-    
-    # Clean up quotes and apostrophes
-    text = re.sub(r'["""]', '"', text)
-    text = re.sub(r'[''']', "'", text)
-    
-    return text.strip()
-
-def extract_urls_from_text(text: str) -> List[str]:
-    """Extract URLs from text content"""
-    url_pattern = r'https?://(?:[-\w.])+(?:[:\d]+)?(?:/(?:[\w/_.])*(?:\?(?:[\w&=%.])*)?(?:#(?:[\w.])*)?)?'
-    urls = re.findall(url_pattern, text)
-    return list(set(urls))  # Remove duplicates
-
-def validate_url(url: str) -> Tuple[bool, str]:
-    """
-    Validate URL format and basic security checks
-    
-    Returns:
-        Tuple of (is_valid, error_message)
-    """
-    try:
-        if not url or not url.strip():
-            return False, "Empty URL"
-        
-        url = url.strip()
-        
-        # Basic format check
-        parsed = urlparse(url)
-        
-        if not parsed.scheme:
-            return False, "Missing scheme (http:// or https://)"
-        
-        if parsed.scheme not in ['http', 'https']:
-            return False, f"Invalid scheme: {parsed.scheme}"
-        
-        if not parsed.netloc:
-            return False, "Invalid domain"
-        
-        # Check for suspicious patterns
-        suspicious_patterns = [
-            r'localhost',
-            r'127\.0\.0\.1',
-            r'192\.168\.',
-            r'10\.',
-            r'172\.(1[6-9]|2[0-9]|3[01])\.'
-        ]
-        
-        for pattern in suspicious_patterns:
-            if re.search(pattern, parsed.netloc, re.IGNORECASE):
-                return False, "Access to internal networks not allowed"
-        
-        return True, "Valid URL"
-        
-    except Exception as e:
-        return False, f"URL validation error: {str(e)}"
-
-def parse_urls_from_file(file_content: bytes, filename: str) -> List[str]:
-    """
-    Parse URLs from uploaded file content
-    
-    Args:
-        file_content: File content as bytes
-        filename: Original filename for format detection
-    
-    Returns:
-        List of extracted URLs
-    """
-    try:
-        # Decode content
-        try:
-            content = file_content.decode('utf-8')
-        except UnicodeDecodeError:
-            content = file_content.decode('latin-1')
-        
-        urls = []
-        
-        # Handle different file formats
-        if filename.lower().endswith('.csv'):
-            # Try to parse as CSV
-            reader = csv.DictReader(io.StringIO(content))
-            for row in reader:
-                # Look for URL column (flexible naming)
-                url_columns = ['url', 'URL', 'link', 'Link', 'href', 'address']
-                for col in url_columns:
-                    if col in row and row[col]:
-                        urls.append(row[col].strip())
-                        break
-        else:
-            # Treat as plain text (one URL per line)
-            lines = content.split('\n')
-            for line in lines:
-                line = line.strip()
-                if line and not line.startswith('#'):  # Skip comments
-                    # Extract URLs from line
-                    extracted = extract_urls_from_text(line)
-                    if extracted:
-                        urls.extend(extracted)
-                    elif validate_url(line)[0]:  # Check if line itself is a URL
-                        urls.append(line)
-        
-        # Remove duplicates while preserving order
-        seen = set()
-        unique_urls = []
-        for url in urls:
-            if url not in seen:
-                seen.add(url)
-                unique_urls.append(url)
-        
-        return unique_urls
-        
-    except Exception as e:
-        logger.error(f"Error parsing URLs from file: {e}")
-        return []
-
-def calculate_text_similarity(text1: str, text2: str) -> float:
-    """
-    Calculate similarity between two texts using simple methods
-    
-    Returns:
-        Similarity score between 0 and 1
-    """
-    if not text1 or not text2:
-        return 0.0
-    
-    # Simple character-level similarity
-    text1 = text1.lower().strip()
-    text2 = text2.lower().strip()
-    
-    if text1 == text2:
-        return 1.0
-    
-    # Jaccard similarity on words
-    words1 = set(text1.split())
-    words2 = set(text2.split())
-    
-    if not words1 and not words2:
-        return 1.0
-    if not words1 or not words2:
-        return 0.0
-    
-    intersection = len(words1.intersection(words2))
-    union = len(words1.union(words2))
-    
-    return intersection / union if union > 0 else 0.0
-
-def detect_content_type(text: str) -> str:
-    """
-    Detect the type of content based on text analysis
-    
-    Returns:
-        Content type string
-    """
-    if not text:
-        return "empty"
-    
-    text_lower = text.lower()
-    
-    # Check for common patterns
-    if any(word in text_lower for word in ['abstract:', 'introduction:', 'conclusion:', 'references:']):
-        return "academic"
-    elif any(word in text_lower for word in ['news', 'reported', 'according to', 'sources say']):
-        return "news"
-    elif any(word in text_lower for word in ['review', 'rating', 'stars', 'recommend']):
-        return "review"
-    elif any(word in text_lower for word in ['blog', 'posted by', 'share this']):
-        return "blog"
-    elif re.search(r'\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b', text):
-        return "dated_content"
-    else:
-        return "general"
-
-def extract_metadata_from_text(text: str) -> Dict[str, Any]:
-    """
-    Extract metadata from text content
-    
-    Returns:
-        Dictionary of extracted metadata
-    """
-    metadata = {}
-    
-    # Extract dates
-    date_patterns = [
-        r'\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b',
-        r'\b\d{4}[/-]\d{1,2}[/-]\d{1,2}\b',
-        r'\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{1,2},? \d{4}\b'
-    ]
-    
-    dates = []
-    for pattern in date_patterns:
-        dates.extend(re.findall(pattern, text, re.IGNORECASE))
-    
-    if dates:
-        metadata['extracted_dates'] = dates[:5]  # Limit to first 5
-    
-    # Extract numbers and statistics
-    numbers = re.findall(r'\b\d{1,3}(?:,\d{3})*(?:\.\d+)?\b', text)
-    if numbers:
-        metadata['numbers'] = numbers[:10]  # Limit to first 10
-    
-    # Extract email addresses
-    emails = re.findall(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', text)
-    if emails:
-        metadata['emails'] = emails[:5]
-    
-    # Extract phone numbers (basic pattern)
-    phones = re.findall(r'\b\d{3}[-.]?\d{3}[-.]?\d{4}\b', text)
-    if phones:
-        metadata['phones'] = phones[:5]
-    
-    # Extract capitalized words (potential names/entities)
-    capitalized = re.findall(r'\b[A-Z][a-z]+(?:\s[A-Z][a-z]+)*\b', text)
-    if capitalized:
-        # Filter common words
-        common_words = {'The', 'This', 'That', 'There', 'Then', 'They', 'These', 'Those'}
-        filtered = [word for word in capitalized if word not in common_words]
-        metadata['capitalized_terms'] = list(set(filtered))[:20]
-    
-    return metadata
-
-def generate_content_hash(text: str) -> str:
-    """Generate a hash for content deduplication"""
-    # Normalize text for hashing
-    normalized = re.sub(r'\s+', ' ', text.lower().strip())
-    return hashlib.md5(normalized.encode('utf-8')).hexdigest()
-
-def format_file_size(size_bytes: int) -> str:
-    """Format file size in human readable format"""
-    if size_bytes == 0:
-        return "0 B"
-    
-    size_names = ["B", "KB", "MB", "GB"]
-    i = 0
-    while size_bytes >= 1024 and i < len(size_names) - 1:
-        size_bytes /= 1024.0
-        i += 1
-    
-    return f"{size_bytes:.1f} {size_names[i]}"
-
-def estimate_reading_time(text: str, words_per_minute: int = 200) -> int:
-    """Estimate reading time in minutes"""
-    word_count = len(text.split())
-    return max(1, round(word_count / words_per_minute))
-
-def truncate_text(text: str, max_length: int, suffix: str = "...") -> str:
-    """Truncate text to maximum length with suffix"""
-    if len(text) <= max_length:
-        return text
-    
-    return text[:max_length - len(suffix)] + suffix
-
-def create_filename_safe_string(text: str, max_length: int = 50) -> str:
-    """Create a filesystem-safe string from text"""
-    # Remove/replace problematic characters
-    safe_text = re.sub(r'[<>:"/\\|?*]', '_', text)
-    safe_text = re.sub(r'\s+', '_', safe_text)
-    safe_text = safe_text.strip('._')
-    
-    # Truncate if too long
-    if len(safe_text) > max_length:
-        safe_text = safe_text[:max_length].rstrip('_')
-    
-    return safe_text or "untitled"
-
-def validate_dataset_format(data: List[Dict[str, Any]], required_fields: List[str]) -> Tuple[bool, List[str]]:
-    """
-    Validate dataset format against required fields
-    
-    Returns:
-        Tuple of (is_valid, list_of_errors)
-    """
-    errors = []
-    
-    if not data:
-        errors.append("Dataset is empty")
-        return False, errors
-    
-    # Check each item
-    for i, item in enumerate(data[:10]):  # Check first 10 items
-        if not isinstance(item, dict):
-            errors.append(f"Item {i} is not a dictionary")
-            continue
-        
-        # Check required fields
-        for field in required_fields:
-            if field not in item:
-                errors.append(f"Item {i} missing required field: {field}")
-            elif not item[field]:  # Check for empty values
-                errors.append(f"Item {i} has empty value for field: {field}")
-    
-    return len(errors) == 0, errors
-
-def create_progress_message(current: int, total: int, operation: str = "Processing") -> str:
-    """Create a formatted progress message"""
-    percentage = (current / total * 100) if total > 0 else 0
-    return f"{operation} {current}/{total} ({percentage:.1f}%)"
-
-def sanitize_text_for_json(text: str) -> str:
-    """Sanitize text for safe JSON serialization"""
-    if not text:
-        return ""
-    
-    # Replace problematic characters
-    text = text.replace('\x00', '')  # Remove null bytes
-    text = re.sub(r'[\x00-\x1f\x7f-\x9f]', ' ', text)  # Remove control characters
-    
-    return text
-
-def extract_domain_from_url(url: str) -> str:
-    """Extract domain from URL"""
-    try:
-        parsed = urlparse(url)
-        return parsed.netloc.lower()
-    except:
-        return "unknown"
-
-def analyze_text_quality(text: str) -> Dict[str, Any]:
-    """
-    Analyze text quality and return metrics
-    
-    Returns:
-        Dictionary with quality metrics
-    """
-    if not text:
-        return {'score': 0.0, 'issues': ['Empty text']}
-    
-    issues = []
-    score = 1.0
-    
-    # Length checks
-    word_count = len(text.split())
-    if word_count < 10:
-        issues.append('Too short (< 10 words)')
-        score -= 0.3
-    elif word_count < 50:
-        score -= 0.1
-    
-    # Character checks
-    if len(text) < 100:
-        issues.append('Very short content')
-        score -= 0.2
-    
-    # Language quality checks
-    uppercase_ratio = sum(1 for c in text if c.isupper()) / len(text)
-    if uppercase_ratio > 0.3:
-        issues.append('Excessive uppercase')
-        score -= 0.2
-    
-    # Punctuation checks
-    sentence_endings = text.count('.') + text.count('!') + text.count('?')
-    if word_count > 50 and sentence_endings < 2:
-        issues.append('Few sentence endings')
-        score -= 0.1
-    
-    # Excessive repetition check
-    words = text.lower().split()
-    if len(words) > 10:
-        unique_words = set(words)
-        if len(unique_words) / len(words) < 0.5:
-            issues.append('High word repetition')
-            score -= 0.2
-    
-    # Special character checks
-    special_char_ratio = sum(1 for c in text if not c.isalnum() and not c.isspace()) / len(text)
-    if special_char_ratio > 0.1:
-        issues.append('Many special characters')
-        score -= 0.1
-    
-    return {
-        'score': max(0.0, score),
-        'word_count': word_count,
-        'char_count': len(text),
-        'uppercase_ratio': uppercase_ratio,
-        'special_char_ratio': special_char_ratio,
-        'issues': issues
-    }
-
-# Dataset template utilities
-def create_classification_example(text: str, label: str, confidence: float = 1.0) -> Dict[str, Any]:
-    """Create a text classification example"""
-    return {
-        'text': text,
-        'label': label,
-        'confidence': confidence
-    }
-
-def create_ner_example(text: str, entities: List[Dict[str, Any]]) -> Dict[str, Any]:
-    """Create a named entity recognition example"""
-    return {
-        'text': text,
-        'entities': entities
-    }
-
-def create_qa_example(context: str, question: str, answer: str, answer_start: int = None) -> Dict[str, Any]:
-    """Create a question answering example"""
-    example = {
-        'context': context,
-        'question': question,
-        'answer': answer
-    }
-    
-    if answer_start is not None:
-        example['answer_start'] = answer_start
-    
-    return example
-
-def create_summarization_example(text: str, summary: str) -> Dict[str, Any]:
-    """Create a text summarization example"""
-    return {
-        'text': text,
-        'summary': summary
-    }