app.py

# app.py
from flask import Flask, request, render_template, jsonify, send_file
from parser import parse_python_code
import os
import json
import io
import subprocess  # To call process_hf_dataset.py
from database import init_chromadb, store_program, query_programs, load_chromadb_from_hf, DB_NAME
import logging

# Set up logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

# User-configurable variables
UPLOAD_DIR = "./uploads"  # Directory for uploads

app = Flask(__name__)

def reconstruct_code(parts):
    """Reconstruct the original code from parsed parts."""
    sorted_parts = sorted(parts, key=lambda p: p['location'][0])
    return ''.join(part['source'] for part in sorted_parts)

@app.route('/', methods=['GET', 'POST'])
def index():
    if request.method == 'POST':
        parts = None
        filename = 'unnamed.py'
        code_input = None
        query_results = None

        # Handle file upload or pasted code (parsing)
        if 'file' in request.files and request.files['file'].filename:
            file = request.files['file']
            if not file.filename.endswith('.py'):
                return 'Invalid file type. Please upload a Python file.', 400
            filename = file.filename
            file_path = os.path.join(UPLOAD_DIR, filename)
            file.save(file_path)
            with open(file_path, 'r') as f:
                code_input = f.read()
            parts, sequence = parse_python_code(code_input)
            # Store in ChromaDB
            client = init_chromadb()
            vectors = [part['vector'] for part in parts]
            store_program(client, code_input, sequence, vectors, DB_NAME)
            logger.info(f"Stored code: {filename}")
        elif 'code' in request.form and request.form['code'].strip():
            code_input = request.form['code']
            filename = request.form.get('filename', 'unnamed.py') or 'unnamed.py'
            if not filename.endswith('.py'):
                filename += '.py'
            parts, sequence = parse_python_code(code_input)
            vectors = [part['vector'] for part in parts]
            client = init_chromadb()
            store_program(client, code_input, sequence, vectors, DB_NAME)
            logger.info(f"Stored code: {filename}")
        elif 'query_ops' in request.form and request.form['query_ops'].strip():
            # Handle query for operations (category sequence)
            operations = [op.strip() for op in request.form['query_ops'].split(',')]
            client = load_chromadb_from_hf()
            query_results = query_programs(client, operations, DB_NAME)
            logger.info(f"Queried operations: {operations}")
            return render_template(
                'results_partial.html',
                parts=None,
                filename=filename,
                reconstructed_code=None,
                code_input=None,
                query_results=query_results
            )
        elif 'semantic_query' in request.form and request.form['semantic_query'].strip():
            # Handle semantic query (natural language description)
            semantic_query = request.form['semantic_query']
            client = load_chromadb_from_hf()
            query_results = query_programs(client, None, DB_NAME, semantic_query=semantic_query)
            logger.info(f"Queried semantically: {semantic_query}")
            return render_template(
                'results_partial.html',
                parts=None,
                filename=filename,
                reconstructed_code=None,
                code_input=None,
                query_results=query_results
            )
        elif 'process_hf' in request.form:
            # Trigger processing of Hugging Face dataset with fresh database
            try:
                # Reset ChromaDB collection
                client = init_chromadb()
                try:
                    client.delete_collection(DB_NAME)
                    logger.info(f"Deleted ChromaDB collection: {DB_NAME}")
                except Exception as e:
                    logger.warning(f"Failed to delete collection {DB_NAME}: {e}")
                collection = client.create_collection(DB_NAME)
                logger.info(f"Created fresh ChromaDB collection: {DB_NAME}")
                
                # Process dataset
                result = subprocess.run(['python', 'process_hf_dataset.py'], check=True, capture_output=True, text=True, cwd=os.path.dirname(__file__))
                logger.info(f"Process Hugging Face dataset output: {result.stdout}")
                if result.stderr:
                    logger.error(f"Process Hugging Face dataset errors: {result.stderr}")
                return render_template(
                    'results_partial.html',
                    parts=None,
                    filename="Hugging Face Dataset Processed",
                    reconstructed_code=None,
                    code_input=None,
                    query_results=None,
                    message="Hugging Face dataset processed and stored successfully with fresh database."
                )
            except subprocess.CalledProcessError as e:
                logger.error(f"Error processing Hugging Face dataset: {e.stderr}")
                return f"Error processing Hugging Face dataset: {e.stderr}", 500
        elif 'load_dataset' in request.form:
            # Trigger loading of Hugging Face dataset without resetting
            try:
                # Check if collection exists, get or create if needed
                client = init_chromadb()
                collection = client.get_or_create_collection(DB_NAME)
                logger.info(f"Using existing or new ChromaDB collection: {DB_NAME}")
                
                # Process dataset
                result = subprocess.run(['python', 'process_hf_dataset.py'], check=True, capture_output=True, text=True, cwd=os.path.dirname(__file__))
                logger.info(f"Load Hugging Face dataset output: {result.stdout}")
                if result.stderr:
                    logger.error(f"Load Hugging Face dataset errors: {result.stderr}")
                return render_template(
                    'results_partial.html',
                    parts=None,
                    filename="Hugging Face Dataset Loaded",
                    reconstructed_code=None,
                    code_input=None,
                    query_results=None,
                    message="Hugging Face dataset loaded and stored successfully."
                )
            except subprocess.CalledProcessError as e:
                logger.error(f"Error loading Hugging Face dataset: {e.stderr}")
                return f"Error loading Hugging Face dataset: {e.stderr}", 500
        elif 'reset_db' in request.form:
            # Reset ChromaDB collection (no repopulation with samples)
            try:
                client = init_chromadb()
                try:
                    client.delete_collection(DB_NAME)
                    logger.info(f"Deleted ChromaDB collection: {DB_NAME}")
                except Exception as e:
                    logger.warning(f"Failed to delete collection {DB_NAME}: {e}")
                collection = client.create_collection(DB_NAME)
                logger.info(f"Created fresh ChromaDB collection: {DB_NAME}")
                # Verify collection creation by checking if it's iterable (fix for 'NoneType' error)
                if collection is None or not hasattr(collection, 'add'):
                    raise ValueError("ChromaDB collection creation failed")
                return render_template(
                    'results_partial.html',
                    parts=None,
                    filename="Database Reset",
                    reconstructed_code=None,
                    code_input=None,
                    query_results=None,
                    message="Database reset successfully."
                )
            except Exception as e:
                logger.error(f"Error resetting database: {e}")
                return f"Error resetting database: {e}", 500

        if parts:
            indexed_parts = [{'index': i + 1, **part} for i, part in enumerate(parts)]
            reconstructed_code = reconstruct_code(indexed_parts)
            return render_template(
                'results_partial.html',
                parts=indexed_parts,
                filename=filename,
                reconstructed_code=reconstructed_code,
                code_input=code_input,
                query_results=None
            )
        return 'No file, code, or query provided', 400

    # Initial page load (start empty, no default population)
    logger.info("Application started, database empty until triggered by buttons")
    return render_template('index.html', parts=None, filename=None, reconstructed_code=None, code_input=None, query_results=None)

@app.route('/export_json', methods=['POST'])
def export_json():
    parts = request.json.get('parts', [])
    export_data = [{'vector': part['vector'], 'source': part['source'], 'description': generate_description_tokens([part['category']], [part['vector']])} for part in parts]
    json_str = json.dumps(export_data, indent=2)
    buffer = io.BytesIO(json_str.encode('utf-8'))
    buffer.seek(0)
    return send_file(
        buffer,
        as_attachment=True,
        download_name='code_vectors.json',
        mimetype='application/json'
    )

def generate_description_tokens(sequence, vectors):
    """Generate semantic description tokens for a program based on its sequence and vectors."""
    tokens = []
    category_descriptions = {
        'import': 'imports module',
        'function': 'defines function',
        'assigned_variable': 'assigns variable',
        'input_variable': 'input parameter',
        'returned_variable': 'returns value',
        'if': 'conditional statement',
        'return': 'returns result',
        'try': 'try block',
        'except': 'exception handler',
        'expression': 'expression statement',
        'spacer': 'empty line or comment'
    }
    
    for cat, vec in zip(sequence, vectors):
        if cat in category_descriptions:
            tokens.append(f"{category_descriptions[cat]}:{cat}")
            # Add vector-derived features (e.g., level, span) as tokens
            tokens.append(f"level:{vec[1]}")
            tokens.append(f"span:{vec[3]:.2f}")
    return " ".join(tokens)

if __name__ == '__main__':
    if not os.path.exists(UPLOAD_DIR):
        os.makedirs(UPLOAD_DIR)
    app.run(host="0.0.0.0", port=7860)  # Bind to all interfaces for Hugging Face Spaces