app.py

from flask import Flask, render_template, request, jsonify, send_file
from huggingface_hub import HfApi
import requests
import base64
import markdown
import json
import mimetypes
import os
import io
from pathlib import Path

app = Flask(__name__)

GITHUB_API = "https://api.github.com/repos/"

def generate_file_tree(paths):
    """Generate a simple file tree from a list of paths."""
    print("generating file tree") 
    tree = ["📁 Root"]
    sorted_paths = sorted(paths)
    for path in sorted_paths:
        parts = path.split('/')
        indent = "  " * (len(parts) - 1)
        tree.append(f"{indent}📄 {parts[-1]}")
    print("generating file tree - Complete") 
    
    return "\n".join(tree) + "\n\n"

def get_all_files(owner, repo, path="", is_hf=False):
    """Recursively fetch all files from a repository."""
    if is_hf:
        api_url = f"https://huggingface.co/api/spaces/{owner}/{repo}/tree/main/{path}".rstrip('/')
    else:
        api_url = f"{GITHUB_API}{owner}/{repo}/contents/{path}".rstrip('/')
    
    try:
        response = requests.get(api_url, headers={"Accept": "application/json"}, timeout=10)
        response.raise_for_status()
        
        # Check if the response is JSON
        if response.headers.get('Content-Type', '').startswith('application/json'):
            items = response.json()
        else:
            print(f"Received non-JSON response from {api_url}: {response.text[:100]}...")
            return None
        
        files = []
        for item in items:
            if isinstance(item, dict) and item.get('type') == 'file':
                files.append(item)
            elif isinstance(item, dict) and item.get('type') == 'dir':
                sub_files = get_all_files(owner, repo, item['path'], is_hf)
                if sub_files:
                    files.extend(sub_files)
        return files
        
    except requests.exceptions.RequestException as e:
        print(f"Error fetching repository contents from {api_url}: {str(e)}")
        return None

def get_hf_files(repo, name):
    """Fetch all files from a Hugging Face Space with robust error handling."""
    api = HfApi(token=os.getenv('HF_TOKEN'))
    try:
        # Use HfApi to list files, which is more reliable for Spaces
        file_list = api.list_repo_files(repo_id=f'{repo}/{name}', repo_type="space")
        print(f"Files in {repo}/{name}: {file_list}")
        processed_files = []
        
        if not os.path.exists(name):
            os.makedirs(name)

        for file_path in file_list:
            # Fetch raw file content with strict validation
            raw_url = f"https://huggingface.co/spaces/{repo}/{name}/raw/main/{file_path}"
            try:
                response = requests.get(raw_url, timeout=10)
                response.raise_for_status()
                
                # Ensure we get raw content, not HTML or JSON
                content_type = response.headers.get('Content-Type', '').lower()
                if content_type.startswith('text/html'):
                    print(f"Warning: Received HTML instead of raw content for {file_path}: {response.text[:100]}...")
                    continue
                if content_type.startswith('application/json'):
                    print(f"Warning: Received JSON instead of raw content for {file_path}: {response.text[:100]}...")
                    continue
                
                # Verify it's a valid file (e.g., text/plain or binary)
                if not content_type.startswith(('text/plain', 'application/octet-stream', 'text/')) and 'text/' not in content_type:
                    print(f"Unexpected content type for {file_path}: {content_type}")
                    continue
                
            except requests.exceptions.RequestException as e:
                print(f"Error downloading {file_path} from {raw_url}: {str(e)}")
                continue
            
            # Process file
            filename = os.path.basename(file_path)
            if "." in filename:
                pf, sf = filename.rsplit(".", 1)
                f_name = f"{pf}.{sf}"
            else:
                pf = filename
                sf = ""
                f_name = pf
            
            local_path = os.path.join(name, file_path)
            os.makedirs(os.path.dirname(local_path), exist_ok=True)
            
            with open(local_path, 'wb') as file:
                file.write(response.content)
            
            processed_files.append({"path": file_path})

        print(f"Processed files: {processed_files}")
        return processed_files
    
    except Exception as e:
        print(f"Error processing Hugging Face files for {repo}/{name}: {str(e)}")
        return []

def get_repo_contents(url):
    """Parse URL and fetch repository contents with robust error handling."""
    try:
        if "huggingface.co" in url.lower():
            parts = url.rstrip('/').split('/')
            owner, repo = parts[-2], parts[-1]
            # Ensure the Space exists and is accessible
            try:
                api = HfApi()
                api.list_repo_files(repo_id=f'{owner}/{repo}', repo_type="space")  # Pre-check
            except Exception as e:
                raise Exception("HfApi Error")

            files = get_hf_files(owner, repo)
            if not files:  # Empty list is valid, but check for errors
                raise Exception("No files found in the Hugging Face Space")
            return owner, repo, files, True
        else:  # Assume GitHub URL
            parts = url.rstrip('/').split('/')
            owner, repo = parts[-2], parts[-1]
            files = get_all_files(owner, repo, "", False)
            if files is None:
                raise Exception("Failed to fetch GitHub repository contents")
            return owner, repo, files, False
    except Exception as e:
        print(f"Error processing URL {url}: {str(e)}")
        return None, None, f"Error fetching repo contents: {str(e)}", False

def process_file_content(file_info, owner, repo, is_hf=False):
    """Process individual file content from a repository."""
    content = ""
    file_path = file_info['path']
    
    try:
        if is_hf:
            file_url = f"https://huggingface.co/spaces/{owner}/{repo}/raw/main/{file_path}"
            response = requests.get(file_url, timeout=10)
            response.raise_for_status()
            
            # Ensure we get raw content, not HTML or JSON
            content_type = response.headers.get('Content-Type', '').lower()
            if content_type.startswith('text/html'):
                raise Exception(f"Received HTML instead of raw content for {file_path}: {response.text[:100]}...")
            if content_type.startswith('application/json'):
                raise Exception(f"Received JSON instead of raw content for {file_path}: {response.text[:100]}...")
            
            content_raw = response.content
            size = len(content_raw)
            file_extension = file_path.split('.')[-1] if '.' in file_path else ''
            mime_type, _ = mimetypes.guess_type(file_path)
            is_text = (mime_type and mime_type.startswith('text')) or file_extension in ['py', 'md', 'txt', 'js', 'html', 'css', 'json'] or "Dockerfile" in file_path
            
            if is_text:
                try:
                    text_content = content_raw.decode('utf-8')
                    if file_extension == 'json':
                        try:
                            json_data = json.loads(text_content)
                            formatted_json = json.dumps(json_data, indent=2)
                            content = f"### File: {file_path}\n```json\n{formatted_json}\n```\n\n"
                        except json.JSONDecodeError:
                            content = f"### File: {file_path}\n```json\n{text_content}\n```\n[Note: Invalid JSON format]\n\n"
                    else:
                        content = f"### File: {file_path}\n```{file_extension or 'text'}\n{text_content}\n```\n\n"
                except UnicodeDecodeError:
                    content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
            else:
                content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
        else:  # GitHub
            file_url = f"{GITHUB_API}{owner}/{repo}/contents/{file_path}"
            response = requests.get(file_url, headers={"Accept": "application/json"}, timeout=10)
            response.raise_for_status()
            
            # Ensure we get JSON, not HTML
            if response.headers.get('Content-Type', '').startswith('text/html'):
                raise Exception(f"Received HTML instead of JSON for {file_path}: {response.text[:100]}...")
            
            data = response.json()
            if 'content' in data:
                content_raw = base64.b64decode(data['content'])
                size = data['size']
                file_extension = file_path.split('.')[-1] if '.' in file_path else ''
                mime_type, _ = mimetypes.guess_type(file_path)
                is_text = (mime_type and mime_type.startswith('text')) or file_extension in ['py', 'md', 'txt', 'js', 'html', 'css', 'json']
                
                if is_text:
                    try:
                        text_content = content_raw.decode('utf-8')
                        if file_extension == 'json':
                            try:
                                json_data = json.loads(text_content)
                                formatted_json = json.dumps(json_data, indent=2)
                                content = f"### File: {file_path}\n```json\n{formatted_json}\n```\n\n"
                            except json.JSONDecodeError:
                                content = f"### File: {file_path}\n```json\n{text_content}\n```\n[Note: Invalid JSON format]\n\n"
                        else:
                            content = f"### File: {file_path}\n```{file_extension or 'text'}\n{text_content}\n```\n\n"
                    except UnicodeDecodeError:
                        content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
                else:
                    content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
            else:
                content = f"### File: {file_path}\n[No content available]\n\n"
    except Exception as e:
        content = f"### File: {file_path}\n[Error fetching file content: {str(e)}]\n\n"
    
    return content

def process_uploaded_file(file):
    """Process uploaded file content."""
    content = ""
    filename = file.filename
    file_extension = filename.split('.')[-1] if '.' in filename else ''
    
    try:
        content_raw = file.read()  # Read file content into memory
        size = len(content_raw)    # Compute size in bytes
        
        mime_type, _ = mimetypes.guess_type(filename)
        is_text = (mime_type and mime_type.startswith('text')) or file_extension in ['py', 'md', 'txt', 'js', 'html', 'css', 'json'] or "Dockerfile" in file_path
        
        if is_text:
            try:
                text_content = content_raw.decode('utf-8')
                if file_extension == 'json':
                    try:
                        json_data = json.loads(text_content)
                        formatted_json = json.dumps(json_data, indent=2)
                        content = f"### File: {filename}\n```json\n{formatted_json}\n```\n\n"
                    except json.JSONDecodeError:
                        content = f"### File: {filename}\n```json\n{text_content}\n```\n[Note: Invalid JSON format]\n\n"
                else:
                    content = f"### File: {filename}\n```{file_extension or 'text'}\n{text_content}\n```\n\n"
            except UnicodeDecodeError:
                content = f"### File: {filename}\n[Binary file - {size} bytes]\n\n"
        else:
            content = f"### File: {filename}\n[Binary file - {size} bytes]\n\n"
    except Exception as e:
        content = f"### File: {filename}\n[Error processing file: {str(e)}]\n\n"
    
    return content

def create_markdown_document(url=None, files=None):
    """Create markdown document from repo contents or uploaded files."""
    if url:
        owner, repo, contents, is_hf = get_repo_contents(url)
        
        if isinstance(contents, str):  # Error case
            return f"Error: {contents}"
        
        markdown_content = f"# {'Space' if is_hf else 'Repository'}: {owner}/{repo}\n\n"
        markdown_content += "## File Structure\n```\n"
        markdown_content += generate_file_tree([item['path'] for item in contents])
        markdown_content += "```\n\n"
        markdown_content += f"Below are the contents of all files in the {'space' if is_hf else 'repository'}:\n\n"
        
        for item in contents:
            markdown_content += process_file_content(item, owner, repo, is_hf)
    else:  # Handle uploaded files
        markdown_content = "# Uploaded Files\n\n"
        markdown_content += "## File Structure\n```\n"
        markdown_content += generate_file_tree([file.filename for file in files])
        markdown_content += "```\n\n"
        markdown_content += "Below are the contents of all uploaded files:\n\n"
        for file in files:
            markdown_content += process_uploaded_file(file)
    
    return markdown_content

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/process', methods=['POST'])
def process():
    # Ensure consistent response structure as JSON, even for errors
    response_data = {'markdown': '', 'html': '', 'filename': '', 'error': None}
    
    try:
        if 'files[]' in request.files:
            files = request.files.getlist('files[]')
            if not files:
                response_data['error'] = 'No files uploaded'
                return jsonify(response_data), 400
            
            markdown_content = create_markdown_document(files=files)
            response_data['markdown'] = "```markdown\n" + markdown_content + "\n```"
            response_data['html'] = markdown.markdown(markdown_content)
            response_data['filename'] = "uploaded_files_summary.md"
        else:
            repo_url = request.json.get('repo_url', '').strip()
            if not repo_url:
                response_data['error'] = 'Please provide a repository URL or upload files'
                return jsonify(response_data), 400
            
            markdown_content = create_markdown_document(repo_url)
            owner, repo, contents, is_hf = get_repo_contents(repo_url)
            if not owner:
                response_data['error'] = markdown_content
                return jsonify(response_data), 400
            
            response_data['markdown'] = markdown_content
            response_data['html'] = markdown.markdown(markdown_content)
            response_data['filename'] = f"{owner}_{repo}_summary.md"
    
    except Exception as e:
        response_data['error'] = f"Server error processing request: {str(e)}"
        return jsonify(response_data), 500
    
    return jsonify(response_data)

@app.route('/download', methods=['POST'])
def download():
    markdown_content = request.json.get('markdown', '')
    filename = request.json.get('filename', 'document.md')
    
    buffer = io.BytesIO()
    buffer.write(markdown_content.encode('utf-8'))
    buffer.seek(0)
    
    return send_file(
        buffer,
        as_attachment=True,
        download_name=filename,
        mimetype='text/markdown'
    )

with open("html_template.html", "r") as f:
    html_template = f.read()
f.close()

if not os.path.exists('templates'):
    os.makedirs('templates')
with open('templates/index.html', 'w') as f:
    f.write(html_template)

if __name__ == '__main__':
    app.run(host="0.0.0.0", port=7860, debug=True)