Update app.py

app.py

@@ -7,211 +7,150 @@ import json
 import mimetypes
 import os
 import io
-import uuid
 from pathlib import Path
 
 app = Flask(__name__)
 
 GITHUB_API = "https://api.github.com/repos/"
-#HF_API = "https://huggingface.co/api/spaces/"
 
 def generate_file_tree(paths):
     """Generate a simple file tree from a list of paths."""
-    print("generating file tree") 
     tree = ["📁 Root"]
     sorted_paths = sorted(paths)
     for path in sorted_paths:
         parts = path.split('/')
         indent = "  " * (len(parts) - 1)
         tree.append(f"{indent}📄 {parts[-1]}")
-    print("generating file tree - Complete") 
-    
     return "\n".join(tree) + "\n\n"
 
-def get_all_files(owner, repo, path="", is_hf=False):
-    """Recursively fetch all files from a repository."""
-    if is_hf:
-        api_url = f"https://huggingface.co/api/spaces/{owner}/{repo}/tree/main/{path}".rstrip('/')
-    else:
-        api_url = f"{GITHUB_API}{owner}/{repo}/contents/{path}".rstrip('/')
-    
+def get_all_files(owner, repo, path=""):
+    """Recursively fetch all files from a GitHub repository."""
+    api_url = f"{GITHUB_API}{owner}/{repo}/contents/{path}".rstrip('/')
     try:
-        response = requests.get(api_url, headers={"Accept": "application/json"})
+        response = requests.get(api_url, headers={"Accept": "application/json"}, timeout=10)
         response.raise_for_status()
-        
-        # Check if the response is JSON
-        if response.headers.get('Content-Type', '').startswith('application/json'):
+        content_type = response.headers.get('Content-Type', '')
+        if not content_type.startswith('application/json'):
+            print(f"Error: Received non-JSON response from GitHub for {api_url}: {response.text[:100]}...")
+            return None
+        try:
             items = response.json()
-        else:
-            # If not JSON, it might be HTML (e.g., error page)
-            print(f"Received non-JSON response: {response.text[:100]}...")
+        except json.JSONDecodeError as e:
+            print(f"Error: Failed to parse JSON from GitHub for {api_url}: {str(e)}. Response: {response.text[:100]}...")
             return None
-        
         files = []
         for item in items:
             if isinstance(item, dict) and item.get('type') == 'file':
                 files.append(item)
             elif isinstance(item, dict) and item.get('type') == 'dir':
-                sub_files = get_all_files(owner, repo, item['path'], is_hf)
+                sub_files = get_all_files(owner, repo, item['path'])
                 if sub_files:
                     files.extend(sub_files)
         return files
-        
-    except requests.exceptions.RequestException as e:
-        print(f"Error fetching repository contents: {str(e)}")
+    except requests.RequestException as e:
+        print(f"Error fetching GitHub contents from {api_url}: {str(e)}")
         return None
 
-        
-def get_hf_files(repo, name, path=""):
+def get_hf_files(owner, repo):
+    """Fetch all files from a Hugging Face Space."""
     api = HfApi()
+    repo_id = f"{owner}/{repo}"
     try:
-        file_list = api.list_repo_files(repo_id=f'{repo}/{name}', repo_type="space")
-        print(f"Files in {repo}/{name}: {file_list}")
-        processed_files = []
-        
-        if not os.path.exists(name):
-            os.makedirs(name)
+        file_list = api.list_repo_files(repo_id=repo_id, repo_type="space")
+        print(f"Files in HF Space {repo_id}: {file_list}")
+    except Exception as e:
+        print(f"Error listing files for HF Space {repo_id}: {str(e)}")
+        return []
+    
+    processed_files = []
+    local_dir = repo
+    if not os.path.exists(local_dir):
+        os.makedirs(local_dir)
 
-        for file_path in file_list:
-            # Handle nested directories
-            if "/" in file_path:
-                dir_part, file_part = file_path.split("/", 1)
-                dir_path = os.path.join(name, dir_part)
-                if not os.path.exists(dir_path):
-                    os.makedirs(dir_path)
-                if "/" in file_part:
-                    processed_files.extend(get_hf_files(repo, name, dir_part))
-                    continue
-            
-            # Fetch raw file content
-            raw_url = f"https://huggingface.co/spaces/{repo}/{name}/raw/main/{file_path}"
+    for file_path in file_list:
+        raw_url = f"https://huggingface.co/spaces/{owner}/{repo}/raw/main/{file_path}"
+        try:
             response = requests.get(raw_url, timeout=10)
             response.raise_for_status()
-            
-            # Ensure we get raw content, not HTML
-            if response.headers.get('Content-Type', '').startswith('text/html'):
-                print(f"Received HTML instead of raw content for {file_path}: {response.text[:100]}...")
+            content_type = response.headers.get('Content-Type', '')
+            if content_type.startswith('text/html'):
+                print(f"Error: Received HTML instead of raw content for {raw_url}: {response.text[:100]}...")
                 continue
-            
-            # Process file
-            filename = os.path.basename(file_path)
-            if "." in filename:
-                pf, sf = filename.rsplit(".", 1)
-                f_name = f"{pf}.{sf}"
-            else:
-                pf = filename
-                sf = ""
-                f_name = pf
-            
-            local_path = os.path.join(name, file_path)
+            local_path = os.path.join(local_dir, file_path)
             os.makedirs(os.path.dirname(local_path), exist_ok=True)
-            
             with open(local_path, 'wb') as file:
                 file.write(response.content)
-            
             processed_files.append({"path": file_path})
-
-        print(f"Processed files: {processed_files}")
-        return processed_files
+        except requests.RequestException as e:
+            print(f"Error downloading {file_path} from {raw_url}: {str(e)}")
     
-    except Exception as e:
-        print(f"Error processing Hugging Face files: {str(e)}")
-        return []
+    print(f"Processed files from HF Space {repo_id}: {processed_files}")
+    return processed_files
 
 def get_repo_contents(url):
     """Parse URL and fetch repository contents."""
     try:
-        if "huggingface.co" in url:
-            parts = url.rstrip('/').split('/')
-            owner, repo = parts[-2], parts[-1]
-            # Fallback approach: manually fetch known files or use a simpler file list
-            # For now, assume a flat structure and fetch known files directly
-            # This is a workaround until a proper API token or endpoint is confirmed
-            known_files = [
-                {'path': 'app.py', 'type': 'file'},
-                {'path': 'README.md', 'type': 'file'}
-                # Add more known paths or implement HTML scraping if needed
-            ]
-            #files = get_all_files(owner, repo, "", True) or known_files
+        parts = url.rstrip('/').split('/')
+        owner, repo = parts[-2], parts[-1]
+        if "huggingface.co" in url.lower():
             files = get_hf_files(owner, repo)
+            if not files and files is not None:  # Empty list is valid, None indicates error
+                raise Exception("No files found in the Hugging Face Space")
             return owner, repo, files, True
         else:  # Assume GitHub URL
-            parts = url.rstrip('/').split('/')
-            owner, repo = parts[-2], parts[-1]
-            files = get_all_files(owner, repo, "", False)
+            files = get_all_files(owner, repo)
             if files is None:
                 raise Exception("Failed to fetch GitHub repository contents")
             return owner, repo, files, False
     except Exception as e:
+        print(f"Error processing URL {url}: {str(e)}")
         return None, None, f"Error fetching repo contents: {str(e)}", False
 
 def process_file_content(file_info, owner, repo, is_hf=False):
     """Process individual file content from a repository."""
     content = ""
     file_path = file_info['path']
-    
     try:
         if is_hf:
             file_url = f"https://huggingface.co/spaces/{owner}/{repo}/raw/main/{file_path}"
+            response = requests.get(file_url, timeout=10)
+            response.raise_for_status()
+            content_raw = response.content
+            content_type = response.headers.get('Content-Type', '')
+            if content_type.startswith('text/html'):
+                raise Exception(f"Received HTML instead of raw content: {response.text[:100]}...")
         else:
             file_url = f"{GITHUB_API}{owner}/{repo}/contents/{file_path}"
-        
-        response = requests.get(file_url)
-        response.raise_for_status()
-        
-        if is_hf:
-            content_raw = response.content
-            size = len(content_raw)
-            file_extension = file_path.split('.')[-1] if '.' in file_path else ''
-            mime_type, _ = mimetypes.guess_type(file_path)
-            is_text = (mime_type and mime_type.startswith('text')) or file_extension in ['py', 'md', 'txt', 'js', 'html', 'css', 'json']
-            
-            if is_text:
-                try:
-                    text_content = content_raw.decode('utf-8')
-                    if file_extension == 'json':
-                        try:
-                            json_data = json.loads(text_content)
-                            formatted_json = json.dumps(json_data, indent=2)
-                            content = f"### File: {file_path}\n```json\n{formatted_json}\n```\n\n"
-                        except json.JSONDecodeError:
-                            content = f"### File: {file_path}\n```json\n{text_content}\n```\n[Note: Invalid JSON format]\n\n"
-                    else:
-                        content = f"### File: {file_path}\n```{file_extension or 'text'}\n{text_content}\n```\n\n"
-                except UnicodeDecodeError:
-                    content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
-            else:
-                content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
-        else:  # GitHub
+            response = requests.get(file_url, headers={"Accept": "application/json"}, timeout=10)
+            response.raise_for_status()
             data = response.json()
-            if 'content' in data:
-                content_raw = base64.b64decode(data['content'])
-                size = data['size']
-                file_extension = file_path.split('.')[-1] if '.' in file_path else ''
-                mime_type, _ = mimetypes.guess_type(file_path)
-                is_text = (mime_type and mime_type.startswith('text')) or file_extension in ['py', 'md', 'txt', 'js', 'html', 'css', 'json']
-                
-                if is_text:
+            if 'content' not in data:
+                raise Exception("No content available in GitHub API response")
+            content_raw = base64.b64decode(data['content'])
+
+        size = len(content_raw)
+        file_extension = file_path.split('.')[-1] if '.' in file_path else ''
+        mime_type, _ = mimetypes.guess_type(file_path)
+        is_text = (mime_type and mime_type.startswith('text')) or file_extension in ['py', 'md', 'txt', 'js', 'html', 'css', 'json']
+
+        if is_text:
+            try:
+                text_content = content_raw.decode('utf-8')
+                if file_extension == 'json':
                     try:
-                        text_content = content_raw.decode('utf-8')
-                        if file_extension == 'json':
-                            try:
-                                json_data = json.loads(text_content)
-                                formatted_json = json.dumps(json_data, indent=2)
-                                content = f"### File: {file_path}\n```json\n{formatted_json}\n```\n\n"
-                            except json.JSONDecodeError:
-                                content = f"### File: {file_path}\n```json\n{text_content}\n```\n[Note: Invalid JSON format]\n\n"
-                        else:
-                            content = f"### File: {file_path}\n```{file_extension or 'text'}\n{text_content}\n```\n\n"
-                    except UnicodeDecodeError:
-                        content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
+                        json_data = json.loads(text_content)
+                        formatted_json = json.dumps(json_data, indent=2)
+                        content = f"### File: {file_path}\n```json\n{formatted_json}\n```\n\n"
+                    except json.JSONDecodeError:
+                        content = f"### File: {file_path}\n```json\n{text_content}\n```\n[Note: Invalid JSON format]\n\n"
                 else:
-                    content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
-            else:
-                content = f"### File: {file_path}\n[No content available]\n\n"
+                    content = f"### File: {file_path}\n```{file_extension or 'text'}\n{text_content}\n```\n\n"
+            except UnicodeDecodeError:
+                content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
+        else:
+            content = f"### File: {file_path}\n[Binary file - {size} bytes]\n\n"
     except Exception as e:
-        content = f"### File: {file_path}\n[Error fetching file content: {str(e)}]\n\n"
-    
+        content = f"### File: {file_path}\n[Error fetching content: {str(e)}]\n\n"
     return content
 
 def process_uploaded_file(file):
@@ -219,11 +158,9 @@ def process_uploaded_file(file):
     content = ""
     filename = file.filename
     file_extension = filename.split('.')[-1] if '.' in filename else ''
-    
     try:
-        content_raw = file.read()  # Read file content into memory
-        size = len(content_raw)    # Compute size in bytes
-        
+        content_raw = file.read()
+        size = len(content_raw)
         mime_type, _ = mimetypes.guess_type(filename)
         is_text = (mime_type and mime_type.startswith('text')) or file_extension in ['py', 'md', 'txt', 'js', 'html', 'css', 'json']
         
@@ -245,26 +182,22 @@ def process_uploaded_file(file):
             content = f"### File: {filename}\n[Binary file - {size} bytes]\n\n"
     except Exception as e:
         content = f"### File: {filename}\n[Error processing file: {str(e)}]\n\n"
-    
     return content
 
 def create_markdown_document(url=None, files=None):
     """Create markdown document from repo contents or uploaded files."""
     if url:
         owner, repo, contents, is_hf = get_repo_contents(url)
-        
         if isinstance(contents, str):  # Error case
             return f"Error: {contents}"
-        
         markdown_content = f"# {'Space' if is_hf else 'Repository'}: {owner}/{repo}\n\n"
         markdown_content += "## File Structure\n```\n"
         markdown_content += generate_file_tree([item['path'] for item in contents])
         markdown_content += "```\n\n"
         markdown_content += f"Below are the contents of all files in the {'space' if is_hf else 'repository'}:\n\n"
-        
         for item in contents:
             markdown_content += process_file_content(item, owner, repo, is_hf)
-    else:  # Handle uploaded files
+    else:
         markdown_content = "# Uploaded Files\n\n"
         markdown_content += "## File Structure\n```\n"
         markdown_content += generate_file_tree([file.filename for file in files])
@@ -272,7 +205,6 @@ def create_markdown_document(url=None, files=None):
         markdown_content += "Below are the contents of all uploaded files:\n\n"
         for file in files:
             markdown_content += process_uploaded_file(file)
-    
     return markdown_content
 
 @app.route('/')
@@ -281,31 +213,27 @@ def index():
 
 @app.route('/process', methods=['POST'])
 def process():
-    # Ensure consistent response structure
     response_data = {'markdown': '', 'html': '', 'filename': '', 'error': None}
     
     if 'files[]' in request.files:
         files = request.files.getlist('files[]')
-        if not files:
+        if not files or all(not file.filename for file in files):
             response_data['error'] = 'No files uploaded'
             return jsonify(response_data), 400
-        
         markdown_content = create_markdown_document(files=files)
         response_data['markdown'] = "```markdown\n" + markdown_content + "\n```"
         response_data['html'] = markdown.markdown(markdown_content)
         response_data['filename'] = "uploaded_files_summary.md"
     else:
-        repo_url = request.json.get('repo_url')
+        repo_url = request.json.get('repo_url', '').strip()
         if not repo_url:
             response_data['error'] = 'Please provide a repository URL or upload files'
             return jsonify(response_data), 400
-        
         markdown_content = create_markdown_document(repo_url)
         owner, repo, contents, is_hf = get_repo_contents(repo_url)
-        if not owner:
-            response_data['error'] = markdown_content  # Error message from get_repo_contents
+        if not owner:  # Error case
+            response_data['error'] = markdown_content
             return jsonify(response_data), 400
-        
         response_data['markdown'] = markdown_content
         response_data['html'] = markdown.markdown(markdown_content)
         response_data['filename'] = f"{owner}_{repo}_summary.md"
@@ -314,28 +242,24 @@ def process():
 
 @app.route('/download', methods=['POST'])
 def download():
-    markdown_content = request.json.get('markdown')
-    filename = request.json.get('filename')
-    
+    markdown_content = request.json.get('markdown', '')
+    filename = request.json.get('filename', 'document.md')
     buffer = io.BytesIO()
     buffer.write(markdown_content.encode('utf-8'))
     buffer.seek(0)
-    
     return send_file(
         buffer,
         as_attachment=True,
         download_name=filename,
         mimetype='text/markdown'
     )
-with open("html_template.html", "r") as f:
-    html_template=f.read()
-f.close()
-
-
-if not os.path.exists('templates'):
-    os.makedirs('templates')
-with open('templates/index.html', 'w') as f:
-    f.write(html_template)
 
 if __name__ == '__main__':
+    # Ensure templates directory and index.html exist
+    if not os.path.exists('templates'):
+        os.makedirs('templates')
+    # You’ll need to provide your own index.html or use a placeholder
+    if not os.path.exists('templates/index.html'):
+        with open('templates/index.html', 'w') as f:
+            f.write('<!DOCTYPE html><html><body><h1>Repo to Markdown</h1><p>Use POST /process to convert repo contents.</p></body></html>')
     app.run(host="0.0.0.0", port=7860, debug=True)