import os import subprocess from file_utils import validate_file_types, get_file_summary, read_file_content, summarize_content def validate_url(url): return url.startswith('https://') def clone_repo(url, repo_dir, hf_token, hf_user): env = os.environ.copy() env['GIT_LFS_SKIP_SMUDGE'] = '1' token_url = url.replace('https://', f'https://{hf_user}:{hf_token}@') result = subprocess.run(["git", "clone", token_url, repo_dir], env=env, capture_output=True, text=True) if result.returncode != 0: return False, result.stderr return True, None def extract_repo_content(url, hf_token, hf_user): if not validate_url(url): return [{"header": {"name": "Error", "type": "error", "size": 0}, "content": "Invalid URL"}] repo_dir = "./temp_repo" if os.path.exists(repo_dir): subprocess.run(["rm", "-rf", repo_dir]) success, error = clone_repo(url, repo_dir, hf_token, hf_user) if not success: return [{"header": {"name": "Error", "type": "error", "size": 0}, "content": f"Failed to clone repository: {error}"}] file_types = validate_file_types(repo_dir) extracted_content = [] for file_path, file_type in file_types.items(): file_summary = get_file_summary(file_path, file_type) content = {"header": file_summary} if file_type in SUPPORTED_FILE_TYPES and file_summary["size"] <= 32 * 1024: try: file_content = read_file_content(file_path) content["content"] = file_content content["summary"] = summarize_content(file_content) except Exception as e: content["content"] = f"Failed to read file content: {str(e)}" content["summary"] = "" else: content["content"] = "File too large or binary, content not captured." content["summary"] = "" extracted_content.append(content) subprocess.run(["rm", "-rf", repo_dir]) return extracted_content