import streamlit as st import requests import base64 import os import asyncio from huggingface_hub import HfApi, snapshot_download import plotly.express as px import zipfile import tempfile import shutil # Initialize the Hugging Face API api = HfApi() # Directories for saving files HTML_DIR = "generated_html_pages" ZIP_DIR = "generated_zips" SNAPSHOT_DIR = "snapshot_downloads" for directory in [HTML_DIR, ZIP_DIR, SNAPSHOT_DIR]: if not os.path.exists(directory): os.makedirs(directory) # Default list of Hugging Face usernames default_users = { "users": [ "awacke1", "rogerxavier", "jonatasgrosman", "kenshinn", "Csplk", "DavidVivancos", "cdminix", "Jaward", "TuringsSolutions", "Severian", "Wauplin", "phosseini", "Malikeh1375", "gokaygokay", "MoritzLaurer", "mrm8488", "TheBloke", "lhoestq", "xw-eric", "Paul", "Muennighoff", "ccdv", "haonan-li", "chansung", "lukaemon", "hails", "pharmapsychotic", "KingNish", "merve", "ameerazam08", "ashleykleynhans" ] } async def fetch_user_content(username): try: models = list(await asyncio.to_thread(api.list_models, author=username)) datasets = list(await asyncio.to_thread(api.list_datasets, author=username)) return { "username": username, "models": models, "datasets": datasets } except Exception as e: return {"username": username, "error": str(e)} def download_user_page(username): url = f"https://huggingface.co/{username}" try: response = requests.get(url) response.raise_for_status() html_content = response.text html_file_path = os.path.join(HTML_DIR, f"{username}.html") with open(html_file_path, "w", encoding='utf-8') as html_file: html_file.write(html_content) return html_file_path, html_content, None except Exception as e: return None, None, str(e) @st.cache_resource def create_zip_of_files(files, zip_name): zip_file_path = os.path.join(ZIP_DIR, zip_name) with zipfile.ZipFile(zip_file_path, 'w') as zipf: for file in files: zipf.write(file, arcname=os.path.basename(file)) return zip_file_path @st.cache_resource def get_download_link(file_path, link_text): with open(file_path, 'rb') as f: data = f.read() b64 = base64.b64encode(data).decode() return f'{link_text}' async def fetch_all_users(usernames): tasks = [fetch_user_content(username) for username in usernames] return await asyncio.gather(*tasks) def perform_snapshot_download(repo_id, repo_type): try: temp_dir = tempfile.mkdtemp() snapshot_download(repo_id=repo_id, repo_type=repo_type, local_dir=temp_dir) zip_name = f"{repo_id.replace('/', '_')}_{repo_type}.zip" zip_path = os.path.join(SNAPSHOT_DIR, zip_name) shutil.make_archive(zip_path[:-4], 'zip', temp_dir) shutil.rmtree(temp_dir) return zip_path except Exception as e: return str(e) st.title("Hugging Face User Page Downloader & Zipper 📄➕📦") user_input = st.text_area( "Enter Hugging Face usernames (one per line):", value="\n".join(default_users["users"]), height=300 ) if st.button("Show User Content and Download Snapshots"): if user_input: username_list = [username.strip() for username in user_input.split('\n') if username.strip()] user_data_list = asyncio.run(fetch_all_users(username_list)) stats = {"username": [], "models_count": [], "datasets_count": []} successful_html_files = [] snapshot_downloads = [] st.markdown("### User Content Overview") for user_data in user_data_list: username = user_data["username"] with st.container(): st.markdown(f"**{username}** [🔗 Profile](https://huggingface.co/{username})") if "error" in user_data: st.warning(f"{username}: {user_data['error']} - Something went wrong! ⚠️") else: models = user_data["models"] datasets = user_data["datasets"] html_file_path, html_content, download_error = download_user_page(username) if html_file_path and html_content: successful_html_files.append(html_file_path) st.success(f"✅ Successfully downloaded {username}'s page.") # Add expander to view HTML content with st.expander(f"View {username}'s HTML page"): st.markdown(html_content, unsafe_allow_html=True) else: st.error(f"❌ Failed to download {username}'s page: {download_error}") stats["username"].append(username) stats["models_count"].append(len(models)) stats["datasets_count"].append(len(datasets)) with st.expander(f"🧠 Models ({len(models)})", expanded=False): if models: for model in models: model_name = model.modelId.split("/")[-1] st.markdown(f"- [{model_name}](https://huggingface.co/{model.modelId})") if st.button(f"Download Snapshot: {model_name}", key=f"model_{model.modelId}"): with st.spinner(f"Downloading snapshot for {model_name}..."): result = perform_snapshot_download(model.modelId, "model") if isinstance(result, str): st.error(f"Failed to download {model_name}: {result}") else: snapshot_downloads.append(result) st.success(f"Successfully downloaded snapshot for {model_name}") else: st.markdown("No models found. 🤷‍♂️") with st.expander(f"📚 Datasets ({len(datasets)})", expanded=False): if datasets: for dataset in datasets: dataset_name = dataset.id.split("/")[-1] st.markdown(f"- [{dataset_name}](https://huggingface.co/datasets/{dataset.id})") if st.button(f"Download Snapshot: {dataset_name}", key=f"dataset_{dataset.id}"): with st.spinner(f"Downloading snapshot for {dataset_name}..."): result = perform_snapshot_download(dataset.id, "dataset") if isinstance(result, str): st.error(f"Failed to download {dataset_name}: {result}") else: snapshot_downloads.append(result) st.success(f"Successfully downloaded snapshot for {dataset_name}") else: st.markdown("No datasets found. 🤷‍♀️") st.markdown("---") if successful_html_files: html_zip_path = create_zip_of_files(successful_html_files, "HuggingFace_User_Pages.zip") html_download_link = get_download_link(html_zip_path, "📥 Download All HTML Pages as ZIP") st.markdown(html_download_link, unsafe_allow_html=True) else: st.warning("No HTML files were successfully downloaded to create a ZIP archive.") if snapshot_downloads: snapshot_zip_path = create_zip_of_files(snapshot_downloads, "HuggingFace_Snapshots.zip") snapshot_download_link = get_download_link(snapshot_zip_path, "📥 Download All Snapshots as ZIP") st.markdown(snapshot_download_link, unsafe_allow_html=True) if stats["username"]: st.markdown("### User Content Statistics") fig_models = px.bar( x=stats["username"], y=stats["models_count"], labels={'x': 'Username', 'y': 'Number of Models'}, title="Number of Models per User" ) st.plotly_chart(fig_models) fig_datasets = px.bar( x=stats["username"], y=stats["datasets_count"], labels={'x': 'Username', 'y': 'Number of Datasets'}, title="Number of Datasets per User" ) st.plotly_chart(fig_datasets) else: st.warning("Please enter at least one username. Don't be shy! 😅") st.sidebar.markdown(""" ## How to use: 1. The text area is pre-filled with a list of Hugging Face usernames. You can edit this list or add more usernames. 2. Click **'Show User Content and Download Snapshots'**. 3. View each user's models and datasets along with a link to their Hugging Face profile. 4. For each model or dataset, you can click the "Download Snapshot" button to download a snapshot. 5. **Download ZIP archives** containing all the HTML pages and snapshots by clicking the download links. 6. Check out the statistics visualizations below! 7. **New feature:** You can now view the HTML content of each user's page by clicking on the expander. """)