inittial commit

app.py

@@ -0,0 +1,102 @@
+import streamlit as st
+import pandas as pd
+
+# Set page config
+st.set_page_config(layout="wide", page_title="Macrocosmos HF Dataset Explorer")
+
+# Custom CSS
+st.markdown("""
+<style>
+    .stApp {
+        max-width: 1200px;
+        margin: 0 auto;
+    }
+    .st-emotion-cache-1r6slb0 {
+        background-color: #f0f2f6;
+        border-radius: 10px;
+        padding: 20px;
+        margin-bottom: 20px;
+    }
+    .st-emotion-cache-1wivap2 {
+        background-color: #ffffff;
+        border-radius: 10px;
+        padding: 20px;
+        box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+    }
+    .company-logo {
+        max-width: 200px;
+        margin-bottom: 20px;
+    }
+</style>
+""", unsafe_allow_html=True)
+
+# Company logo
+st.image("macrocosmos-black.png", use_column_width=False, width=200)
+
+# Title and description
+st.title("🤗 Hugging Face Large Dataset Explorer")
+st.markdown("Explore massive datasets hosted on Hugging Face, totaling approximately 100GB of data.")
+
+# Function to load dataset information
+def load_datasets():
+    return [
+        {"Source": "X", "DataSet repo link": "https://huggingface.co/datasets/icedwind/x_dataset_19", "Number of rows": "332 MLN"},
+        {"Source": "X", "DataSet repo link": "https://huggingface.co/datasets/wenknow/x_dataset", "Number of rows": "9.9 K"},
+        {"Source": "X", "DataSet repo link": "https://huggingface.co/datasets/arrmlet/reddit_dataset_123", "Number of rows": "89 K"},
+        {"Source": "Reddit", "DataSet repo link": "https://huggingface.co/datasets/dataverse-scraping/reddit_dataset_192", "Number of rows": "249 MLN"},
+        {"Source": "Reddit", "DataSet repo link": "https://huggingface.co/datasets/icedwind/reddit_dataset_226", "Number of rows": "303 MLN"},
+        {"Source": "Reddit", "DataSet repo link": "https://huggingface.co/datasets/arrmlet/reddit_dataset_123", "Number of rows": "1.12 MLN"},
+        {"Source": "Reddit", "DataSet repo link": "https://huggingface.co/datasets/chris241/reddit_dataset_75", "Number of rows": "132 MLN"},
+        {"Source": "Reddit", "DataSet repo link": "https://huggingface.co/datasets/wenknow/reddit_dataset_242", "Number of rows": "130 MLN"},
+        {"Source": "Reddit", "DataSet repo link": "https://huggingface.co/datasets/mgrtsv/reddit_dataset_231", "Number of rows": "31.2 MLN"},
+        {"Source": "Reddit", "DataSet repo link": "https://huggingface.co/datasets/PlanAPlanB/reddit_dataset_9", "Number of rows": "26.9 MLN"}
+    ]
+
+# Load datasets
+datasets = load_datasets()
+df = pd.DataFrame(datasets)
+
+# Display statistics
+col1, col2 = st.columns(2)
+with col1:
+    total_rows = sum(float(str(rows).split()[0].replace(',', '')) for rows in df['Number of rows'])
+    st.metric("Total Rows", f"{total_rows:.2f} Million")
+with col2:
+    st.metric("Total Datasets", len(df))
+
+# Display the dataset table
+st.subheader("Dataset Overview")
+st.dataframe(
+    df,
+    column_config={
+        "Source": st.column_config.TextColumn("Source"),
+        "DataSet repo link": st.column_config.LinkColumn("Repository"),
+        "Number of rows": st.column_config.TextColumn("Rows"),
+    },
+    hide_index=True,
+    use_container_width=True
+)
+
+# Add a note about the size of the datasets
+st.info("⚠️ These datasets are very large, totaling approximately 100GB. They are not available for direct download through this interface. Please visit the individual dataset links for more information on accessing the data.")
+
+# Add instructions for using the datasets
+st.subheader("How to Use These Datasets")
+code = '''
+from datasets import load_dataset
+
+dataset = load_dataset("username/dataset_name")
+'''
+st.code(code, language='python')
+
+st.markdown("""
+1. Click on the dataset link to visit its Hugging Face page.
+2. On the dataset page, you'll find information about the dataset's content, structure, and usage.
+3. Use the code above to load a dataset, replacing `"username/dataset_name"` with the actual dataset identifier.
+4. For these large datasets, consider using streaming or loading specific subsets to manage memory usage.
+5. Always check the dataset's license and usage restrictions before incorporating it into your project.
+""")
+
+# Footer
+st.markdown("---")
+st.markdown("Created by Macrocosmos with ❤️ ")

requirements.txt

@@ -0,0 +1,3 @@
+pandas
+streamlit
+streamlit-card