Automated update from GitHub

.github/workflows/deploy.yml

@@ -0,0 +1,44 @@
+name: Deploy to Hugging Face Space
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout source repo
+        uses: actions/checkout@v3
+
+      - name: Clone HF Space repo
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "kitamura-felipe"
+          git clone https://huggingface.co/spaces/felipekitamura/omnibin space
+          rsync -av --exclude='.git' ./ space/
+
+          echo '---' > temp_readme.md
+          echo 'title: Omnibin' >> temp_readme.md
+          echo 'emoji: ⚡' >> temp_readme.md
+          echo 'colorFrom: pink' >> temp_readme.md
+          echo 'colorTo: yellow' >> temp_readme.md
+          echo 'sdk: gradio' >> temp_readme.md
+          echo 'sdk_version: 5.29.0' >> temp_readme.md
+          echo 'app_file: app.py' >> temp_readme.md
+          echo 'pinned: false' >> temp_readme.md
+          echo 'license: mit' >> temp_readme.md
+          echo 'short_description: A Python package for generating comprehensive binary classi' >> temp_readme.md
+          echo '---' >> temp_readme.md
+          echo '' >> temp_readme.md
+          cat README.md >> temp_readme.md
+          mv temp_readme.md space/README.md
+
+          cd space
+          git add .
+          git commit -m "Automated update from GitHub" || echo "No changes"
+          git push https://USER:[email protected]/spaces/felipekitamura/omnibin HEAD:main
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,26 @@
+name: Run Tests
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          pip install pytest
+
+      - name: Run tests
+        run: pytest tests/

space/space/.github/workflows/update-hf-space.yml

@@ -3,11 +3,33 @@ name: Deploy to Hugging Face Space
 on:
   push:
     branches:
-      - main  # or the branch you want to trigger deployment
+      - main
 
 jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout source repo
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install pytest
+
+      - name: Run tests
+        run: pytest tests/
+
   deploy:
     runs-on: ubuntu-latest
+    needs: test  # 👈 only runs if 'test' job passes
 
     steps:
       - name: Checkout source repo
@@ -35,6 +57,7 @@ jobs:
           echo '' >> temp_readme.md
           cat README.md >> temp_readme.md
           mv temp_readme.md space/README.md
+
           cd space
           git add .
           git commit -m "Automated update from GitHub" || echo "No changes"

space/space/.gitignore

@@ -48,4 +48,5 @@ htmlcov/
 *.xlsx
 *.xls
 *.db
-*.sqlite3 
+*.sqlite3 
+plots/

space/space/README.md

@@ -1,4 +1,6 @@
-[![CI/CD](https://github.com/kitamura-felipe/omnibin/actions/workflows/update-hf-space.yml/badge.svg)](https://github.com/kitamura-felipe/omnibin/actions/workflows/update-hf-space.yml)
+[![Tests](https://github.com/kitamura-felipe/omnibin/actions/workflows/test.yml/badge.svg)](https://github.com/kitamura-felipe/omnibin/actions/workflows/test.yml)
+[![Deploy](https://github.com/kitamura-felipe/omnibin/actions/workflows/deploy.yml/badge.svg)](https://github.com/kitamura-felipe/omnibin/actions/workflows/deploy.yml)
+
 
 # Omnibin
 

space/space/space/space/README.md

@@ -1,4 +1,4 @@
-[![CI/CD](https://github.com/felipekitamura/omnibin/actions/workflows/update-hf-space.yml/badge.svg)](https://github.com/felipekitamura/omnibin/actions/workflows/update-hf-space.yml)
+[![CI/CD](https://github.com/kitamura-felipe/omnibin/actions/workflows/update-hf-space.yml/badge.svg)](https://github.com/kitamura-felipe/omnibin/actions/workflows/update-hf-space.yml)
 
 # Omnibin
 

space/space/space/space/space/README.md

@@ -1,3 +1,5 @@
+[![CI/CD](https://github.com/felipekitamura/omnibin/actions/workflows/update-hf-space.yml/badge.svg)](https://github.com/felipekitamura/omnibin/actions/workflows/update-hf-space.yml)
+
 # Omnibin
 
 A Python package for generating comprehensive binary classification reports with visualizations and confidence intervals.

space/space/space/space/space/space/.github/workflows/update-hf-space.yml

@@ -19,6 +19,22 @@ jobs:
           git config --global user.name "kitamura-felipe"
           git clone https://huggingface.co/spaces/felipekitamura/omnibin space
           rsync -av --exclude='.git' ./ space/
+
+          echo '---' > temp_readme.md
+          echo 'title: Omnibin' >> temp_readme.md
+          echo 'emoji: ⚡' >> temp_readme.md
+          echo 'colorFrom: pink' >> temp_readme.md
+          echo 'colorTo: yellow' >> temp_readme.md
+          echo 'sdk: gradio' >> temp_readme.md
+          echo 'sdk_version: 5.29.0' >> temp_readme.md
+          echo 'app_file: app.py' >> temp_readme.md
+          echo 'pinned: false' >> temp_readme.md
+          echo 'license: mit' >> temp_readme.md
+          echo 'short_description: A Python package for generating comprehensive binary classi' >> temp_readme.md
+          echo '---' >> temp_readme.md
+          echo '' >> temp_readme.md
+          cat README.md >> temp_readme.md
+          mv temp_readme.md space/README.md
           cd space
           git add .
           git commit -m "Automated update from GitHub" || echo "No changes"

space/space/space/space/space/space/README.md

@@ -97,18 +97,3 @@ Here are examples of the visualizations generated by Omnibin:
 
 ### Metrics Summary
 ![Metrics Summary](results/plots/metrics_summary.png) 
-
-
-
----
-title: Omnibin
-emoji: ⚡
-colorFrom: pink
-colorTo: yellow
-sdk: gradio
-sdk_version: 5.29.0
-app_file: app.py
-pinned: false
-license: mit
-short_description: A Python package for generating comprehensive binary classi
----

space/space/space/space/space/space/space/README.md

@@ -1,16 +1,3 @@
----
-title: Omnibin
-emoji: ⚡
-colorFrom: pink
-colorTo: yellow
-sdk: gradio
-sdk_version: 5.29.0
-app_file: app.py
-pinned: false
-license: mit
-short_description: A Python package for generating comprehensive binary classi
----
-
 # Omnibin
 
 A Python package for generating comprehensive binary classification reports with visualizations and confidence intervals.
@@ -109,4 +96,19 @@ Here are examples of the visualizations generated by Omnibin:
 <img src="results/plots/prediction_distribution.png">
 
 ### Metrics Summary
-![Metrics Summary](results/plots/metrics_summary.png) 
+![Metrics Summary](results/plots/metrics_summary.png) 
+
+
+
+---
+title: Omnibin
+emoji: ⚡
+colorFrom: pink
+colorTo: yellow
+sdk: gradio
+sdk_version: 5.29.0
+app_file: app.py
+pinned: false
+license: mit
+short_description: A Python package for generating comprehensive binary classi
+---

space/space/space/space/space/space/space/space/space/space/space/.github/workflows/update-hf-space.yml

@@ -0,0 +1,27 @@
+name: Deploy to Hugging Face Space
+
+on:
+  push:
+    branches:
+      - main  # or the branch you want to trigger deployment
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout source repo
+        uses: actions/checkout@v3
+
+      - name: Clone HF Space repo
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "kitamura-felipe"
+          git clone https://huggingface.co/spaces/felipekitamura/omnibin space
+          rsync -av --exclude='.git' ./ space/
+          cd space
+          git add .
+          git commit -m "Automated update from GitHub" || echo "No changes"
+          git push https://USER:[email protected]/spaces/felipekitamura/omnibin HEAD:main
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}

space/space/space/space/space/space/space/space/space/space/space/.gitignore

@@ -0,0 +1,51 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual Environment
+venv/
+env/
+ENV/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# Distribution
+dist/
+build/
+update.bat
+
+
+# Coverage
+.coverage
+htmlcov/
+
+# Results and data
+*.xlsx
+*.xls
+*.db
+*.sqlite3 

space/space/space/space/space/space/space/space/space/space/space/CHANGELOG.md

@@ -0,0 +1,15 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.0] - 2024-03-19
+
+### Added
+- Initial release
+- Basic binary classification metrics and visualizations
+- Comprehensive reporting functionality
+- Confidence interval calculations
+- Example usage and documentation 

space/space/space/space/space/space/space/space/space/space/space/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Felipe Campos Kitamura
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

space/space/space/space/space/space/space/space/space/space/space/example.py

@@ -0,0 +1,36 @@
+import pandas as pd
+import numpy as np
+import os
+from omnibin import generate_binary_classification_report, ColorScheme
+
+# Define paths
+RESULTS_DIR = os.path.join(os.path.dirname(__file__), "results")
+
+# Ensure results directory exists
+os.makedirs(RESULTS_DIR, exist_ok=True)
+
+# Generate random data
+data = pd.DataFrame({
+    'y_true': (y:=np.random.choice([0,1],1000,p:=[.9,.1])),
+    'y_pred': np.where(
+        y,
+        np.random.beta(3,1.5,1000)*.9+.1,  # Positive cases: less skewed towards 1.0
+        np.random.beta(1.5,3,1000)*.9+.1   # Negative cases: less skewed towards 0.1
+    )
+})
+
+y_true = data['y_true'].values
+y_scores = data['y_pred'].values
+
+# Generate comprehensive classification report
+report_path = generate_binary_classification_report(
+    y_true=y_true,
+    y_scores=y_scores,
+    output_path=os.path.join(RESULTS_DIR, "classification_report.pdf"),
+    n_bootstrap=1000,
+    random_seed=42,  # Set a fixed random seed for reproducibility
+    dpi=72,
+    color_scheme=ColorScheme.DEFAULT
+)
+
+print(f"Report generated and saved to: {report_path}")

space/space/space/space/space/space/space/space/space/space/space/pyproject.toml

@@ -0,0 +1,31 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "omnibin"
+version = "0.1.5"
+description = "A package for generating comprehensive binary classification reports with visualizations and confidence intervals"
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+authors = [
+    { name = "Felipe Campos Kitamura", email = "[email protected]" }
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "numpy>=1.21.0",
+    "pandas>=1.3.0",
+    "scikit-learn>=1.0.0",
+    "matplotlib>=3.4.0",
+    "scipy>=1.7.0",
+    "seaborn>=0.11.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/kitamura-felipe/omnibin"
+Repository = "https://github.com/kitamura-felipe/omnibin.git" 

space/space/space/space/space/space/space/space/space/space/space/space/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

space/space/space/space/space/space/space/space/space/space/space/space/README.md

@@ -0,0 +1,112 @@
+---
+title: Omnibin
+emoji: ⚡
+colorFrom: pink
+colorTo: yellow
+sdk: gradio
+sdk_version: 5.29.0
+app_file: app.py
+pinned: false
+license: mit
+short_description: A Python package for generating comprehensive binary classi
+---
+
+# Omnibin
+
+A Python package for generating comprehensive binary classification reports with visualizations and confidence intervals.
+
+## Try it Online
+
+You can try Omnibin directly in your browser through our [Hugging Face Space](https://felipekitamura-omnibin.hf.space).
+
+## Installation
+
+```bash
+pip install omnibin
+```
+
+## Usage
+
+```python
+import pandas as pd
+from omnibin import generate_binary_classification_report, ColorScheme
+
+# Load your data
+data = pd.read_csv("data/scores.csv")
+y_true = data['y_true'].values
+y_scores = data['y_pred'].values
+
+# Generate comprehensive classification report
+report_path = generate_binary_classification_report(
+    y_true=y_true,                    # Array of true binary labels (0 or 1)
+    y_scores=y_scores,                # Array of predicted probabilities or scores
+    output_path="classification_report.pdf",  # Path to save the PDF report
+    n_bootstrap=1000,                 # Number of bootstrap iterations for confidence intervals
+    random_seed=42,                   # Random seed for reproducibility
+    dpi=300,                          # DPI for plot resolution
+    color_scheme=ColorScheme.DEFAULT  # Color scheme for plots (DEFAULT, MONOCHROME, or VIBRANT)
+)
+```
+
+## Input Format
+
+The input data should be provided as:
+- `y_true`: Array of true binary labels (0 or 1)
+- `y_pred`: Array of predicted probabilities or scores
+
+## Features
+
+- Generates a comprehensive PDF report with:
+  - ROC curve with confidence bands
+  - Precision-Recall curve with confidence bands
+  - Metrics vs. threshold plots
+  - Confusion matrix at optimal threshold
+  - Calibration plot
+  - Summary table with confidence intervals
+- Calculates optimal threshold using Youden's J statistic
+- Provides confidence intervals using bootstrapping
+- Supports both probability and score-based predictions
+
+## Metrics Included
+
+- Accuracy
+- Sensitivity (Recall)
+- Specificity
+- Positive Predictive Value (Precision)
+- Matthews Correlation Coefficient
+- F1 Score
+- AUC-ROC
+- AUC-PR
+
+All metrics include 95% confidence intervals calculated through bootstrapping.
+
+## Output
+
+The package generates a PDF report containing:
+1. ROC and Precision-Recall curves with confidence bands
+2. Metrics plotted across different thresholds
+3. Confusion matrix at the optimal threshold
+4. Calibration plot
+5. Summary table with all metrics and their confidence intervals 
+
+## Example
+
+Here are examples of the visualizations generated by Omnibin:
+
+### ROC and Precision-Recall Curves
+![ROC and PR Curves](results/plots/roc_pr.png)
+
+### Metrics vs Threshold
+<img src="results/plots/metrics_threshold.png">
+
+### Confusion Matrix
+<img src="results/plots/confusion_matrix.png">
+
+### Calibration Plot
+<img src="results/plots/calibration.png">
+
+### Prediction Distribution
+<img src="results/plots/prediction_distribution.png">
+
+### Metrics Summary
+![Metrics Summary](results/plots/metrics_summary.png) 

space/space/space/space/space/space/space/space/space/space/space/space/app.py

@@ -0,0 +1,90 @@
+import gradio as gr
+import pandas as pd
+import os
+import shutil
+from omnibin import generate_binary_classification_report, ColorScheme
+
+# Define results directory
+RESULTS_DIR = "/tmp/results"
+
+# Map string color schemes to enum values
+COLOR_SCHEME_MAP = {
+    "DEFAULT": ColorScheme.DEFAULT,
+    "MONOCHROME": ColorScheme.MONOCHROME,
+    "VIBRANT": ColorScheme.VIBRANT
+}
+
+def process_csv(csv_file, n_bootstrap=1000, dpi=72, color_scheme="DEFAULT"):
+    # Convert string color scheme to enum
+    color_scheme_enum = COLOR_SCHEME_MAP[color_scheme]
+    
+    # Read the CSV file
+    df = pd.read_csv(csv_file.name)
+    
+    # Check if required columns exist
+    required_columns = ['y_true', 'y_pred']
+    if not all(col in df.columns for col in required_columns):
+        raise ValueError("CSV file must contain 'y_true' and 'y_pred' columns")
+    
+    # Clean up results directory if it exists
+    if os.path.exists(RESULTS_DIR):
+        shutil.rmtree(RESULTS_DIR)
+    
+    # Create fresh results directory
+    os.makedirs(RESULTS_DIR, exist_ok=True)
+    
+    # Generate the report
+    report_path = generate_binary_classification_report(
+        y_true=df['y_true'].values,
+        y_scores=df['y_pred'].values,
+        output_path=os.path.join(RESULTS_DIR, "classification_report.pdf"),
+        n_bootstrap=n_bootstrap,
+        random_seed=42,
+        dpi=dpi,
+        color_scheme=color_scheme_enum
+    )
+    
+    # Get paths to individual plots
+    plots_dir = os.path.join(RESULTS_DIR, "plots")
+    plot_paths = {
+        "ROC and PR Curves": os.path.join(plots_dir, "roc_pr.png"),
+        "Metrics vs Threshold": os.path.join(plots_dir, "metrics_threshold.png"),
+        "Confusion Matrix": os.path.join(plots_dir, "confusion_matrix.png"),
+        "Calibration Plot": os.path.join(plots_dir, "calibration.png"),
+        "Prediction Distribution": os.path.join(plots_dir, "prediction_distribution.png"),
+        "Metrics Summary": os.path.join(plots_dir, "metrics_summary.png")
+    }
+    
+    # Return both the PDF and the plot images
+    return report_path, *plot_paths.values()
+
+# Create the Gradio interface
+iface = gr.Interface(
+    fn=process_csv,
+    inputs=[
+        gr.File(label="Upload CSV file with 'y_true' and 'y_pred' columns"),
+        gr.Number(label="Number of Bootstrap Iterations", value=1000, minimum=100, maximum=10000),
+        gr.Number(label="DPI", value=72, minimum=50, maximum=300),
+        gr.Dropdown(label="Color Scheme", choices=["DEFAULT", "MONOCHROME", "VIBRANT"], value="DEFAULT")
+    ],
+    outputs=[
+        gr.File(label="Classification Report PDF"),
+        gr.Image(label="ROC and PR Curves"),
+        gr.Image(label="Metrics vs Threshold"),
+        gr.Image(label="Confusion Matrix"),
+        gr.Image(label="Calibration Plot"),
+        gr.Image(label="Prediction Distribution"),
+        gr.Image(label="Metrics Summary")
+    ],
+    title="Binary Classification Report Generator",
+    description="Upload a CSV file containing 'y_true' and 'y_pred' columns to generate a binary classification report.\n\n"
+                "'y_true': reference standard (0s or 1s).\n\n"
+                "'y_pred': model prediction (continuous value between 0 and 1).\n\n"
+                "This application takes approximately 35 seconds to generate the report.\n",
+
+    examples=[["scores.csv", 1000, 72, "DEFAULT"]],
+    cache_examples=False
+)
+
+if __name__ == "__main__":
+    iface.launch()

space/space/space/space/space/space/space/space/space/space/space/space/omnibin/__init__.py

@@ -0,0 +1,4 @@
+from .metrics import generate_binary_classification_report, ColorScheme
+
+__version__ = "0.1.5"
+__all__ = ["generate_binary_classification_report"] 

space/space/space/space/space/space/space/space/space/space/space/space/omnibin/metrics.py

@@ -0,0 +1,62 @@
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+from tqdm import tqdm
+import os
+from sklearn.metrics import (
+    accuracy_score, recall_score, precision_score, f1_score, roc_auc_score,
+    average_precision_score, confusion_matrix, matthews_corrcoef, roc_curve,
+    precision_recall_curve
+)
+from sklearn.calibration import calibration_curve
+from matplotlib.backends.backend_pdf import PdfPages
+from enum import Enum
+from .utils import (
+    ColorScheme, calculate_metrics_by_threshold, bootstrap_curves,
+    calculate_optimal_threshold, calculate_metrics_summary,
+    calculate_confidence_intervals, create_output_directories,
+    plot_roc_pr_curves, plot_metrics_threshold, plot_confusion_matrix,
+    plot_calibration, plot_metrics_summary, plot_prediction_distribution
+)
+
+def generate_binary_classification_report(y_true, y_scores, output_path="omnibin_report.pdf", n_bootstrap=1000, random_seed=42, dpi=300, color_scheme=ColorScheme.DEFAULT):
+    # Set random seed for reproducibility
+    if random_seed is not None:
+        np.random.seed(random_seed)
+    
+    # Set DPI for all figures
+    plt.rcParams['figure.dpi'] = dpi
+    
+    # Get color scheme
+    colors = color_scheme.value
+
+    # Calculate metrics and optimal threshold
+    metrics_df = calculate_metrics_by_threshold(y_true, y_scores)
+    best_thresh = calculate_optimal_threshold(y_true, y_scores)
+    metrics_summary = calculate_metrics_summary(y_true, y_scores, best_thresh)
+    conf_intervals = calculate_confidence_intervals(y_true, y_scores, best_thresh, n_bootstrap)
+
+    # Create output directories
+    plots_dir = create_output_directories(output_path)
+
+    # Calculate confidence intervals for curves
+    tpr_ci, precision_ci, common_fpr, common_recall = bootstrap_curves(y_true, y_scores, n_boot=n_bootstrap)
+
+    with PdfPages(output_path) as pdf:
+        # Generate and save all plots
+        plots = [
+            plot_roc_pr_curves(y_true, y_scores, tpr_ci, precision_ci, common_fpr, common_recall, colors, dpi, plots_dir),
+            plot_metrics_threshold(metrics_df, colors, dpi, plots_dir),
+            plot_confusion_matrix(y_true, y_scores, best_thresh, colors, dpi, plots_dir),
+            plot_calibration(y_true, y_scores, colors, dpi, plots_dir),
+            plot_metrics_summary(metrics_summary, conf_intervals, dpi, plots_dir),
+            plot_prediction_distribution(y_true, y_scores, best_thresh, colors, dpi, plots_dir)
+        ]
+        
+        # Save all plots to PDF
+        for plot in plots:
+            pdf.savefig(plot, dpi=dpi)
+            plt.close(plot)
+
+    return output_path

space/space/space/space/space/space/space/space/space/space/space/space/omnibin/utils.py

@@ -0,0 +1,263 @@
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+from tqdm import tqdm
+from sklearn.metrics import (
+    accuracy_score, recall_score, precision_score, f1_score, roc_auc_score,
+    average_precision_score, confusion_matrix, matthews_corrcoef, roc_curve,
+    precision_recall_curve
+)
+from sklearn.calibration import calibration_curve
+from enum import Enum
+import os
+
+class ColorScheme(Enum):
+    DEFAULT = {
+        'positive_class': 'tab:blue',
+        'negative_class': 'tab:orange',
+        'roc_curve': 'tab:blue',
+        'pr_curve': 'tab:blue',
+        'threshold_line': 'black',
+        'calibration_curve': 'tab:blue',
+        'calibration_reference': 'gray',
+        'metrics_colors': ['tab:blue', 'tab:red', 'tab:green', 'tab:purple', 'tab:orange', 'tab:brown', 'tab:pink'],
+        'cmap': 'Blues'
+    }
+    
+    MONOCHROME = {
+        'positive_class': '#404040',
+        'negative_class': '#808080',
+        'roc_curve': '#000000',
+        'pr_curve': '#000000',
+        'threshold_line': '#000000',
+        'calibration_curve': '#000000',
+        'calibration_reference': '#808080',
+        'metrics_colors': ['#000000', '#404040', '#606060', '#808080', '#A0A0A0', '#C0C0C0', '#E0E0E0'],
+        'cmap': 'Greys'
+    }
+    
+    VIBRANT = {
+        'positive_class': '#FF6B6B',
+        'negative_class': '#4ECDC4',
+        'roc_curve': '#FF6B6B',
+        'pr_curve': '#4ECDC4',
+        'threshold_line': '#2C3E50',
+        'calibration_curve': '#FF6B6B',
+        'calibration_reference': '#95A5A6',
+        'metrics_colors': ['#FF6B6B', '#4ECDC4', '#45B7D1', '#96CEB4', '#FFEEAD', '#D4A5A5', '#9B59B6'],
+        'cmap': 'Greens'
+    }
+
+def calculate_metrics_by_threshold(y_true, y_scores):
+    """Calculate various metrics across different thresholds."""
+    thresholds = np.linspace(0, 1, 100)
+    metrics_by_threshold = []
+
+    for t in tqdm(thresholds, desc="Calculating metrics across thresholds"):
+        y_pred = (y_scores >= t).astype(int)
+        acc = accuracy_score(y_true, y_pred)
+        sens = recall_score(y_true, y_pred)
+        spec = recall_score(y_true, y_pred, pos_label=0)
+        ppv = precision_score(y_true, y_pred, zero_division=0)
+        mcc = matthews_corrcoef(y_true, y_pred)
+        f1 = f1_score(y_true, y_pred)
+        metrics_by_threshold.append([t, acc, sens, spec, ppv, mcc, f1])
+
+    return pd.DataFrame(metrics_by_threshold, columns=[
+        "Threshold", "Accuracy", "Sensitivity", "Specificity",
+        "PPV", "MCC", "F1 Score"
+    ])
+
+def bootstrap_metric(metric_func, y_true, y_scores, n_boot=1000):
+    """Calculate bootstrap confidence intervals for a given metric."""
+    stats = []
+    for _ in tqdm(range(n_boot), desc="Bootstrap iterations", leave=False):
+        indices = np.random.choice(range(len(y_true)), len(y_true), replace=True)
+        try:
+            stats.append(metric_func(y_true[indices], y_scores[indices]))
+        except:
+            continue
+    return np.percentile(stats, [2.5, 97.5])
+
+def bootstrap_curves(y_true, y_scores, n_boot=1000):
+    """Calculate bootstrap confidence intervals for ROC and PR curves."""
+    tprs = []
+    fprs = []
+    precisions = []
+    recalls = []
+    
+    base_fpr, base_tpr, _ = roc_curve(y_true, y_scores)
+    base_precision, base_recall, _ = precision_recall_curve(y_true, y_scores)
+    
+    common_fpr = np.linspace(0, 1, 100)
+    common_recall = np.linspace(0, 1, 100)
+    
+    for _ in tqdm(range(n_boot), desc="Bootstrap iterations for curves", leave=False):
+        indices = np.random.choice(range(len(y_true)), len(y_true), replace=True)
+        try:
+            fpr, tpr, _ = roc_curve(y_true[indices], y_scores[indices])
+            tpr_interp = np.interp(common_fpr, fpr, tpr)
+            tprs.append(tpr_interp)
+            
+            precision, recall, _ = precision_recall_curve(y_true[indices], y_scores[indices])
+            sort_idx = np.argsort(recall)
+            recall = recall[sort_idx]
+            precision = precision[sort_idx]
+            precision_interp = np.interp(common_recall, recall, precision)
+            precisions.append(precision_interp)
+        except:
+            continue
+    
+    tpr_ci = np.percentile(tprs, [2.5, 97.5], axis=0)
+    precision_ci = np.percentile(precisions, [2.5, 97.5], axis=0)
+    
+    return tpr_ci, precision_ci, common_fpr, common_recall
+
+def calculate_optimal_threshold(y_true, y_scores):
+    """Calculate the optimal threshold using ROC curve."""
+    fpr, tpr, roc_thresholds = roc_curve(y_true, y_scores)
+    j_scores = tpr - fpr
+    return roc_thresholds[np.argmax(j_scores)]
+
+def calculate_metrics_summary(y_true, y_scores, best_thresh):
+    """Calculate summary metrics at the optimal threshold."""
+    y_pred_opt = (y_scores >= best_thresh).astype(int)
+    
+    return {
+        "Accuracy": accuracy_score(y_true, y_pred_opt),
+        "Sensitivity": recall_score(y_true, y_pred_opt),
+        "Specificity": recall_score(y_true, y_pred_opt, pos_label=0),
+        "PPV": precision_score(y_true, y_pred_opt, zero_division=0),
+        "MCC": matthews_corrcoef(y_true, y_pred_opt),
+        "F1 Score": f1_score(y_true, y_pred_opt),
+        "AUC-ROC": roc_auc_score(y_true, y_scores),
+        "AUC-PR": average_precision_score(y_true, y_scores)
+    }
+
+def calculate_confidence_intervals(y_true, y_scores, best_thresh, n_bootstrap=1000):
+    """Calculate confidence intervals for all metrics."""
+    metric_functions = {
+        "Accuracy": lambda yt, ys: accuracy_score(yt, ys >= best_thresh),
+        "Sensitivity": lambda yt, ys: recall_score(yt, ys >= best_thresh),
+        "Specificity": lambda yt, ys: recall_score(yt, ys >= best_thresh, pos_label=0),
+        "PPV": lambda yt, ys: precision_score(yt, ys >= best_thresh, zero_division=0),
+        "MCC": lambda yt, ys: matthews_corrcoef(yt, ys >= best_thresh),
+        "F1 Score": lambda yt, ys: f1_score(yt, ys >= best_thresh),
+        "AUC-ROC": lambda yt, ys: roc_auc_score(yt, ys),
+        "AUC-PR": lambda yt, ys: average_precision_score(yt, ys)
+    }
+    
+    return {
+        name: bootstrap_metric(func, y_true, y_scores, n_boot=n_bootstrap)
+        for name, func in metric_functions.items()
+    }
+
+def create_output_directories(output_path):
+    """Create necessary output directories for plots and PDF."""
+    output_dir = os.path.dirname(output_path)
+    if output_dir:
+        os.makedirs(output_dir, exist_ok=True)
+    
+    plots_dir = os.path.join(output_dir, "plots")
+    os.makedirs(plots_dir, exist_ok=True)
+    
+    return plots_dir
+
+def plot_roc_pr_curves(y_true, y_scores, tpr_ci, precision_ci, common_fpr, common_recall, colors, dpi, plots_dir):
+    """Generate ROC and PR curves with confidence intervals."""
+    plt.figure(figsize=(12, 5), dpi=dpi)
+    
+    plt.subplot(1, 2, 1)
+    fpr, tpr, _ = roc_curve(y_true, y_scores)
+    plt.plot(fpr, tpr, label="ROC curve", color=colors['roc_curve'])
+    plt.fill_between(common_fpr, tpr_ci[0], tpr_ci[1], alpha=0.3, color=colors['roc_curve'])
+    plt.plot([0, 1], [0, 1], "k--")
+    plt.xlabel("False Positive Rate")
+    plt.ylabel("True Positive Rate")
+    plt.title("ROC Curve")
+    plt.legend()
+
+    plt.subplot(1, 2, 2)
+    precision, recall, _ = precision_recall_curve(y_true, y_scores)
+    plt.plot(recall, precision, label="PR curve", color=colors['pr_curve'])
+    plt.fill_between(common_recall, precision_ci[0], precision_ci[1], alpha=0.3, color=colors['pr_curve'])
+    plt.xlabel("Recall")
+    plt.ylabel("Precision")
+    plt.title("Precision-Recall Curve")
+    plt.legend()
+    
+    plt.savefig(os.path.join(plots_dir, "roc_pr.png"), dpi=dpi, bbox_inches='tight')
+    return plt.gcf()
+
+def plot_metrics_threshold(metrics_df, colors, dpi, plots_dir):
+    """Generate metrics vs threshold plot."""
+    plt.figure(figsize=(10, 6), dpi=dpi)
+    for i, col in enumerate(metrics_df.columns[1:]):
+        plt.plot(metrics_df["Threshold"], metrics_df[col], label=col, 
+                color=colors['metrics_colors'][i % len(colors['metrics_colors'])])
+    plt.xlabel("Threshold")
+    plt.ylabel("Metric Value")
+    plt.title("Metrics Across Thresholds")
+    plt.legend()
+    
+    plt.savefig(os.path.join(plots_dir, "metrics_threshold.png"), dpi=dpi, bbox_inches='tight')
+    return plt.gcf()
+
+def plot_confusion_matrix(y_true, y_scores, best_thresh, colors, dpi, plots_dir):
+    """Generate confusion matrix plot."""
+    cm = confusion_matrix(y_true, y_scores >= best_thresh)
+    plt.figure(figsize=(5, 4), dpi=dpi)
+    sns.heatmap(cm, annot=True, fmt="d", cmap=colors['cmap'], cbar=False, annot_kws={"size": 12})
+    plt.title("Confusion Matrix (Optimal Threshold)", fontsize=12)
+    plt.xlabel("Predicted Label", fontsize=12)
+    plt.ylabel("True Label", fontsize=12)
+    
+    plt.savefig(os.path.join(plots_dir, "confusion_matrix.png"), dpi=dpi, bbox_inches='tight')
+    return plt.gcf()
+
+def plot_calibration(y_true, y_scores, colors, dpi, plots_dir):
+    """Generate calibration plot."""
+    plt.figure(figsize=(6, 6), dpi=dpi)
+    prob_true, prob_pred = calibration_curve(y_true, y_scores, n_bins=10, strategy='uniform')
+    plt.plot(prob_pred, prob_true, marker='o', label='Calibration curve', color=colors['calibration_curve'])
+    plt.plot([0, 1], [0, 1], linestyle='--', color=colors['calibration_reference'])
+    plt.xlabel('Predicted Probability')
+    plt.ylabel('True Probability')
+    plt.title('Calibration Plot')
+    plt.legend()
+    
+    plt.savefig(os.path.join(plots_dir, "calibration.png"), dpi=dpi, bbox_inches='tight')
+    return plt.gcf()
+
+def plot_metrics_summary(metrics_summary, conf_intervals, dpi, plots_dir):
+    """Generate metrics summary table plot."""
+    fig, ax = plt.subplots(figsize=(8, 6), dpi=dpi)
+    ax.axis("off")
+    table_data = [
+        [k, f"{v:.3f}", f"[{conf_intervals[k][0]:.3f}, {conf_intervals[k][1]:.3f}]"]
+        for k, v in metrics_summary.items()
+    ]
+    table = ax.table(cellText=table_data, colLabels=["Metric", "Value", "95% CI"], loc="center")
+    table.auto_set_font_size(False)
+    table.set_fontsize(10)
+    table.scale(1.2, 1.2)
+    ax.set_title("Performance Metrics at Optimal Threshold", fontweight="bold")
+    
+    plt.savefig(os.path.join(plots_dir, "metrics_summary.png"), dpi=dpi, bbox_inches='tight')
+    return plt.gcf()
+
+def plot_prediction_distribution(y_true, y_scores, best_thresh, colors, dpi, plots_dir):
+    """Generate prediction distribution histogram."""
+    plt.figure(figsize=(10, 6), dpi=dpi)
+    plt.hist(y_scores[y_true == 1], bins=50, alpha=0.5, label='Positive Class', color=colors['positive_class'])
+    plt.hist(y_scores[y_true == 0], bins=50, alpha=0.5, label='Negative Class', color=colors['negative_class'])
+    plt.axvline(x=best_thresh, color=colors['threshold_line'], linestyle='--', 
+                label=f'Optimal Threshold ({best_thresh:.3f})')
+    plt.xlabel('Predicted Probability')
+    plt.ylabel('Count')
+    plt.title('Distribution of Predictions')
+    plt.legend()
+    
+    plt.savefig(os.path.join(plots_dir, "prediction_distribution.png"), dpi=dpi, bbox_inches='tight')
+    return plt.gcf() 

space/space/space/space/space/space/space/space/space/space/space/space/requirements.txt

@@ -0,0 +1,6 @@
+pandas
+numpy
+gradio
+matplotlib
+seaborn
+scikit-learn

space/space/space/space/space/space/space/space/space/space/space/tests/__init__.py

@@ -0,0 +1,3 @@
+"""
+Unit tests for the omnibin package.
+""" 

space/space/space/space/space/space/space/space/space/space/space/tests/test_metrics.py

@@ -0,0 +1,141 @@
+import unittest
+import os
+import pandas as pd
+import numpy as np
+from omnibin.metrics import generate_binary_classification_report
+
+class TestMetrics(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        """Set up test data"""
+        # Create synthetic test data
+        np.random.seed(42)
+        n_samples = 1000
+        cls.y_true = np.random.binomial(1, 0.3, n_samples)
+        cls.y_scores = np.random.beta(2, 5, n_samples)
+        
+        # Adjust scores to have some correlation with true labels
+        cls.y_scores[cls.y_true == 1] = np.random.beta(5, 2, sum(cls.y_true == 1))
+        
+        # Create test output directory
+        cls.test_output_dir = "test_outputs"
+        os.makedirs(cls.test_output_dir, exist_ok=True)
+
+    def test_report_generation(self):
+        """Test the main report generation function"""
+        output_path = os.path.join(self.test_output_dir, "test_report.pdf")
+        
+        # Generate report
+        result_path = generate_binary_classification_report(
+            y_true=self.y_true,
+            y_scores=self.y_scores,
+            output_path=output_path,
+            n_bootstrap=100  # Use smaller number for testing
+        )
+        
+        # Test that file was created
+        self.assertTrue(os.path.exists(result_path))
+        self.assertTrue(os.path.getsize(result_path) > 0)
+        
+        # Clean up
+        os.remove(result_path)
+
+    def test_input_validation(self):
+        """Test input validation"""
+        # Test with invalid y_true values
+        with self.assertRaises(ValueError):
+            generate_binary_classification_report(
+                y_true=np.array([0, 1, 2]),  # Invalid label
+                y_scores=np.array([0.1, 0.5, 0.9])
+            )
+        
+        # Test with invalid y_scores values
+        with self.assertRaises(ValueError):
+            generate_binary_classification_report(
+                y_true=np.array([0, 1, 0]),
+                y_scores=np.array([-0.1, 1.5, 0.9])  # Values outside [0,1]
+            )
+        
+        # Test with mismatched lengths
+        with self.assertRaises(ValueError):
+            generate_binary_classification_report(
+                y_true=np.array([0, 1]),
+                y_scores=np.array([0.1, 0.5, 0.9])
+            )
+
+    def test_bootstrap_consistency(self):
+        """Test that bootstrap results are consistent"""
+        output_path1 = os.path.join(self.test_output_dir, "test_report1.pdf")
+        output_path2 = os.path.join(self.test_output_dir, "test_report2.pdf")
+        
+        # Generate two reports with same data and seed
+        np.random.seed(42)
+        generate_binary_classification_report(
+            y_true=self.y_true,
+            y_scores=self.y_scores,
+            output_path=output_path1,
+            n_bootstrap=100
+        )
+        
+        np.random.seed(42)
+        generate_binary_classification_report(
+            y_true=self.y_true,
+            y_scores=self.y_scores,
+            output_path=output_path2,
+            n_bootstrap=100
+        )
+        
+        # Compare file sizes (they should be similar)
+        size1 = os.path.getsize(output_path1)
+        size2 = os.path.getsize(output_path2)
+        self.assertAlmostEqual(size1, size2, delta=1000)  # Allow small differences due to PDF compression
+        
+        # Clean up
+        os.remove(output_path1)
+        os.remove(output_path2)
+
+    def test_edge_cases(self):
+        """Test edge cases"""
+        # Test with all positive labels
+        output_path = os.path.join(self.test_output_dir, "all_positive.pdf")
+        generate_binary_classification_report(
+            y_true=np.ones(100),
+            y_scores=np.random.random(100),
+            output_path=output_path,
+            n_bootstrap=100
+        )
+        self.assertTrue(os.path.exists(output_path))
+        os.remove(output_path)
+        
+        # Test with all negative labels
+        output_path = os.path.join(self.test_output_dir, "all_negative.pdf")
+        generate_binary_classification_report(
+            y_true=np.zeros(100),
+            y_scores=np.random.random(100),
+            output_path=output_path,
+            n_bootstrap=100
+        )
+        self.assertTrue(os.path.exists(output_path))
+        os.remove(output_path)
+        
+        # Test with perfect predictions
+        output_path = os.path.join(self.test_output_dir, "perfect.pdf")
+        generate_binary_classification_report(
+            y_true=np.array([0, 1, 0, 1]),
+            y_scores=np.array([0.1, 0.9, 0.2, 0.8]),
+            output_path=output_path,
+            n_bootstrap=100
+        )
+        self.assertTrue(os.path.exists(output_path))
+        os.remove(output_path)
+
+    @classmethod
+    def tearDownClass(cls):
+        """Clean up test outputs"""
+        if os.path.exists(cls.test_output_dir):
+            for file in os.listdir(cls.test_output_dir):
+                os.remove(os.path.join(cls.test_output_dir, file))
+            os.rmdir(cls.test_output_dir)
+
+if __name__ == '__main__':
+    unittest.main() 

space/space/tests/test_metrics.py

@@ -2,6 +2,8 @@ import unittest
 import os
 import pandas as pd
 import numpy as np
+import warnings
+from sklearn.exceptions import UndefinedMetricWarning
 from omnibin.metrics import generate_binary_classification_report
 
 class TestMetrics(unittest.TestCase):
@@ -109,12 +111,15 @@ class TestMetrics(unittest.TestCase):
         
         # Test with all negative labels
         output_path = os.path.join(self.test_output_dir, "all_negative.pdf")
-        generate_binary_classification_report(
-            y_true=np.zeros(100),
-            y_scores=np.random.random(100),
-            output_path=output_path,
-            n_bootstrap=100
-        )
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", category=UserWarning)
+            warnings.filterwarnings("ignore", category=UndefinedMetricWarning)
+            generate_binary_classification_report(
+                y_true=np.zeros(100),
+                y_scores=np.random.random(100),
+                output_path=output_path,
+                n_bootstrap=100
+            )
         self.assertTrue(os.path.exists(output_path))
         os.remove(output_path)
         
@@ -134,7 +139,13 @@ class TestMetrics(unittest.TestCase):
         """Clean up test outputs"""
         if os.path.exists(cls.test_output_dir):
             for file in os.listdir(cls.test_output_dir):
-                os.remove(os.path.join(cls.test_output_dir, file))
+                file_path = os.path.join(cls.test_output_dir, file)
+                if os.path.isfile(file_path):
+                    os.remove(file_path)
+                elif os.path.isdir(file_path):
+                    for subfile in os.listdir(file_path):
+                        os.remove(os.path.join(file_path, subfile))
+                    os.rmdir(file_path)
             os.rmdir(cls.test_output_dir)
 
 if __name__ == '__main__':