config fix and cleanup

config.json

@@ -6,14 +6,14 @@
   "classifier_dropout": null,
   "hidden_act": "relu",
   "hidden_dropout_prob": 0.02,
-  "hidden_size": 512,
+  "hidden_size": 896,
   "initializer_range": 0.02,
-  "intermediate_size": 1024,
+  "intermediate_size": 1792,
   "layer_norm_eps": 1e-12,
   "max_position_embeddings": 4096,
   "model_type": "bert",
-  "num_attention_heads": 8,
-  "num_hidden_layers": 12,
+  "num_attention_heads": 14,
+  "num_hidden_layers": 20,
   "pad_token_id": 0,
   "position_embedding_type": "absolute",
   "torch_dtype": "float32",

examples/in_silico_perturbation.ipynb

@@ -1,159 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e10ac0c9-40ce-41fb-b6fa-3d62b76f2e57",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from geneformer import InSilicoPerturber\n",
-    "from geneformer import InSilicoPerturberStats\n",
-    "from geneformer import EmbExtractor"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "cbd6851c-060e-4967-b816-e605ffe58b23",
-   "metadata": {
-    "tags": []
-   },
-   "source": [
-    "### in silico perturbation in deletion mode to determine genes whose deletion in the dilated cardiomyopathy (dcm) state significantly shifts the embedding towards non-failing (nf) state"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c53e98cd-c603-4878-82ba-db471181bb55",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# first obtain start, goal, and alt embedding positions\n",
-    "# this function was changed to be separate from perturb_data\n",
-    "# to avoid repeating calcuations when parallelizing perturb_data\n",
-    "cell_states_to_model={\"state_key\": \"disease\", \n",
-    "                      \"start_state\": \"dcm\", \n",
-    "                      \"goal_state\": \"nf\", \n",
-    "                      \"alt_states\": [\"hcm\"]}\n",
-    "\n",
-    "filter_data_dict={\"cell_type\":[\"Cardiomyocyte1\",\"Cardiomyocyte2\",\"Cardiomyocyte3\"]}\n",
-    "\n",
-    "# OF NOTE: token_dictionary_file must be set to the gc-30M token dictionary if using a 30M series model\n",
-    "# (otherwise the EmbExtractor will use the current default model dictionary)\n",
-    "# 30M token dictionary: https://huggingface.co/ctheodoris/Geneformer/blob/main/geneformer/gene_dictionaries_30m/token_dictionary_gc30M.pkl\n",
-    "embex = EmbExtractor(model_type=\"CellClassifier\", # if using previously fine-tuned cell classifier model\n",
-    "                     num_classes=3,\n",
-    "                     filter_data=filter_data_dict,\n",
-    "                     max_ncells=1000,\n",
-    "                     emb_layer=0,\n",
-    "                     summary_stat=\"exact_mean\",\n",
-    "                     forward_batch_size=256,\n",
-    "                     nproc=16)\n",
-    "\n",
-    "state_embs_dict = embex.get_state_embs(cell_states_to_model,\n",
-    "                                       \"../fine_tuned_models/gf-6L-30M-i2048_CellClassifier_cardiomyopathies_220224\", # example 30M fine-tuned model\n",
-    "                                       \"path/to/input_data\",\n",
-    "                                       \"path/to/output_directory\",\n",
-    "                                       \"output_prefix\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "981e1190-62da-4543-b7d3-6e2a2d6a6d56",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "# OF NOTE: token_dictionary_file must be set to the gc-30M token dictionary if using a 30M series model\n",
-    "# (otherwise the InSilicoPerturber will use the current default model dictionary)\n",
-    "# 30M token dictionary: https://huggingface.co/ctheodoris/Geneformer/blob/main/geneformer/gene_dictionaries_30m/token_dictionary_gc30M.pkl\n",
-    "isp = InSilicoPerturber(perturb_type=\"delete\",\n",
-    "                        perturb_rank_shift=None,\n",
-    "                        genes_to_perturb=\"all\",\n",
-    "                        combos=0,\n",
-    "                        anchor_gene=None,\n",
-    "                        model_type=\"CellClassifier\", # if using previously fine-tuned cell classifier model\n",
-    "                        num_classes=3,\n",
-    "                        emb_mode=\"cell\",\n",
-    "                        cell_emb_style=\"mean_pool\",\n",
-    "                        filter_data=filter_data_dict,\n",
-    "                        cell_states_to_model=cell_states_to_model,\n",
-    "                        state_embs_dict=state_embs_dict,\n",
-    "                        max_ncells=2000,\n",
-    "                        emb_layer=0,\n",
-    "                        forward_batch_size=400,\n",
-    "                        nproc=16)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "0525a663-871a-4ce0-a135-cc203817ffa9",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# outputs intermediate files from in silico perturbation\n",
-    "\n",
-    "isp.perturb_data(\"../fine_tuned_models/gf-6L-30M-i2048_CellClassifier_cardiomyopathies_220224\", # example 30M fine-tuned model\n",
-    "                 \"path/to/input_data\",\n",
-    "                 \"path/to/isp_output_directory\",\n",
-    "                 \"output_prefix\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "f8aadabb-516a-4dc0-b307-6de880e64e26",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# OF NOTE: token_dictionary_file must be set to the gc-30M token dictionary if using a 30M series model\n",
-    "# (otherwise the InSilicoPerturberStats will use the current default model dictionary)\n",
-    "# 30M token dictionary: https://huggingface.co/ctheodoris/Geneformer/blob/main/geneformer/gene_dictionaries_30m/token_dictionary_gc30M.pkl\n",
-    "ispstats = InSilicoPerturberStats(mode=\"goal_state_shift\",\n",
-    "                                  genes_perturbed=\"all\",\n",
-    "                                  combos=0,\n",
-    "                                  anchor_gene=None,\n",
-    "                                  cell_states_to_model=cell_states_to_model)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ffecfae6-e737-43e3-99e9-fa37ff46610b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# extracts data from intermediate files and processes stats to output in final .csv\n",
-    "ispstats.get_stats(\"path/to/isp_output_directory\", # this should be the directory \n",
-    "                   None,\n",
-    "                   \"path/to/isp_stats_output_directory\",\n",
-    "                   \"output_prefix\")"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.15"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

examples/multitask_cell_classification.ipynb

@@ -1,420 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "866f100c-e11a-4e7b-a37c-831775d845a7",
-   "metadata": {},
-   "source": [
-    "# Geneformer Multi-Task Cell Classifier Tutorial\n",
-    "\n",
-    "This tutorial demonstrates how to use the Geneformer Multi-Task Cell Classifier and optimizatize hyperparameter for fine-tuning"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "311ba456-b44d-40c7-941d-3fc03bcda85a",
-   "metadata": {},
-   "source": [
-    "## 1. Installation and Imports\n",
-    "\n",
-    "First import the necessary modules."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "cd9defdc-0524-4c3b-a741-27117ed3a5be",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from geneformer import MTLClassifier"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "790e9c3c-f6d9-44b3-b9a5-05725760f4fd",
-   "metadata": {},
-   "source": [
-    "## 2. Set up Paths and Parameters\n",
-    "\n",
-    "Now, let's set up the necessary paths and parameters for our classifier. We'll also define our task columns, which are specific columns from our dataset that represent the classification tasks we want to train the model on."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "04a04197-8e45-47f8-a86f-202209ea10ae",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Define paths\n",
-    "pretrained_path = \"/path/to/pretrained/Geneformer/model\" \n",
-    "# input data is tokenized rank value encodings generated by Geneformer tokenizer (see tokenizing_scRNAseq_data.ipynb)\n",
-    "train_path = \"/path/to/train/data.dataset\"\n",
-    "val_path = \"/path/to/val/data.dataset\"\n",
-    "test_path = \"/path/to/test/data.dataset\"\n",
-    "results_dir = \"/path/to/results/directory\"\n",
-    "model_save_path = \"/path/to/model/save/path\"\n",
-    "tensorboard_log_dir = \"/path/to/tensorboard/log/dir\"\n",
-    "\n",
-    "# Define tasks and hyperparameters\n",
-    "# task_columns should be a list of column names from your dataset\n",
-    "# Each column represents a specific classification task (e.g. cell type, disease state)\n",
-    "task_columns = [\"cell_type\", \"disease_state\"]  # Example task columns\n",
-    "\n",
-    "hyperparameters = {\n",
-    "    \"learning_rate\": {\"type\": \"float\", \"low\": 1e-5, \"high\": 1e-3, \"log\": True},\n",
-    "    \"warmup_ratio\": {\"type\": \"float\", \"low\": 0.005, \"high\": 0.01},\n",
-    "    \"weight_decay\": {\"type\": \"float\", \"low\": 0.01, \"high\": 0.1},\n",
-    "    \"dropout_rate\": {\"type\": \"float\", \"low\": 0.0, \"high\": 0.7},\n",
-    "    \"lr_scheduler_type\": {\"type\": \"categorical\", \"choices\": [\"cosine\"]},\n",
-    "    \"task_weights\": {\"type\": \"float\", \"low\": 0.1, \"high\": 2.0}\n",
-    "}"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "31857690-a739-435a-aefd-f171fafc1b78",
-   "metadata": {},
-   "source": [
-    "In the code above, we've defined `task_columns` as `[\"cell_type\", \"disease_state\"]`. This means our model will be trained to classify cells based on two tasks:\n",
-    "1. Identifying the cell type\n",
-    "2. Determining the disease state\n",
-    "3. Note: \"unique_cell_id\" is a required column in the dataset for logging and inference purposes\n",
-    "\n",
-    "These column names should correspond to actual columns in your dataset. Each column should contain the labels for that specific classification task.\n",
-    "\n",
-    "For example, your dataset might look something like this:\n",
-    "\n",
-    "    | unique_cell_id | input_ids | ... | cell_type | disease_state |\n",
-    "    |----------------|-----------|-----|-----------|---------------|\n",
-    "    | cell1          | ...       | ... | neuron    | healthy       |\n",
-    "    | cell2          | ...       | ... | astrocyte | diseased      |\n",
-    "    | ...            | ...       | ... | ...       | ...           |\n",
-    "The model will learn to predict classes within 'cell_type' and 'disease_state' "
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "b9e3050a-6162-4c01-b6fd-8784bf4ab1e4",
-   "metadata": {},
-   "source": [
-    "## 3. Initialize the MTLClassifier\n",
-    "\n",
-    "Now, let's create an instance of the MTLClassifier with our defined parameters and task columns."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e27caac9-670c-409d-9313-50201c665cb9",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "mc = MTLClassifier(\n",
-    "    task_columns=task_columns,  # Our defined classification tasks\n",
-    "    study_name=\"MTLClassifier_example\",\n",
-    "    pretrained_path=pretrained_path,\n",
-    "    train_path=train_path,\n",
-    "    val_path=val_path,\n",
-    "    test_path=test_path,\n",
-    "    model_save_path=model_save_path,\n",
-    "    results_dir=results_dir,\n",
-    "    tensorboard_log_dir=tensorboard_log_dir,\n",
-    "    hyperparameters=hyperparameters,\n",
-    "    n_trials=15,  # Number of trials for hyperparameter optimization (at least 50 suggested)\n",
-    "    epochs=1,    # Number of training epochs (1 suggested to prevent overfitting)\n",
-    "    batch_size=8, # Adjust based on available GPU memory\n",
-    "    seed=42\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "0d729444-e3ad-4584-9659-0c464ac97462",
-   "metadata": {},
-   "source": [
-    "## 4. Run Hyperparameter Optimization\n",
-    "\n",
-    "Now, let's run the Optuna study to optimize our hyperparameters for both classification tasks."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "9298aa3e-6a52-4aa8-b9ff-b63d97beac93",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "mc.run_optuna_study()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "af23075d-d07b-43d3-bc5d-4df4d5d7199b",
-   "metadata": {},
-   "source": [
-    "## 5. Evaluate the Model on Test Data\n",
-    "\n",
-    "After optimization, we can evaluate our model on the test dataset. This will provide performance metrics for both classification tasks. CSV containing following keys will be generated in specified results directiory \"Cell ID, task(1...n) True,task(1.,.n) Pred,task(1...n) Probabilities\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "461bf8d3-b964-4ff4-994f-9f3d313d4614",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "mc.load_and_evaluate_test_model()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "31cfeb2d-6673-4b02-a79c-2533cc5e4d28",
-   "metadata": {},
-   "source": [
-    "## 6. (Optional) Manual Hyperparameter Tuning\n",
-    "\n",
-    "If you prefer to set hyperparameters manually, you can use the following approach:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "8ee6b99f-42e9-4abf-a292-aa9047735e0e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "manual_hyperparameters = {\n",
-    "    \"learning_rate\": 0.001,\n",
-    "    \"warmup_ratio\": 0.01,\n",
-    "    \"weight_decay\": 0.1,\n",
-    "    \"dropout_rate\": 0.1,\n",
-    "    \"lr_scheduler_type\": \"cosine\",\n",
-    "    \"task_weights\": [1, 1],  # Weights for each task (cell_type, disease_state)\n",
-    "    \"max_layers_to_freeze\": 2\n",
-    "}\n",
-    "\n",
-    "mc_manual = MTLClassifier(\n",
-    "    task_columns=task_columns,\n",
-    "    study_name=\"mtl_manual\",\n",
-    "    pretrained_path=pretrained_path,\n",
-    "    train_path=train_path,\n",
-    "    val_path=val_path,\n",
-    "    test_path=test_path,\n",
-    "    model_save_path=model_save_path,\n",
-    "    results_dir=results_dir,\n",
-    "    tensorboard_log_dir=tensorboard_log_dir,\n",
-    "    manual_hyperparameters=manual_hyperparameters,\n",
-    "    use_manual_hyperparameters=True,\n",
-    "    epochs=10,\n",
-    "    batch_size=32,\n",
-    "    seed=42\n",
-    ")\n",
-    "\n",
-    "mc_manual.run_manual_tuning()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "dbaac008-fc00-4b71-8e78-89b2d922d9d8",
-   "metadata": {},
-   "source": [
-    "# Geneformer In Silico Perturber Tutorial (MTL Quantized)\n",
-    "This demonstrates how to use the Geneformer In Silico Perturber with a Multi-Task Learning (MTL) model in a quantized configuration to optimize runtime and memory."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "2e15ad57-736c-48f0-be87-39cf5015bc5c",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from geneformer import InSilicoPerturber, EmbExtractor, InSilicoPerturberStats"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "43c18140-151e-4d44-95b4-a9b3a47172cf",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Define paths\n",
-    "model_directory = \"/path/to/model/save/path\"\n",
-    "input_data_file = \"/path/to/input/data.dataset\"\n",
-    "output_directory = \"/path/to/output/directory\"\n",
-    "output_prefix = \"mtl_quantized_perturbation\"\n",
-    "\n",
-    "# Define parameters\n",
-    "perturb_type = \"delete\"  # or \"overexpress\"\n",
-    "\n",
-    "# Define cell states to model\n",
-    "cell_states_to_model = {\n",
-    "    \"state_key\": \"disease_state\", \n",
-    "    \"start_state\": \"disease\", \n",
-    "    \"goal_state\": \"control\"\n",
-    "}\n",
-    "\n",
-    "# Define filter data\n",
-    "filter_data_dict = {\n",
-    "    \"cell_type\": [\"Fibroblast\"]\n",
-    "}"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "3010d0bf-b23c-45c1-ac12-8c472dc8b7a1",
-   "metadata": {},
-   "source": [
-    "## 3. Extract State Embeddings\n",
-    "\n",
-    "Before we initialize the InSilicoPerturber, we need to extract the state embeddings using the EmbExtractor."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "215f0a90-8041-417d-a5d3-b2483626c3b2",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Initialize EmbExtractor\n",
-    "embex = EmbExtractor(\n",
-    "    filter_data_dict=filter_data_dict,\n",
-    "    max_ncells=1000, # Number of cells to extract embeddings for\n",
-    "    emb_layer=0,  # Use the second to last layer\n",
-    "    emb_mode = \"cls\",\n",
-    "    summary_stat=\"exact_mean\",\n",
-    "    forward_batch_size=8, # Adjust based on available GPU memory\n",
-    "    nproc=4\n",
-    ")\n",
-    "\n",
-    "# Extract state embeddings\n",
-    "state_embs_dict = embex.get_state_embs(\n",
-    "    cell_states_to_model,\n",
-    "    model_directory=model_directory,\n",
-    "    input_data_file=input_data_file,\n",
-    "    output_directory=output_directory,\n",
-    "    output_prefix=output_prefix\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "23f14e36-4529-4fb2-8af9-7f4875cf81e3",
-   "metadata": {},
-   "source": [
-    "## 4. Initialize the InSilicoPerturber\n",
-    "\n",
-    "Now that we have our state embeddings, let's create an instance of the InSilicoPerturber with MTL and quantized configurations."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "09f985a1-91bc-4e8d-8001-a3663531b570",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Initialize InSilicoPerturber\n",
-    "isp = InSilicoPerturber(\n",
-    "    perturb_type=perturb_type,\n",
-    "    genes_to_perturb=\"all\",  # Perturb all genes\n",
-    "    model_type=\"MTLCellClassifier-Quantized\",  # Use quantized MTL model\n",
-    "    emb_mode=\"cls\",  # Use CLS token embedding\n",
-    "    cell_states_to_model=cell_states_to_model,\n",
-    "    state_embs_dict=state_embs_dict,\n",
-    "    max_ncells=1000,  # Number of cells to perturb (larger number increases power)\n",
-    "    emb_layer=0,  \n",
-    "    forward_batch_size=8, # Adjust based on available GPU memory\n",
-    "    nproc=1\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "cfcc2c1e-fd7f-4a36-99fc-ac7f43e5be6b",
-   "metadata": {},
-   "source": [
-    "## 5. Run In Silico Perturbation\n",
-    "\n",
-    "Run the in silico perturbation on the dataset."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "cf030c09-8ae4-45a7-aaf7-3fc2af4fe296",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Run perturbation and output intermediate files\n",
-    "isp.perturb_data(\n",
-    "    model_directory=model_directory,\n",
-    "    input_data_file=input_data_file,\n",
-    "    output_directory=output_directory,\n",
-    "    output_prefix=output_prefix\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "bb8ec074-6f2f-422b-a973-37ed32a15c38",
-   "metadata": {},
-   "source": [
-    "## 6. Process Results with InSilicoPerturberStats\n",
-    "\n",
-    "After running the perturbation, we'll use InSilicoPerturberStats to process the intermediate files and generate the final statistics."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "0a748043-43fc-47ad-ace5-f0ae3dd34674",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Initialize InSilicoPerturberStats\n",
-    "ispstats = InSilicoPerturberStats(\n",
-    "    mode=\"goal_state_shift\",\n",
-    "    genes_perturbed=\"all\",\n",
-    "    combos=0,\n",
-    "    anchor_gene=None,\n",
-    "    cell_states_to_model=cell_states_to_model\n",
-    ")\n",
-    "\n",
-    "# Process stats and output final .csv\n",
-    "ispstats.get_stats(\n",
-    "    input_data_file,\n",
-    "    None,\n",
-    "    output_directory,\n",
-    "    output_prefix\n",
-    ")"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

examples/pretraining_new_model/obtain_nonzero_median_digests.ipynb

@@ -1,365 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "charged-worcester",
-   "metadata": {},
-   "source": [
-    "# Obtain non-zero median expression value of each gene across Genecorpus-30M"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "28e87f2a-a33e-4fe3-81af-ad4cd62fcc1b",
-   "metadata": {},
-   "source": [
-    "#### Upon request, we are providing the code that we used for obtaining the non-zero median expression value of each gene across the broad range of cell types represented in Genecorpus-30M that we use as a normalization factor to prioritize genes that uniquely distinguish cell state.\n",
-    "\n",
-    "#### Please read the important information below before using this code.\n",
-    "\n",
-    "#### If using Geneformer, to ensure consistency of the normalization factor used for each gene for all future datasets,  <ins>**users should use the Geneformer transcriptome tokenizer to tokenize their datasets and should not re-calculate this normalization factor for their individual dataset** </ins>. This code for re-calculating the normalization factor should only be used by users who are pretraining a new model from scratch with a new pretraining corpus other than Genecorpus-30M.\n",
-    "\n",
-    "#### It is critical that this calculation is performed on a large-scale pretraining corpus that has tens of millions of cells from a broad range of human tissues.  <ins>**The richness of variable cell states in the pretraining corpus is what allows this normalization factor to accomplish the goal of prioritizing genes that uniquely distinguish cell states.** </ins> This normalization factor for each gene is calculated once from the large-scale pretraining corpus and is used for all future datasets presented to the model. \n",
-    "\n",
-    "#### Of note, as discussed in the Methods, we only included droplet-based sequencing platforms in the pretraining corpus to assure expression value unit comparability for the calculation of this normalization factor. Users wishing to pretrain a new model from scratch with a new pretraining corpus should choose either droplet-based or plate-based platforms for calculating this normalization factor, or they should exercise caution that including both platforms may cause unintended effects on the results. Once the normalization factor is calculated however, data from any platform can be used with the model because the expression value units will be consistent within each individual cell.\n",
-    "\n",
-    "#### Please see the Methods in the manuscript for a description of the procedure enacted by this code, an excerpt of which is below for convenience:\n",
-    "\n",
-    "#### \"To accomplish this, we first calculated the non-zero median value of expression of each detected gene across all cells passing quality filtering from the entire Genecorpus-30M. We aggregated the transcript count distribution for each gene in a memory-efficient manner by scanning through chunks of .loom data using loompy, normalizing the gene transcript counts in each cell by the total transcript count of that cell to account for varying sequencing depth and updating the normalized count distribution of the gene within the t-digest data structure developed for accurate online accumulation of rank-based statistics. We then normalized the genes in each single-cell transcriptome by the non-zero median value of expression of that gene across Genecorpus-30M and ordered the genes by the rank of their normalized expression in that specific cell. Of note, we opted to use the non-zero median value of expression rather than include zeros in the distribution so as not to weight the value by tissue representation within Genecorpus-30M, assuming that a representative range of transcript values would be observed within the cells in which each gene was detected. This normalization factor for each gene is calculated once from the pretraining corpus and is used for all future datasets presented to the model. The provided tokenizer code includes this normalization procedure and should be used for tokenizing new datasets presented to Geneformer to ensure consistency of the normalization factor used for each gene.\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "textile-destruction",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "import numpy as np\n",
-    "import loompy as lp\n",
-    "import pandas as pd\n",
-    "import crick\n",
-    "import pickle\n",
-    "import math\n",
-    "from tqdm.notebook import tqdm"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "4af8cfef-05f2-47e0-b8d2-71ca025059c7",
-   "metadata": {
-    "tags": []
-   },
-   "source": [
-    "### The following code is an example of how the nonzero median expression values are obtained for a single input file. This calculation should be run as a script to be parallelized for all dataset files."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 30,
-   "id": "physical-intro",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "input_file = \"study1.loom\"\n",
-    "current_database = \"database1\"\n",
-    "\n",
-    "rootdir = f\"/path/to/{current_database}/data/\"\n",
-    "output_file = input_file.replace(\".loom\", \".gene_median_digest_dict.pickle\")\n",
-    "outdir = rootdir.replace(\"/data/\", \"/tdigest/\")\n",
-    "\n",
-    "with lp.connect(f\"{rootdir}{input_file}\") as data:\n",
-    "    # define coordinates of protein-coding or miRNA genes\n",
-    "    coding_miRNA_loc = np.where((data.ra.gene_type == \"protein_coding\") | (data.ra.gene_type == \"miRNA\"))[0]\n",
-    "    coding_miRNA_genes = data.ra[\"ensembl_id\"][coding_miRNA_loc]\n",
-    "    \n",
-    "    # initiate tdigests\n",
-    "    median_digests = [crick.tdigest.TDigest() for _ in range(len(coding_miRNA_loc))]\n",
-    "    \n",
-    "    # initiate progress meters\n",
-    "    progress = tqdm(total=len(coding_miRNA_loc))\n",
-    "    last_view_row = 0\n",
-    "    progress.update(0)\n",
-    "    \n",
-    "    for (ix, selection, view) in data.scan(items=coding_miRNA_loc, axis=0):\n",
-    "        # define coordinates of cells passing filter\n",
-    "        filter_passed_loc = np.where(view.ca.filter_pass == 1)[0]\n",
-    "        subview = view.view[:, filter_passed_loc]\n",
-    "        # normalize by total counts per cell and multiply by 10,000 to allocate bits to precision\n",
-    "        subview_norm_array = subview[:,:]/subview.ca.n_counts*10_000\n",
-    "         # if integer, convert to float to prevent error with filling with nan\n",
-    "        if np.issubdtype(subview_norm_array.dtype, np.integer):\n",
-    "            subview_norm_array = subview_norm_array.astype(np.float32)\n",
-    "        # mask zeroes from distribution tdigest by filling with nan\n",
-    "        nonzero_data = np.ma.masked_equal(subview_norm_array, 0.0).filled(np.nan)\n",
-    "        # update tdigests\n",
-    "        [median_digests[i+last_view_row].update(nonzero_data[i,:]) for i in range(nonzero_data.shape[0])]\n",
-    "        # update progress meters\n",
-    "        progress.update(view.shape[0])\n",
-    "        last_view_row = last_view_row + view.shape[0]\n",
-    "        \n",
-    "median_digest_dict = dict(zip(coding_miRNA_genes, median_digests))\n",
-    "with open(f\"{outdir}{output_file}\", \"wb\") as fp:\n",
-    "    pickle.dump(median_digest_dict, fp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "190a3754-aafa-4ccf-ba97-951c94ea3030",
-   "metadata": {
-    "tags": []
-   },
-   "source": [
-    "### After the above code is run as a script in parallel for all datasets to obtain the nonzero median tdigests for their contained genes, the following code can be run to merge the tdigests across all datasets."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "distributed-riding",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# merge new tdigests into total tdigest dict\n",
-    "def merge_digest(dict_key_ensembl_id, dict_value_tdigest, new_tdigest_dict):\n",
-    "    new_gene_tdigest = new_tdigest_dict.get(dict_key_ensembl_id)\n",
-    "    if new_gene_tdigest is not None:\n",
-    "        dict_value_tdigest.merge(new_gene_tdigest)\n",
-    "        return dict_value_tdigest\n",
-    "    elif new_gene_tdigest is None:\n",
-    "        return dict_value_tdigest"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "distinct-library",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# use tdigest1.merge(tdigest2) to merge tdigest1, tdigest2, ...tdigestn\n",
-    "# then, extract median by tdigest1.quantile(0.5)\n",
-    "\n",
-    "databases = [\"database1\", \"database2\", \"...databaseN\"]\n",
-    "\n",
-    "# obtain gene list\n",
-    "gene_info = pd.read_csv(\"/path/to/gene_info_table.csv\", index_col=0)\n",
-    "func_gene_list = [i for i in gene_info[(gene_info[\"gene_type\"] == \"protein_coding\") | (gene_info[\"gene_type\"] == \"miRNA\")][\"ensembl_id\"]]\n",
-    "\n",
-    "# initiate tdigests\n",
-    "median_digests = [crick.tdigest.TDigest() for _ in range(len(func_gene_list))]\n",
-    "total_tdigest_dict = dict(zip(func_gene_list, median_digests))\n",
-    "\n",
-    "# merge tdigests\n",
-    "for current_database in databases:\n",
-    "    rootdir = f\"/path/to/{current_database}/tdigest/\"\n",
-    "    \n",
-    "    for subdir, dirs, files in os.walk(rootdir):\t\n",
-    "        for file in files:\n",
-    "            if file.endswith(\".gene_median_digest_dict.pickle\"):\n",
-    "                with open(f\"{rootdir}{file}\", \"rb\") as fp:\n",
-    "                    tdigest_dict = pickle.load(fp)\n",
-    "                total_tdigest_dict = {k: merge_digest(k,v,tdigest_dict) for k, v in total_tdigest_dict.items()}\n",
-    "\n",
-    "# save dict of merged tdigests\n",
-    "with open(f\"/path/to/total_gene_tdigest_dict.pickle\", \"wb\") as fp:\n",
-    "    pickle.dump(total_tdigest_dict, fp)\n",
-    "\n",
-    "# extract medians and save dict\n",
-    "total_median_dict = {k: v.quantile(0.5) for k, v in total_tdigest_dict.items()}\n",
-    "with open(f\"/path/to/total_gene_median_dict.pickle\", \"wb\") as fp:\n",
-    "    pickle.dump(total_median_dict, fp)\n",
-    "\n",
-    "# save dict of only detected genes' medians    \n",
-    "detected_median_dict = {k: v for k, v in total_median_dict.items() if not math.isnan(v)}\n",
-    "with open(f\"/path/to/detected_gene_median_dict.pickle\", \"wb\") as fp:\n",
-    "    pickle.dump(detected_median_dict, fp)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "e8e17ad6-79ac-4f34-aa0c-1eaa1bace2e5",
-   "metadata": {
-    "tags": []
-   },
-   "source": [
-    "### The below code displays some characteristics of the genes detected in the pretraining corpus."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 38,
-   "id": "decent-switzerland",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "gene_detection_counts_dict = {k: v.size() for k, v in total_tdigest_dict.items()}"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 44,
-   "id": "polished-innocent",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/home1/ct68/miniconda3/lib/python3.8/site-packages/seaborn/distributions.py:2557: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).\n",
-      "  warnings.warn(msg, FutureWarning)\n"
-     ]
-    },
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAABR8AAAMRCAYAAABlG8GWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAABcSAAAXEgFnn9JSAAC/KUlEQVR4nOzdd5hjZ3X48e/Z7l2vK240G0wzBgOmmmp6NT/TQmjBlCS0ACGE3gklJARCLyGYGgi9hRqwgYBpxnRMMTZgsI1x2+Lt5/fHe8d7dUfSSBpdaWb2+3kePaN7dcs7M1ea0dF5z4nMRJIkSZIkSZLGbdm0ByBJkiRJkiRpaTL4KEmSJEmSJKkVBh8lSZIkSZIktcLgoyRJkiRJkqRWGHyUJEmSJEmS1AqDj5IkSZIkSZJaYfBRkiRJkiRJUisMPkqSJEmSJElqhcFHSZIkSZIkSa0w+ChJkiRJkiSpFQYfJUmSJEmSJLXC4KMkSZIkSZKkVhh8lCRJkiRJktQKg4+SJEmSJEmSWrFi2gOQJEmTERFXAY4GDgf2A1YDG4BLgPOA0zPzT9Ma3yAi4mTgkbVVj8rMk/tsfwTwm9qqczLziDbGJqm3iDib8toz4xqZefZ0RqM9XUQcD3yltsq/DZLUIoOPkiQtYRFxLHAScF863/j32v4c4AvAe4GvZWa2OkANLCJOAe4wzmNmZozzeJImw9cDSdJi4rRrSZL6iIjlEXFpRGR1e/uA271m0mNtjOeGEfFF4HvA3zFA4LFyOPDXwKnAryLiERHh/wuSJEmSRuKbCUmS+rsJsE9t+Ss9trtZY7tT2hpQP1E8AzgduEufTRO4mDLtuld24zWBdwPfGOsgJUmSJO0xnHYtSVJ/zWltp/TY7k61+7uAr7Yymj6qDMX/AB7V5eE/AB8DPgt8F7gwM3dW+60GrgPcFrgf5XtZXtv3ei0OW6M7DXjXtAchaUHw9UCStGAZfJQkqb/ja/fPzMw/9NiuHnz8QWZe3N6QenoDswOPG4FXAK/JzMu77ZSZW4EfVbc3R8SRwHMptSKtAbZwnZmZb5n2IKRB2Myjdb4eSJIWLKddS5LUQ5VJeNvaqq5TriNiFXDrubZrU0Q8Gnh8Y/X5wO0y8+W9Ao/dZOavM/PRlO/prDEOU5IkSdIexuCjJEm93RjYr7bcK6h4K2BtbfmUdobTXURcGWg2uLkEuG1mnjHqcTPzNErNyy+MPDhJkiRJezSDj5Ik9bZY6j0+n85mNwBPycxfzffAmXkZ8JfzPY4kSZKkPZM1HyVJ6u342v2fZOYFPbarBx+/n5mXtjekThFxILPrPH41M989rnNk5q5R9ouI/SlZoYcAB1G6av8J+A1wWmZuG9cY2xQRhwI3Ao4A9gVWAZcDlwG/BX49jkCveouIFcAtgBsABwJbKE2UvtfWzz4i1lGu3+sA+1M+WPhjZg7U1CMiDgeOpVz7BwKbgQuAnwI/zMxeXeYHHV8ARwLHAIdRPoCI6jwXAedQ6gCeN+Lx96mOfR1KBvhaYCuwCTgXOBv4aWZun8/3MR8RsS9wG+DawN7ApZTr4quZeeGYzrEGuD1wdeBgys/gt8C3MvO34zjHYlS7/q5L+dnsQ0lsuRi4kPLc/E1L574ecDRwJeAAYAfld/8r4EeZ+acxnWcl5TXgBpTXgMspz+Fv+ZovScMx+ChJUhdVvcfb1Vad0mO7vYBbzrVdix4GrG6se9OEx3CFiFgOPBL4a+DmdHbNrtsYEZ8BXpyZP5vU+AZVBbseW91uOsD2FwFfAz4MvH/UgO1CEBF3BT5H5wyZl2TmC4c4xq2BU+n8X/M1mfm0Hts3A3HXyMyzI2It8EzgiZQAXrd9Twf+KTM/NsT4TgLeWVt1amYeXz12beBFwAOY/dyCPh2FI2I/4O8p2cLX6TOE8yLivcArMvOiQcddnWN/4OnAwylBn7m2/y3wReA9mXnqANvfHXgycDfmfq+wJSK+A3wMOLlfo62IOBs4vLbqGpl5dp/tXwTUr7l3ZeZJ1WPXBP4JeCCwssvuGRFfAp6VmafP8T30Ov9VgJdRroO9e2zzDcpr2Bd6jPnFmfmiUc6/EFXX3v2Be1MCsl2fk7XtzwXeAbx+vsHgqhHaPwInAFfus2lGxA+BjwLvyMxzRzjXeuA5wOPoLL1S3+ZnwAsy88PDHl+S9kROu5Yk7bEiInvdgJ2UTIcZT+yx3WZKJtyMf+hz3ONb+Dbu21i+iBIImLiIuC0lq+sdlGyRXoFHKG/mHwz8KCJeXmXRLAhVxtrpwJsZIPBYOQD4f8B7mD0FflHJzC8CL2msfl4VlJpTRFwJ+CCdgatvUoKIA4uIawDfA15A/yDHscBHI+JDVZbayKrGTT8CHkr3wGO/ff+W0qDpBfQPPAIcSgkgnhURDxziHHcDfkkJjMwZeKxcHXgM8O9zHHtNRHyQEni+F4MlKayhfEjzb8wuU9GKiHgI8GPgIXQPPELJAL0r8K2I+KsRz/FzygcpXQOPlVsDn4+If68+sFqyIuIY4DzgP4D7MUfgsXIVyvPhVxFxnxHPuyYi3kT5ffwt/QOPUH73NwJeDHx6hPPdiPIa8Cx6BB4rRwEfiog3L/XfvSSNgy+UkiQtUlV23m0aq78xjenMEfEI4H/pHnRJyhTljV0eWw48G/ivhfAGLiIOoGQw3rDHJpspUwo3TWxQ0/FS4PO15WXAeyPiqv12qn6H7wPq210I/MWQ03OvRLmertdYv5Ey9bGbBwIfj4ihgoYzIuKRlMB5c/9LgJ5jj4jlEfF64C10fmAxYydlKurWLo/tC/x3RPzdAOO7DfApugd9EthA+Vl3O88gPgz8RY/HtgJ/pjyPp5bVGxEPp1xfe9VW76L8fLu97q0ATo6IOw5xjkcC76V70HHmXDsb65/M7KZfS81aOj9oq9tOuT66vcZDuc4/GREPHuaEVTO1rwKPp3cw/DLKtd/1EEOe7waUxnKHNx66jN6v+Y+jBFglSX1M/Z98SZI0sqMomUd135n0IKqMlnfR+cb0YuBVwHHAmszcNzPXUzJJHgB8o3GYB1Ma50zby4Cr1ZYTeDdlCur+mbkuMw/KzL0p3+/RlAyskyn1LJeEatr4w4Hf1VZfiRIo65VtBuV3eLf6oYCHZ+bvhxzCG4BrVPd/TalreqXMXJ+ZaykZVU+m1F+ruzvwyiHPBSU7cKZcwS7KlOw7AKszc39KQPIISjZU0yuBJzXW/Yoy/fr6wMrMPCAz11TneQKlHuOMAF4bEXeiv7fR+Ry7DHg5pbzBuszcp7o211ACZzenBG0+Se+AbRlACQrdu7H6a5Tp41fJzDWZeaXM3JcSBDoCuA/wauDMOcY9LjegZN0Fpebnayh1QFdVP9/V1TZvoDNAGsB/VCUh+qqy3t5O53ukXcBbKdncqzPzAMrv4UaU17iZYO+TgXuM/N0tHpcDn6Fc87cB9svMVdX1sR5YR8kIfTWdwciZ38ORg5ykKmnyGcp1XHcxJTP7lpTfx76ZuQ/ld3ITyvPrS8wOEM9lL8qsgZkPED5G+X2uq86xN+V15x8oH0jUPSci5sp2lqQ9Wsyz1rUkSYtWRDyux0PLgNexe9rwl4EPddluNfDa2vIngc/2OeUnM/MPQw6zp4i4H6WuVd39MvPj4zrHAGM4HPg+nRlfXwQekZnnz7Hv8ygZdjN2ATfLzO/32edkylTIGY/KzJP7bH8EpcHNjHMy84ge266iBLP2ra3+y8z8YK/jd9n/AcDHM7NvsGcUEXEKnVNbr6iB15aIuBUl86gecPz3zHxql23vQsmWrAduXpqZc2YFdan5OONTlN/B5h77HVidsz49fhdw28z8Zp/znURnzccZG4D7ZuYpc425Os59gU80Vv8r8Nx+GcgRsTclg69eNuEPwJGZuaXL9rcAvlVbdQlwq8wcKPBXZfTeOTO7vY4REf8D3LO26s3AEwdtihMRtwf+1K9+6xhqPs74DXCvzPx5n30fQfnQoO7/ZeYn++yzjDLN/8a11RuBe2fmV/vsdzQl2HVol4dbq/k46deDiLguJUD9jkGbqlV1Mz9JKY0w4z8z8zED7Hsyna/1AB+nvOZfMsD+R1Cey6/r8fjxlCzHps2UD0x6li+pMiT/j84SGz1r2kqSgMz05s2bN2/evNVuwM0oGVszt4f32O4Oje3uOeFxPqFx/gRuN+Ex/Gfj/F+lZKMMuv8bG/t/YI7tT25sf9Ic2x/R2P7sPtter7Htt6d9LTbGd0qX3/d8bicOeN6ndNn3AY1trkIJ3Na3+RKwbMBzdBvfDylZs3PteyXg/Ma+n5ljn5N6nPM+Q/w+llHqL9b3f9UQ+6+hBLvq+z+ux7aPG/U8A47lvNqxtwH7tnD9nt34Ho6YY/sXdfn9XAZce8DzfbKx77vn2P5eo14PlIDl9i77v2jcP8faOafyejDCOA+mTMmeOc8WShZ5v32OoXyIUB/fhwd9PRlwXMf3+Dn85YD7P62x32/b+l178+bN21K4Oe1akqTZmtMfv9xju+Nr93cCX29lNL2t77JuoIyUcajq/z28tmoH8NjMHKbm3HMpAYUZD6yy2abhgMbyr6YyigUmM/+d8sa/7j+jdIWeqT36AeCg2uN/AB6a8+v6/eTskgXYZXwXUq6juntUDWuG8enMHKZBxQOBa9WWfwU8b9Cdq+/tHxure2Vjt31t1o9/YQ6Y2TYFr8zMXw647dsay83pu03Nn/0nBr0eMvMMdk/bV01mXkCppTpjNWVadj/PprNe4x8pf1varjf6xcz8wIDbvpPyN2/G1SLikBbGJElLgsFHSZJmqwcfz8zeU6WPr93/Xmb2Knrflm6NNSbZCOVBdE7H/Vxm/mKYA2SZPve52qrllO6503BJY/kmC6EJzgLxaKD+u90H+HBVl+2VwG1rj+2gZA816zEO46c54NTnynvpDGIvY3YNw7k0g1VzeVhj+S05ZLOnzPwyJetwxjHVFOmmSxrLg3ZhH1T9+IfM1VhoSnYx3O+oWVf2Or2ez1UA/c6N1cMGE9885PZ7ktMay7fqtWFVU/a+jdWvywGmWo/BwL/DzLwYaJYZaDbIkiRVenUNkyRpj1S98akHUrpmPVYddetvoE5pcVi9dMswXDfB89+hsfy5rlvN7Xt0dtk9jlLba9LOpDQzmKlfeT3grRHxtCkElgdxGqXRz6jOGHTDzNwQEQ+k1B2c6TR8DKUj9XGNzZ+bmV+bx7hgdh3Fuca3JSK+QMlGnHErSvORgQ4BnDro+aogVjNIPur1/31211sMSiONZu3YZvDmMRFxBvDWMWWDnQacUN1fRgks/2X2qck4BT+uslwHkpkXRcSl7K7huoySLd4tq/MYSjfnGVvonfHe63w/j4izgGsOs98YTez1oK7qSH09SjOx9ZRyAs0u081mLFejt1vS+buA8uHCJAz8GlA5C7hhbXm/8Q1FkpYWg4+SJHW6BZ0BvK/02O5WdHaaPqWtAfWxscu6fbusa0sze+V6fZr49HNMY/mwEcczL5m5MyLeSmdH48cCD4qIj1A6r34tMxdKV+szM/MtkzpZZv4oIh5Pqbs5oxl4/BTwL2M43ekj7lMPPt5oiH3PyczL5t7sCtehs8kSwJ0iYpSs3Ss1lmdd/5l5ekScxu7n3HJKZt4zIuK/KYHPb2WPxjwDeCO7g49QAkC/jIjPUgLBp2Tmr0c89ricPcI+G+h8TdyH7sHHZsbajzNzR5ft5vJ9phd8nNjrQUTcjZL5e29glDIZzedOXTOr95zM/P0I5xjWZZl50ZD7ND+U2qfrVpIkg4+SJDXUp1wnvYOPx9fuT6PeI5Q6WE3dpmyOXZX5dVBj9ZPGdPhp1XwEeAlwezprku1LmXb8aICI+AWl0+nXgC9n5jmTHuS0ZOa7qgBbt261ZwOPzMwcw6lG+Zk29xnmOvrzkOfq1tm4a1fdEfQa9yMoU4nrz7sjgGdUtx0R8X3KtflVSsDw4kFOmJmfj4jXAH9fW72CEpA8ASAizqvO/zXg1OzTlb4ll4ywz87G8vIe2zWDYb1Kbcyl22vykhER16JMfb/jPA/VrV7xjObflUnV3r1khH0Gvb4kaY9n8FGStEeJiJtRuln3Us+cuojSAKXbdvev3f8T8LAe2/0hMz857DgH1C0T6RiGnLI6ov1pr3Z0c8rdxGTm5RFxZ+AVlG7iq7psdp3q9iiAiPg28HbgXZm5fVJjnaLnU7pFN99oP2bQYNcAhslCnNHMaOuXXdXULYu4nzYD5F2v/8z8VUTclFKXrls9yxWUpio3B54KbK+mor82M78010kz82kR8XPgZczOxoQScL1/dSMizgbeTanHN2zwdhTjCGr3sl9jedQyC6Nct4tCRNyA0sF+HE1V+v3taD63LhnD+QbR5vUlSXs8g4+SpD3NfYAXDrjtgQxWgP7QPtudCrQVfPwppe5jvfHMXB1dx6VbUG5cukZxJ6XqQvz3EfFq4K+AE4Fj6Z3Vcovq9oyIeEhmfm8iA52CKBH2t9D9Z/F4hqyT18cogYBJXjdTuf4z83fAfSLiWMq1eS/g2j02X0kJUt47Ij5HyUrt2wQoM98WEe8HHkxpKHVbeteRPQJ4AfDUiHhiZk6qLl8bmvVzR/39tnldTE1VC/kDzA48/gD4KPAdSubxecDlwNZ6LdKIOJ7eswjmYlBQkpYAg4+SJC1Smbk9Ir5B5xS420TEqmG77o6gW6bTUZn585bPOzFVnbGXAy+PiPWUenvHAbehBGWaGWrXBr4cEbfNzB9NdLCT84/M7kQ744ER8ZTM/PcxnGeU2qXNemvjysLspnn9n5+Z3aZityIzT6fUuHxqRBxGKRNwa8p1eVNmB4fvAXwpIm6dmX2zPKvH3wG8owo63YRy3d+WUpLg4MYu+wDviYgVmXnyvL6x6WleK/uNeJxR91voHgYcXVveATxqiIDz3kOcq9lUaJgMZknSAtXWdClJkjQZzazKA4D7tX3SKrjZnGJ4rbbPOy2ZuSEzv5iZL8nMu1N+zicAn29sug+Dd1heVKpajy9rrD6rsfwvEXHLMZzu8DHs0+ZU4GbToUOqAPXEZeYfM/MjmfkPmXlLSnDwr4GfNTa9ISV4PMyxt2fmtzPz3zPzQZQs71tS6v41G7K8NiIWa6CoWavxqBGPM+p+C939G8uvHDLTtVnHsZ/mc2vJ/l2RpD2JwUdJ0h4lM1+UmdHtBvxPbdMz+mz31dp2p/Xarrod3/K39D6gmeX4hJbPOaPZcOL4CZ136jJza2Z+OjPvQWd3bIDbR8TVpzGutkTEwZRpl/VZM6dSaozWO1OvBP47Iubb+OjYMezzg3mOoZ+fAVsa6+7Q4vkGlpkXZeZ/ULp9f6rx8CPmeeysgpF/S8m4rgcg96WzY/Zi8t3G8lUj4irDHCAiVgM3HtuIFpYbN5bfPeT+txhi2+bv4vCIuOqQ55MkLTAGHyVJAiJiOXC72qpTemy3hpL5M2PUOlZjkZl/At7VWH37iPircZ2j6mzdzRcbyw+IiD2xpMurmJ05daNpDKQN1e//fcCVa6vPBx6SmZsoTZouqT12deDd0aMD04D+35BjXAPcrbH6tHmcv6+qLmizw/2D2zrfKKrmR89orL7GuDI0M/PrwEcaqxfldV/Vwmx2VX7YkIe5H73rYy52zan2A3ejr/623muIc30H2NRY9/Ah9pckLUAGHyVJKm4O1N+U9woqHkdng5epBh8rL2Z2d9Z/j4h5T1eLiH2A/+rx8EeAXbXlI4DHzPeci01mJrPfjC+lIMQLgbvUlncBD83MPwJk5m+oOn/X3Bt45jzOef2IGCaT8OF01nzcBXxmHucfxH83lh8SEddv+ZzD+k2XdeO8NpvHX8zX/fsay0+NiIFqj1Yfujx3/ENaMJrZ9fsNse9DKR9IDKQKmn+8sfrvBv1dSJIWJoOPkiQV9aYtu+icWt1ru+3A/7U2ogFl5rnAPzRW7wd8PSJGzkSKiFtRptTevcd5f06Zilv3rxFxk3mcc2qdrkfN2qyacjQDvefNf0TTFxF3A57XWP2izOzoap2ZHwde3djunyLi9vM4/eurqaxzjfFKzK5F+fkqKNqmk4Gza8vLgQ9FxH6jHrDX9T+PjOJmMHQnjZp688xWbh5/MV/3b6O8ps84DPiPKnNvLv8C3KCVUS0Mv28sDzS9vio/8doRzvdKOrtcX5nSAMn3rpK0SPkCLklSUQ8qnpGZl/TY7vja/W9l5ubWRjSEzHw75c1z3SHA1yLi2RGx16DHiohrRsQ7KIHVI+fY/PnApbXlvYH/jYgHDHq+6pyHRcQLmV2jbpKeEBGfjYh7DPkm9xXAlWrLGylTBxe1qs7a++j8f/HzzA70zXgWncH45cAHqnqRo7ghJZjX89qNiAOBz9E5LTT7jHFsqgytpzdWX58S9B8qEBURx0TE2ylZzN28OyLeHhHHDHHMdcwO/HwtM3c21t0gIn4YEY+p9hn0+P8PuE9j9ULIBB9JZv6BEvSqeyDwyYi4Wrd9IuLAiDgZeGq1qlkHdKn4cmP5ZRHR929DlQX8VUpzrqFk5o+BdzZWPwD48BDZqEdExJOHPbckqR17Yl0mSZI6RMQq4Da1Vaf02G4vFlC9xy6eAOxFZ1OJ9cDLgSdFxEeBzwLfAy6cCUJU2WXXpvwM7g/cmRI4mlNmnhURD6ZMcZ3ZZ3/Km8TTgP+gvAH9dWbuqs4XlGDRDYGbUrJojqMEub430nc+HsuAe1S3P0XEJyi/4zOAX1UdvgGIiEOA2wNPqr7Wvb2qhdi260bE4+Z5jK9k5pnNlVU23AfpDKr+Hnj4zO+xKTN3VNfC99nd3fYw4P0Rcbde+/XwLcpz7QTgRxHxT8AnM/OianyHUQJDz2N2Pbo3ZOZEMpIz8yPV2OrZoUcDZ0TExyglC76RmVdkBFaZdIdTmvUcR6lved3q4Wb26Iy1wEOAx0bEmcDHgG9Srs3f155by4BrUK7hpwHXbBznNT2Of0PKc/X1EfF5SpD5dOAn9Wu5qhd5c+CvKK8z9cD09+idMb5YvBS4J3Cz2rp7Ab+KiC9RmqFcSHmNuxElK3wmYPt7SimKp9T2rWfvta211wPgLcDj2f37PgT4bkS8DPjvzPwtXPG6cTPKVOu/BVZV25/C8A3JnkRpInXj2rr7AXeIiNcDnwZ+UH0IMJOBfn3Kc+r+wJ2AHwOvG/K8kqQWGHyUJKkEOdbWlnsFFW/N7jdT/babiszcGRGPBH4OvITOAOKVKW/mnjSzeURcXG2znv6zIZpdrZvn/XwVdHonnXUzb1XdAHZFxKXVeeY630JwEPDY6gZARGwGNlOCDb2y8b7L5Gq/1X++o3oU0C3Y8M+U633GDuDBmXlhv4Nl5rkR8TBKNuLM7/jOlLqRLxxiXE+i1FS8BiX79p0AEbGBcs2u7bHfl5jdZKVtL6CM6VnAzLTp5ZTg6AMBImIHJUN4DfOvi3hdOjusZ/Vz2U6pe7myx35vzMxPznHsvYATqxsAEbENuIxS67ZXs5o/UwLTkwy2jV1mbo+IuwNfoHwwMmMVJQjZq3HKRZRA+f0a6yeZCdna60Fm/jgiXkNneY/9KNPN/yUiNgFbKUHZZumAzwP/ypDBx8y8PCLuTcmGr3eyP4Da60n1dyUo1+bUynZIkvpb6P/4S5I0CfUp1zuBrw2w3VZK5tGCksXLKW+c+wVHg/Imbl96/z/wE+ABmXnHHo/Xz/sR4Bb07jC8jPLGtN/5dgE/mOtcLZorcLKWkgnYK/D438AdM/PysY5qwiLiRErWXN2zMvMbg+yfmV+kZJDVPa+qHzmoCylBy2YgZD29A48fA+5bdaKemOo59xxKBuNve2y2AjiQ/oHHzXQPBEP/azMoQccD6R543Ao8PzOf1OWxuY4NJfB2JXoHHs8AjqtqwC56VXbtHSkZc80p6t18i/L9n0Fn0yPo7AK/2D0DeEePx9ZR/p40g38fomQh7hjlhNVU+NtRPnzolTm9L+Xn3i3wOEy2tSSpRQYfJUmaXe/x0h7bHV+7f9qkgxzDyMwfZOadKFMk3wj8bsBdzwbeRHkzfYPM/OgQ5/x5Zh5HCRp9nM5akL1spmTGPB04PDOn2S37DcBtKTUcv0kJ2sxlM+UN9h0y88GZubHF8bUuIq5JaaRS94nM7DUduJeXULIQZywD3hsRVxn0AFXDmGOrY/25z6Y/AB6YmfefZuA3Mz9FaTz0KODrzO4Q3M0FlOntJwGHVrVbu3kYJbj5VuCnDDad93zKNX39zPynPuP+AXAU8I+U5+IlAxx7F2Uq7SOBm2bmLwfYZ9HIzA2Z+RTKNN4XUj5U+QMlu3QTJbv8XZRMyOMy8xfVrs0SABdPZsTty8xdmflY4EHM/SHRtykfXP3FfOsiZ+bmzHw0pTTAe+j/WgDl2vwW5Xoe5gMPSVKLYpHPjpAkSQOqGojcALg6ZcrcKkpzlIuBPwLfy8y53tgNc77llHpd16JkZe1PCchspLyRP5NSC3J7r2NMU1UL9HqUab9XpmR+LaeM/8+UINBPMnOQIKX6iIjmP6TXyMyza4+voGTW3pByLW2hXEPfW6iBr1qN2KtSxryeEqy+DDiHcv3/bpSpylXTjetRajoeTMk8S2AD5bn8I0qd0qEzv6qarNeqblejZJatrsZ+KfAL4Id9PqTZY0XELyk/txk3rJqnLDkRcS3K9X0oJRt8I+W6/nZmntvieZdR/q5ch5KRux9wOeXv2C+BH/VpGCdJmhKDj5IkSZqquYKP0kJXdTj/UW3VRmDfUQLAkiQtNU67liRJkqT5eX5j+csGHiVJKgw+SpIkSRIQEatH2OepwF80Vr9pLAOSJGkJMPgoSZIkScXLI+JjEXGPqu5rTxFxREScDLym8dC3gS+0NUBJkhabFdMegCRJkiQtEMuBE6vbhoj4FqWW4/mUOo57A4dRmq3cvNq+bgPw0FEaCUmStFQZfJQkSZKk2dYDd6lugzgPuH9m/rq9IUmStPg47VqSJEmSirOArUPusx14F3CzzPzm+IckSdLiFs4IkCRJ0jRFRPMf0mtk5tnTGIsUEeuBuwG3Bm4EHA4cBKwFErgE+DPwQ+BrwCcz83dTGawkSYuAwUdJkiRJkiRJrXDatSRJkiRJkqRWGHyUJEmSJEmS1AqDj5IkSZIkSZJaYfBRkiRJkiRJUisMPkqSJEmSJElqhcFHSZIkSZIkSa1YMe0BSJKk3iLiFOAOtVV3zMxTpjOa6YqIVcAxwDWBw4B1wE7gEuBi4EzgJ5m5Y1pjnISIyPpyZsYc258MPLK26lGZefL4RyZNX0QcD3ylturUzDx+KoPRghAR1wKuC1wN2AdYBWxk99+OnwC/zszsdQxJ0vwYfJSkJaRLkKGXncBllH+8fwl8G/hcZv5fa4OTRhARa4GHAX8B3A5YPccul0fEd4H/Bj6YmX9qeYiSFrkh/3ZuAC4FzgFOB04FPpOZ20c894uAF3Z56NOZecKIx2wG0e6amV8a5Vi1Y34CuG9j9ecy857zOW4bImIZcB/gwcA9gAMG2G1DRJwOfBL4SGae0+IQJWmP47RrSdozLQf2B64B3A14HvD1iPhBRNx7qiNrSUTcOCJeVLudNO0xqbeIWBkRTwd+B7wNuAtzBx4B9qIEKV8P/CEi3hURh7c3Ukl7kOXAfsDhwO2BpwIfA34fEU+PiOVjPNd9IuLWYzzeyCLiYOBeXR66W0RcZdLj6SciHgL8CvgE8FAGCzwCrKfMMng1cHZEfDUi7t7OKCVpz2PwUZJUdwzw6Yh47bQH0oIbU7JLZm4nTXMw6i0ijqRk4/4L/d84bgH+DGzt8fgK4K+AMyPi/411kJK028GU16tTI2L9GI/7ijEeaz4eQfcZc8sor7FTFxEHRcRngfdTPljtJSmzPi6lZLL2cjvgcxHxubENUpL2YE67lqSl7ZfAv3VZv4IS1LkJJaNs78bjT4mIyzLzBS2PT+oQETcDPs/soGMCXwI+U339bWZuqO13KOV6vitlqt2Va/uupv+bUUmq6/W3cybz8SjgzsChjcdvA3w8Iu6ambvGMI7bR8Q9MnPaAbCT+jz2KKYcJK0+sPoi3V/nzwA+DXwZ+DFwUWburPZbARwB3ILyv9D9KL/fuhu0MWZJ2tMYfJSkpe0PmfmWfhtExAGUNw5/03joeRHxscz8fmuj05z2pEYJEXFd4AuUkgB1XwWenpnf6bVvZp4HfBb4bEQ8A/hL4KWUN5Z7tMw8CTN9tYeoGnL1bcI0gEH+dq4CHgf8M7Cm9tCdgIcD757nGGa8PCI+P61mKBFxczoDcDPjmPkZXzsibjOtmtHVB09fBq7eeOhHwHMz81O99q2ak/2qur0/Ip5AyfJ8Nn5gJUlj5bRrSdrDZeZFmfm3lCljdUH3IvjS2EXEXpQaXc3A45spHb57Bh6bMnNHZr6Xkp30Gna/WZakscjMbZn5OkpdwaZnz+PQ59P5mnUT4EHzON58Paqx/GU6u4l322YiqsYyH2Z24PEjwC36BR67ycwtmfl2yt+O5wHbxjJQSZLBR0nSFZ4D/Kax7h5VUEhq20uB6zbWvTEznzDq9MXqjeTTgAcCm+c7QElqysyPUbKu664XEc2A2KB+BXygse6lY25mM5CIWAM8pLH6XdWt7i8iYt1kRtXhKZSp7nX/DfxFZm4Z9aCZuTUzXwbcitn/F0mSRmDwUZIEXDH96B2N1auBBdFtU0tX9Sb97xqrfwY8fRzHz8yPAv8xjmNJUhf/3WXdLedxvBcAO2rL12E62YUn0lkDcSPwUUq24Yba+vXAAyY2KiAi9qVkJ9adCzxuTPU2qcrO3GMcx5KkPZ01HyVJdd/osu7wUQ4UEYdQ3nxdg/LGZAvwg8z84gD7XplSAP5g4EBgE/An4BfA6dOqfTWXqvbUzSnjPojShflPlOYF350pcj9tVTbrbYDrAftS3kSeD/xfZv5+CkN6CrCqse6J88lcaRrlzegkr8OIuAbl2rkysBa4CPgpcFpmLripfxGxkpIVdAPKVPnLgQuAb2Xmr8Z0jqD8/K8DHFatPh84IzN/MI5zjFPV5fjWlN/hQZROun8Cfgd8c5zXc+O8y4GbAkdTrtUVlC7wH8nMP7Vxzi5jWEv53q9LCVZdDpwFfC0z/zzA/uuB46r996F0Iz4H+Epmbmpn1GP1oy7rDh71YJn5q4h4B/C3tdUvjIj3tnUd9dAMeH545vcRER9uPP4oxlfnchCPZnZjsmdk5sXjPMl8r78qI/Q4dr8uLKe8Lvye8je3laz8iDiI8nf+msBelL8pvwO+mpmXjfE8AdyI8jp9EOV/iospfw++nZm/G9e5JC1ymenNmzdv3pbIDTiZUitq5nbKkPsf1dg/Kf/Mz3WeF9UeuytwCrCry7F6jgdYCTwJ+GGX/eq3C4C3AFcb4Ps5fo5jzXU7YoBz7AU8jdJRs9v3PHP7M/C2QcbdOP4pjeMcP8f2J/X6mQOHUGoobuozztOAO03wmp15I1Yfw0+n+Bwa+3U4x/nuDXy7z3kuBV4N7Ffbp2ObAc5xcmOfk4Z83pxde2w9pUHVxX3G/FPggfP4mewFvAj4Q59znAU8AVjWY8ynjHr+EcZ7L0oNvG19xrsZ+CRwyyGPfUSv3zflTf4rgQt7nPP4MX1//a6HQ6rnweYeY9hKyTq+Uo9jX4MSsLq8x/6XA68F9p3HeOe8Fro8R4a6foBrdxn7cwfc90WN/b5erb9yl5/r0wY8ZnMsdxnh935VSgC96zUF3KHx2C7gmhN83jVfoy8AVk3q/AOM736U+phb+7wubAE+M8LrQvN6fVHtsZtQuns3f3czt+2UTN1rz/P7uxZltsz5fb6/mb8HTwBWTvt34s2bt+nenHYtSarr1iE0B9oxYkVEvJnSrfgOPY7Va99bAmcCrwduOMfmB1GyQX4REc8a9BxtiIgTKfW5Xk355L/f93wA8NeUcTenGLcuIu5EeRPwOEpmXS+3BP43Ip47kYGVjJArNdY1p/9PxCSvw4hYHRHvo7xJvHmfTfehBLd/FBE3GvY841Sd/0fAs+icitl0FPChiHhz1RBimHMcA/yE0uzqsD6bXgN4I/CViGhmP01ERBwcEV+mBA+OpwSue9kLOAE4LSLeX2UKzufct6A8n59JycqduIi4QzWGv6V8f92sAh4DfCcirtXY/4GUANIj6OwWXbeGkhn9zSqzfKHap8u6eWXMZeYfgDc0Vj+7yhKdhJPoLNF1DnBqbfmrdNZDDOCR7Q8LIuJwZr9GvzcXQJZ4RBwVEd+lTE+/I7Oz+utWUz68OC0i/rPqoD6fc/8D8B3Kh1q9XntXUBoYfT8i7jbCOVZFxOsppVEezdwZvkdRXqt/EhHXH/Z8kpYOg4/SFETEdSLiARHxxIh4dkQ8NiJOiIjrD/tGTRqzbm/uLhxw37dRAlt1OykZUj2nvEbECZTsgGv02OQSOmtfzVgDvCIi/mMaz5sq4PRRSnZK0y7KuLtNp1oDvC4imt3FW1MFHv+H2VPULqFkXnTzTxFxUovDmnGHLutOmcB5O0zyOqzeYH6E7l1yoWR7bWysuyrwpYg4cpBzjFtE3ICS3Xd446HL6B1keRyldt2g57gx/X8Hl1KyiOpuT7m2ewWvWhER16ZkCd+xxyYbKb/Hbh5CCZo2g+6DnvuGwBeZ/dqziXkGvIYYwy2Y/ZqyizK1s9vz5AjgMzNB1yrw+AFg79o2/f5eHAV8bAH/j9Ttg4HfdFk3rFdSrvsZVwL+YQzHHcRJjeV3Z+YVH0ZW95vTrB9ZTcNtW7e/G1+dwHn7qoJ536SUQehmI52/z7pHAV8c9YOJ6gPDf6XMJpixk97PyXXAJyLiekOc40DgS5TZAd3Kt23rc75rA9+IiGaDIEl7iIX6B1wLUEQsi4ijI+KREfH6iPhmRGyOiKzdjp/2OBeqiFgZEX8fET+iZNZ8mPKJ9suBt1OmY/0E+HNE/JcdhjUl3ZrLnDPAfg9kd+2nDcCLKbXgVmXmAZTAwE1ovFGJiKMob0Cb/2x/ilLkfU1m7k/JHDiK0hG5+Yb+McCze4zrF8Djq1vzTdIva4/1unWtVRYRT6RMPa2/yfoD8Hzg2Or73j8z11GyAv4K+HHjME+PiElkiRxKmWK1mvJG5D8pb9xWV2PcCziS8rNtBiJfExH7tzy+YxvLWynZUBMzgeuw6RWUzJS631OCdYdm5trMXE/JaHsk5W8GlMDDewc8xzjtBXyMUtuR6v49gHWZuW9m7g1chRIUuaSx73Mi4jpznaCqi/ZRZmfxfZGSMbguM/fLzDXA1YEnA+dV29ySkik5EdXf548zO0j6Y0oW3wGZuT4z11KyNx9PqbVWdwvgfSMGav6L3Zl2pwL3p0xL3rv6XRxICRz9cYRjD2Iv4IOU58sOyrTrW1Je9w6kvN7fjhKsrrsO8Mwq2HEyJUiymRJgO4YyLXPm78U9gWZdz1tRMq0Wogc3lndSgtPzkpkXUQJKdU8bNXA9qIi4PeXvQl23eo7vonN2xOHAndoaV82Nu6z77gTO21NEHEv5X37f2urNlOfH8ZTXsPWZuR8l8HcPStZ03e0pWYLDuhvl7xKU4OaL2P2cOpDyt+vmlNeOujXAWwc5QVXn95OU53bdqcDDgKtk5ura+W5EeY9Tb0y0L/Dhqia4pD3NtOd9e1scN0qGxkb61/QYW32hpXajfAL64wF+fvVb1/pI3rz1uzGPulWUT7HPauy/BdhrgPPUa/tcdcDzLaPUSazvvxN49Bz7XZsSEG3WMLrZHPudNOrPpnGcmzG7htN7gfUD/Hzf0thvE3DYHPudMszrbJfvc+b2J+A2c+x7x+pnWd/vyS1fsz9onO+MCT9nJn0d3orZtbi+0O/6oQSO39fj95oDfI/N5+tJc2x/fI9zbQLuN8e+N6C8+a3v928DjPHfu5yvb307yhvZr/UY6yktXjOv73K+t9GnphmlVubnu+z31DnOdUSv3zvwDxN6jvS6Hi4GbttnvxWU4Ep9nwuBr1f3zwau12f/vYHvN/b//gjjnfNa6PIcGfj6oXz41vzZfHqI/V/U2PfrXX4Ozbp6fZ9TXcYzVM1H4J39xtTY9tTGtu+dwDX5iea1OInnQp/x7Mvs/5++Bxw5wL6PogTx6/ved8jrtX7Ouf6neH6X/Y4ZYJyvbuyzGXjoAPtdi/IBWn3fj07z9+XNm7fp3Mx81KBuSvmUTkOKiOMo08iOrq3+HSUI8Q+Ufzr+jvLG6zT6TE+VWvYyZmfyfDYze00dbLoUuGsO3i35RGZPVXtmZv5nv50y85fAXeic0rwCmFSNwlfRWcPpg8AjMnNDj+0ByMwdlAyoT9dWr6XUM2vbDkrQ6P/6bZSZX6E0pKl7YGujKpp1/ebsjDtmJzLZ6/D5dM48+TlwYr/rJzO3UjIgpz2t8DGZ+bF+G2TmjymZz3V9r6GqZuPfNFb/a2b+2xznupRSL+3sftuNU1V3sDnWTwF/m5nbe+1X/X5PZHZW77MiYvUIQ/nXzHz1CPuN00Mz8+u9Hqxe855A5/81B1I68G4FTsjMn/fZfyOzXx9vPK3SA01V7bunMDsbeRuDZ0HPqfo5vKyx+gkRcbVxnaMuIvZm9nP2XX12ObmxfP+I2Hesg5qtWXLg4pbPN5en0Pn/05mUgO+v59oxM9/J7OvlOSOM4ffA3TNzroznlzE7q3iu1+hr0vlcTOBBmfn+uQaVmb+ivE7XO2yfWM04kLQHMfioUWylFDN+C9OZ/rVoVMXVP8/u6VGXUQqzH5GZj8/Mf8vMkzPzDZn51Mw8jvIP1XMp/7xKrYuI/SPiLcAzGg8ls4MI/bw0M88dYvvmm8ofAK8ZZMcq8PNPjdX3jYheteLGoqpzVq/xdjHwxMzMQfavtnsanW/G/3oCdcze2S9I0PC2xvKxLY+v2TyhVz2stkzsOoyIIyhT7er+LjO71QZtnmsmeD2tD6i+mJkfGHDbd9JZ8+tqc0yzO4nOmo1/ZMBp1FVQ72kDjmscHk/nhw+XA08Y5DWg+iCnWRf3EOAvhxzDnxiilmZLPpmZn51ro8w8h5Lp2PTmzPzRAPt/FfhtY/XNBhvivFw5Ih7X5fbEiHhORLynGtdrKZnJM3ZSgvRzfm9DegudJVBW016pgQfRWYtzC6V0Ry8forPW6F7MnoY+bns3li9p+Xw9VTUam03kHp+ZwwREX0PJnJxxy6oG7jCemZlz1ujOzF2UDvR1/ZqeATydzlqS78nM5pTxfuf8NSXJYkZQ3g9J2oMYfNSg3k35pP+mlKlht8jMxwP/O91hLVxVHad3sPuN9Qbgbpn5tuoPf1eZeX5mvjwzL+u1jTSEfm+gnh8RH6W8ger2T+DLMvOMAc+zndnZDz1FxD7AbRurX5+ZOwc9BiVDr16jcBmzAzvj9rDG8vsyc6hMvSpgVa9NdQClNlObmtmMPVWZa/XXn3VAKxk2lWbW10QaZsBUrsP70Pm/15mZ+aVBT5SZP6Vk0k/DMNfQxZROqHX9mhrcvbF88iAB2ZpPAsN88DEf92osf2SIbG8y85vAt+Y45lzeM0RGelvePsS23+myrhn86KdZy2/gBhnzcG3KNd+8vYGSOfZwSuC47kxK9v/YP5jP0sX5RY3VJ0XEdcd9LnbXb57x8SrLuKsqM/Ojcxxj3Ob9dyMift+oW9/vdkqfQ92dUo93xo+rWQQDqz5c+nBj9fFDHOIi+geIm77RWO75nKo+fGx+QPK6Ic41o1lv8vgRjiFpETP4qIFk5gsy8+2ZeXq/aUXjEMVNI+IREfEPEfH06v7Rc++9oDycUjh6xjMzs/mGQ2pbvzdQLwHux+wMAoB/z8znD3GeHw4ZhLsVnX+DktlvXvrKzEsozSjq2u6ieIfG8udGPM73GsvHjXicQVzC7ClWc/lNY3m/sYyku2b34kmW+Jj0ddj8PX98mHNVhhrfGJ065PZnNZb367ZR9UHdLRqr/2eYE1XB4s8Ps88oqgynGzdWf2SEQzWDBMO+bg0V2GhB0j2bsZdm5uJFlPrAo+6/3xD7TkJSyikcPWzQaUjvoTOov5zdTUbGopqx02wo0m/Kda9tbtXytNrm7KBploZaCP8XfL0KYA5qoNfnyjHsbjYGcGFmNsc6p8z8GZ2N2m5YTfGXtIdYMe0BSDMiYj3wTOCxzP40eWabXwIvzMzmp2cL0ZNq93/FgN3kpCn7EfDsYabT1PYbxg0by78ecorSjO9SOuHOaC2DsAo8NMd98xHrbjWn5TbrHo7Tb/tlW/fQrD+4T9etxmMjnVNu264VVjfp67BZW3LoN3Aj7jNfl2XpujuMQa+hw+h845uUBkDD+v4I+wzrKGb/79wtq28uzUy+q0TEAUP8jMc9pXdYl1ZB90E1s9J+O2ipisrGxnKbr0ejCEqJkjXA89o6SWbujIjn0RnwfmBEHJuZp4/pNCc1lv/I7A9XuvkyJUh89caxnjmWUc3WvCYm+Xej6VaN5atERLO8wiCawdph/i84e8hzDfM3vvn9bRzx+4MSNN6rur8MOJjZv0tJS5TBRy0IEXErSgZIv5pQULK43h8R9wMe1nYW5qgi4hg6MzneMcKbf6lNOylTay8Bfgl8G/jcXA1J+hi2SciBjeVmpt2gmsXcm8cdp4OZPWNgXDW32hz3JSPs05x2vLzbRhFxR2CYaX+fzMw/NNb9kc4pawcMcbz5mvR12Fx/9gjnGmWf+bpkhH0GuobozKgB2FBN4xzWXE0WxqH5+9s+ZJ3bGd2aUBxIyQgcxKSbMjUNWxameS3Md/9e19I4nZqZx9dXVFm6ewNHAnej1IudaXyyDHhuRKzKzGb95LHJzI9GxHfYXaMvgJczhpIj1fTav2qsfu8gZSgyM6s6mPWGW4+IiOcMWcZiUH+glIKaMcrfjWfRfeYHlAZfzaBbL4c2lh9S3eZrmP8LLhnmwFUgu76q32zI5vd3BEOU4ZjDgczOwpS0RBl81NRVb14/Ten6OuPMat2vKUXrrwv8Bbvrjj2Ikh3RdkHrUd2tsTzUFDJpjGa9gWrJsMGCZsBh1CYjzf3aDFy1GSBcO/cmIxsmw2hYj6xug/o55U1j3a/pzEA8KiJWTujDpUlfh83zjVLbd9INeaDda2i/xnLfrvF9TKJOclvXCwzx2jVicHac5ns9tHk9tabK1txAycw9IyLeTCmDcJfaZv8YEd/NzGHq7w3rOXRmI949Iu6QmcOWRmi6C7Pr+w4y5XrGyXQGHw+jBEWHnUkxiFnThiPiKsN8GNCvNmeVFDFo8LGt/w2G+b+gzefUYv3fR9ICY81HTVVEHEwpQDzzx2cL8BjgqMx8ema+uao1+XRKALI+dfkvIuIRkx3xwOpZjxuAHwNExLER8YaI+ElEXBYRGyPiNxHxsYj4m4jYq/vhpCUnGsvj+se5zX/AV829yciaP489SXO64GpmT4duy7Svw0UZhBmzZs3PUZ9nbT4/Z7R1vYz7WJqAqtP6/Zldv/JNEXGlLruM67xfYnbjqVeM4dCP7rLux4M2ZaHMomhqq/HMGV3WzdWxuS1tvfYslP8L/N9H0lgYfNS0vZLdU613AffLzP/sVgsoMy/PzMfRWevmpdU0kYXmJrX7vwTWRMQbKXWenghcn9IFex1l+sKJlMDqWRFx/4mOVJqO5vTC/UY8TrPO0yj1+gbVnOqYwLrMjDHcTmpx3Atdt2ydO07o3JO+DpvrR6lTNs3aZm2Y9TOJxnzAAe03hrHMpXm9jPq76LZfm69dakkVgHwU5X/YGQcy5kYwXTy7sXxcRJzQdcsBRMR+wP+b14i6OyEi2sic+2qXdbfvsm4Smv8b3HNM/xccMY1vpovm9/fBMX1/kZmnTOMbkjQdCzFooz1ERBwKPKy26j8yc5AOcU8GZqbjHQ7ca9xjG4ODavcvAD4EPIHdn/BtA37P7BothwIfjointD1Aacqa/8weMeJxrjnHccfpT43l6HL+PUpmnjSGNxrfYPbv7THtjx66nPeIEY8z6HU4jvONss9Cdj6dWX+rGO151WZn3RnN39+qiLhy1y376/b9TbuOo0aUmd+mdKKue2zVObrNc368sfpl8/hA/qF0Nv4al1V0/q8/Fpl5FrMzTh8eEZPIgG5q/m/Q2u99Spb69ydpQgw+apoeSGcq/2sG2alqVvCl2qq7jnNQ81VlbKyvrbozuwOkZwL3A/bJzKtl5v7A9YB31g8B/FtE3HkS45WmpNmt9VpV5sWwbtZY/sFow5lb1QX5nMbq49s6354iM3cw+437UVU94LZN+jpsrr9p1636G2WfBauqX/jzxupBa63Nd59h/YxSh7qu+bsfRHOf34/QTVwLywspHyzPWAE8v+VzPo/OjMsbMnqjk+b06C8Ajx/x9ok5jj0u72wsH0SpCT9p328sHz+FMbSp+f3daMS/k5L2cDac0TTdrnb/rMxsvvno59vAPav7t+y1UURcdZSBDejSarpN0zo6A/srq6+nA3fKzI5C85l5JvDoiPgZ8Kpq9TLgtRFxTLcp6NIScBrlTdPMcyUogfnmm4meImJfZn/48I0+uzSDBqN0TP0i8Nja8oOBN4xwHHX6d0p2eP0DqTdGxLGZuWUcJ4iIZZm5q7F60tfhNykZRjNOpHRcHcZSLM3xf3RmLj4MeN+gO1fZh8ePeUyzZObmiDiDzuDhA4BPDnmov2gs93vd0iKQmedExLuAv66tflhEvDQzf9XSOX8SEe+ls0P1SyJiqGY3EXEDZgfE/zkzm3UlBz3ed+icwn3jiLhxZp4xyvH6eAelwc1+tXWviojPTjiY/0U6G6/dIyL2bf6/v4h9A9hEeX8DJX7wAMrPX5IGZuajpulGtfs/GXLf82v3+wUYf9fi7Yk9znl5l3W7gIf3+0ckM/+FzgLiN6Czg6K0ZGTmZcDXGqufNOSUsccB9SZNu4B+pRuaHxaMUq+t+abuthFx9xGOo5rMPBt4Y2P1UcC/jOP4EXE/OoPGM+ed9HX4aTozla4bEQO/zkfE9YE7DTG2xaIZaLx71W12UC9gtA8TRtHs3PvAYaZeR8QtmZ2l2UY3YE3ey9ldFgjKNdl29mMz4/KadHmtm0MzM/E84JRRB5SZ32N285mxZz9WsxFe1lh9ZeBtE64H/1mg3oF+HcN/qLRgZeY2ZmezPj8iVk9jPJIWL4OPmqZ6AeoTBu2mV3XUe1Nt3/0nPO6+MnMnpWt33Rcy82cD7P7axvKCmlIujdnrGsvHUmq6zikijmT2m7pPZOZv+uz2x8bytYatD5WZX2R2ltI7I+LqwxynbsTmGkvR85j9hvVJEfHGUd9IRsSaiHg1pVHZ2h6bTew6rIKszcDk6yKi19jq51oBvJkl+L9bVQu0/iHkMuDkiDh4rn2rwPLftDS0bt5CZ7BnLSVLd87ncUSsqfavOx/4wPiGp2mpnt/vbqx+WMu1H88G3tZYPXDAs3pdeXhj9Qe7ZIkP678ayw9rqR7jaygZ7HUPAD4SEXt12X7sqizL5t+Rf4yIe3bbfhAL8P+ClwI7a8uHM8/MxwX4PUpq2ZL7B1aLyn5jOs6cb9qm4LLG8lcG3O9UOgvvHzue4UgL0seZXQPvXyPiEf12qgI+X2L3FCAoU6qbGRBNP6Jz6vVeDJ8hAvAPdGa3HAb8X0QM1WkzIq5ZBcbePsIYlpzM3EyZqtfMEH8C8JWIGLi2XkSsiIiHURoSPI3dzb66+TiTvQ7/ic7sx6OAj0fE3n3OtQp4F9Pr5joJT6bz7991ga9GxK27bVz9jp8OfJDy+x3L9Py5ZOZ5zA72nEgJQPYsZ1T9fj8K3Ljx0CuqzCItDS+n8+/McsoHK236J8q02BmHDbHvvYFmkL8ZOBxF8xgHAiN34+6l+sD/AZQmjnUnAt+OiPsOe8yIuBFwkyF3+1fgt7Xl5cBHI+LxQ557v4j4e+BbQ56/VVVprDc1Vj8sIj4aEQcMepyIWBYR94iI/2F3+SxJewiDj5qmzbX7FwO/nsetqyG7sA57e2Wf7605pt923Wr2eC+rfhYzDuq1rbTYVZkVD6HztWA58O6I+FhE3GVmWk8U142IFwM/ZHbH3xdWU736ne9y4PON1W+MiC9ExIsj4kkR8bjGbX2X45xGKapfd1Xg1Ij4fEQ8NCIOr3+qX/3DfdWIuFdEvCgiTqe8TjyN8X0Qs+hVGeJ3p/N1EErQ7dvVz/fJEXH9ZrAuIg6u3tS8GjgbeC9wjQHOOenr8JvA6xur7wr8NCL+pp7tFxH7V0HQH7C7VmQzy2dJqOrLNX8u16UE9r8dEa+MiKdExLMi4j8o5U/+hVJXeQfwkuYhWxzuM5jdaffxwHer5/9+Mysj4pCI+BtKZmfzzfYXmJ0xpUWs6sL83sbqh7ec/Xg+pW7uKJrToX+TmfMOfFXBqjPmONdYVI0o78TshnA3AD4REWdExEsj4viIuFJEdJRoiIgDIuLWEfEPEXFqNe5jhhzDxcB96QwCrwHeFBE/joi/i4gbdjn3gRFxh4h4akR8AbgA+DfK9PmF5mnMTqa4H/CbiHhtRNw5IvapP1jNPjgmIh5WvW7/kTJN/Z4Yh5D2ODac0TRdCMz8kfpQZv7tNAczZj8BjqstD5ORUd92zXiGIy1MmfmziPhLSvZSfYrUidWNiLiEkl22ku7eAfT7MKDuZZTgVv3v313pXeLgc8yuFUlmvqOa0vVvjXHdrboB7IyIS6vH96Z/9p0qmfmtKPX+PkTnG8Cg8+dLRGyh1Nram/6vl5uAX/Q556Svw2cC16NcizOuBrwVeGtEbKZMcWsGvy+kTJFspYHFAvD3lN/loxvrb17dutlFmXZ9dmN9a5mQmXl5RJxI+TCjHuC+EVX9yojYQAli95qd8R3gYTaVW5JeBjyC3XVIZ7IfT2rxnK+iBMAHLkVUfdBxr8bqcWQ91o9149ryPSLisMxslkCZt8z8ZZR6qu9h9t/zG1W3mQzUrF7Pg/J6M9f74a9QZjzMNYYfRMS9KLWhD6k9dDS7P2TIiLiM8rq1D5OrVTtvmbkjIu4PvJ/OD1L2AZ5S3Yb5uyxpD+MnDpqmenfro6c2inac0VgeaEpClSlV/8fxz+MakLRQZeangDsCveo17kf3gM8W4NmZ+dhB61NVWWePYHZphKFl5huAOzA7A2rGcspzfz29A4/bGb7h1pKXmb+gBJueDVzSZ9M1wJXo/QZnK6VO4rUy83/mOOckr8OtlKBmr660a5kdePw9cNfM7Jntv9hVP7/HUhq6XTLALn8ATsjMd7L7w8wZg+w/ssz8JeVDxl5lVdbTO/D4X8DxmXlhG2PTdFXdrd/fWN129uOlwD8PudvDmf2aNs7g4wfozEBeTvn724oqA/TulO7fZ/fZdOZ/7f3oHXhM4OvA/8vMO2Xm9wccw1cpU7Z7/b0JSrO7/ekfeBzofJOWmZdQpuo/h84mO3Vz/V2G0tRo7EFoSQubwUdNU/0f9ltFxJWmNpLx+1Rj+cYD7nddOrNuzhrLaKQFrprmdT1K3bcfz7H5nyg1164zR/mDXuf6AHAk8CTgY5SMuEvorOM46LG+SZnadX/gi3RO3e3lUkrnyCcAV87MFw573j1BZm6rfr9Xo3SU/jKdjT562Uzp1Po44NDMfEJVp2+Qc07yOtySmQ+m1Ln8bp9NL6M0VbhhZp4x7HkWmyzeBFyL8hz5AiWQsIUSTD6H8vx5DHBkLajcrFvXnLrfxljPz8w7AfehXHP9XkMup3Q7Py4zH1rVONXS9U90NuiYRO3H11EC8oNqToP+SWbO9bo3sMz8LbMbtLUy9bp2zszM91BeP+5HCYAO+lqwkRJwfAHlteV2mfnJEcbwx8y8N3BTSib0nwbYbRvw1erc183MBdtwsvoZvwK4OmW8P2GwMhe/pDTcuidw1blKlEhaesLZHpqPiDgJeGdt1R2zdK0cZN8jKH+IZj51fFVmPnOc45umiPgWcItq8VzgiMzc0WcXIuKFwItqqx6Tmf/ZzgilhSsirkJ5/hxMKVS/ifIP/JnA6Qt1qmKUxiA3o3SCPJCSWbGFMnX7d5Tx/2bQDDl1qmovHkMJHh9KmQa9k/Lm8mJKRv1PsjQhGMf5JnYdRsQ1qnNdmfIh1MWUrNpvpg1J5hQRb6ezgdTfVdnJkxzDeuA2lN/hQZRr80+Uus/fzMyJNMWRtFs1q+jalA/4r0bJSl5JCTZeQnmt/SXw8zb+Nlfnv351O6C67aL8X3A+5QPQX1QZ8YtSlUBS/1u5ht0/319RfrbO5pL2cAYfNS/zCT5W+7+b3VMwdgD3zswvDLF/ACsX4huziHgQnVPqnpmZr+qz/dUo3Xj3rVZdRglYtp69IUnSYlUF/X9DCfrNuGVmfntKQ5IkSVKN0641bc9gd82PFcCnqm5zfQsUR8RhEfF3lCyXY1se40gy80PAN2urXh4RXZvqVHWAvsjuwCPAqw08SpI0p8fQGXi8kNm1lyVJkjQlZj5qIFV3s25Ze+vprLP0B0pdo6ZnZOZHexz7OEpH2Xqx+AspXSTPAC6i1MrZD7gOJdh4E3Y3cDguM08b8FuZqGpq+Tcp0wNnfJ9Sr+p3lE5wt6LUi1td2+Z/gbuPa+qgJEkLXUSsGnYmQ/U/xP/SWS/5lZn57LEOTpIkSSMz+KiBdJlePaxHZebJfY5/FPBxSnBxWLfIzO+MOK7WRcSNKN/bEQPu8lHgrzJzU1tjkiRpoYmIE4HnAm8APtkv+z8i9qF0xX4RsKr20KXA0Zl5bnsjlSRJ0jBWzL2J1L7M/FlE3AB4NKXL6PXn2OWnwGeB9y707p+Z+YOIuCHlDdIjgV5dvX8MvAz44EJtpiFJUstuBpwM7IiI7wI/pHS4vowyQ+BAygyI21IaDjX9jYFHSZKkhcXMRy1IVYfRWwGHAPsD2yjd6H4N/Dgz/zTF4Y0sIlZQOmFek/K9baV0uvtmZv5mmmOTJGmaqszHj424+3ZKh+u3jm9EkiRJGgeDj5Wqa/KRwA2Aq1HqD26m1Bv8AfCjSdffi4hlwK2rcR1GmUp0LvA1G5FIkqSlpKrf+ClKduMwvgI8Z6HWf5YkSdrT7dHBx4hYD5wA3Be4E3BQn80vptQ8/NfM/GOf7cYxrhXAM4En0Nm9ccY2yj/nT8/Ms9sciyRJ0qRU/wPdHrgdcFPgGpT/hdZRygVdSvlg+FfA14DPZebp0xmtJEmSBrHHBh+rwOMFwJohd70IeGxmjjotqK+IOAT4NKXm0VwuozQm+UQbY5EkSZIkSZLmY08OPu5HyWasOws4FTgTuJASmLwh8AA6m4TsBB407gBkROxFmTp0y9rqc4H3UmodHgjck5IRMGMLcKfM/OY4xyJJkiRJkiTNl8HHkj34TuA/M/OHPbZdC7wW+Ova6ouB62TmhWMc078AT6+t+jDw8Mzc2tjuoZROkCurVb+rxrJlTOM4D1hbHVeSJEmSJEl7rqsBmzPz0FF23pODj3sDzwX+JTMvGnCf9wEPra16YWa+ZEzjuSrwS3ZPA/8hcLPM3N5j+2cBr6itenpmvnpMY7ls9erV64888shxHE6SJEmSJEmL1K9//Wu2bt26ITP3GWX/PTb4OIqIuDLweyCqVd/JzFuM6dgvA55TW3WPzPx8n+1XAGcDV6lW/T4zrzamsfzk+te//vV/8pOfjONwkiRJkiRJWqSOPvpofvrTn/40M48eZf9l4x7QUpaZfwB+Vls1ztTA+9XunwN8YY6x7KBMF59x1YgYpEmNJEmSJEmSNBEGH4e3sXZ/3TgOGBHXAI6qrfpSDpaS+sXG8n3GMR5JkiRJkiRpHAw+Du+I2v3zxnTMGzWWTxtwv28DO2rLx4xnOJIkSZIkSdL8GXwcQkTcFji4tuqbYzr0UY3lXw2yU9Xd+g+1Vdcf03gkSZIkSZKkeTP4OJxnNJb/e0zHvWZj+bdD7FvftnkcSZIkSZIkaWoMPg4oIh4CnFBbdQbwiTEdvtmq/KIh9r24dn9lRKwew3gkSZIkSZKkeVsx7QEsBhFxNPC22qodwF9n5q4xnWLvxvKWIfa9vMuxtg6yY0T8pMdD4+ziLUmSJEmSpD2UmY9ziIjDgM/QGSB8VmZ+d4ynWdNY3jbEvs1A417zHIskSZIkSZI0FmY+9hERBwCfBw6vrX5bZr56zKdqZjqu6rKul+Y062YmZE+ZeXS39VVGpM1rJEmSJEmSNC9mPvYQEfsAnwNuWFv9PuDxLZxuY2O5mQnZTzPTsXksSZIkSZIkaSoMPnYREXsDnwVuXlv9YeCRY6zzWHdZY3n/Ifbdr3Z/e2YOVO9RkiRJkiRJapvBx4aIWEup8Xjr2upPAg/NzJ0tnfY3jeWrD7FvfUr4WWMYiyRJkiRJkjQWBh9rImIv4FPA7WurPws8KDO3t3jqnzaWrzXIThGxBrhyn+NIkiRJkiRJU2PwsRIRq4GPA3eqrf4ScP/MHKb79Ch+0Fg+bsD9bkFn06AfjWc4kiRJkiRJ0vwZfAQiYhXwEeButdVfAe6bmYN2nR5ZZv4G+Hlt1V0iIgbY9a6N5U+Pb1SSJEmSJEnS/OzxwceIWAF8ALh3bfXXgBMy8/IJDuVjtfuH0xkInaUa96Nqq84FvtvCuCRJkiRJkqSR7NHBx4hYDrwXuF9t9TeAe2Xmpnke+4iIyNrtlDl2eTNQ71T9qohY2Wf7pwNXqS2/NjNzxOFKkiRJkiRJY7fHBh+rac3vAB5cW30acI/M3Djp8WTm74A31lYdA7yvqkXZISIeAry4tupc4A3tjlCSJEmSJEkazoq5N1mybgs8srHu6sD3Byu3eIU7ZOa5YxrT8ymdtm9WLT8IuHVEvAc4C9gfuBdwh9o+W4G/nERtSkmSJEmSJGkYe3LwcXmXdVce4Tj9pkYPJTM3R8QJwGeAY6vVVwGe1WOXDcAjM/Pr4xqDJEmSJEmSNC577LTrhSozzwNuBbwAOK/HZtsoDWpulJkf67GNJEmSJEmSNFV7bOZjZp4CDDW/esjjnz3q8TNzO/DSiHg5cGvgWsAhlEzH3wNfy8yLxjRUSZIkSZIkqRV7bPBxMcjMncDXqpskSZIkSZK0qDjtWpIkSZIkSVIrDD5KkiRJkiRJaoXTrqUWvf9bvx3r8R56y6uP9XiSJEmSJEltMvNRkiRJkiRJUisMPkqSJEmSJElqhcFHSZIkSZIkSa0w+ChJkiRJkiSpFQYfJUmSJEmSJLXC4KMkSZIkSZKkVhh8lCRJkiRJktQKg4+SJEmSJEmSWmHwUZIkSZIkSVIrDD5KkiRJkiRJaoXBR0mSJEmSJEmtMPgoSZIkSZIkqRUGHyVJkiRJkiS1wuCjJEmSJEmSpFYYfJQkSZIkSZLUCoOPkiRJkiRJklph8FGSJEmSJElSKww+SpIkSZIkSWqFwUdJkiRJkiRJrTD4KEmSJEmSJKkVBh8lSZIkSZIktcLgoyRJkiRJkqRWGHyUJEmSJEmS1AqDj5IkSZIkSZJaYfBRkiRJkiRJUisMPkqSJEmSJElqhcFHSZIkSZIkSa0w+ChJkiRJkiSpFQYfJUmSJEmSJLXC4KMkSZIkSZKkVhh8lCRJkiRJktQKg4+SJEmSJEmSWmHwUZIkSZIkSVIrDD5KkiRJkiRJaoXBR0mSJEmSJEmtMPgoSZIkSZIkqRUGHyVJkiRJkiS1wuCjJEmSJEmSpFYYfJQkSZIkSZLUCoOPkiRJkiRJklph8FGSJEmSJElSKww+SpIkSZIkSWqFwUdJkiRJkiRJrTD4KEmSJEmSJKkVBh8lSZIkSZIktcLgoyRJkiRJkqRWGHyUJEmSJEmS1AqDj5IkSZIkSZJaYfBRkiRJkiRJUisMPkqSJEmSJElqhcFHSZIkSZIkSa0w+ChJkiRJkiSpFQYfJUmSJEmSJLXC4KMkSZIkSZKkVhh8lCRJkiRJktQKg4+SJEmSJEmSWmHwUZIkSZIkSVIrDD5KkiRJkiRJaoXBR0mSJEmSJEmtMPgoSZIkSZIkqRUGHyVJkiRJkiS1wuCjJEmSJEmSpFYYfJQkSZIkSZLUCoOPkiRJkiRJklph8FGSJEmSJElSKww+SpIkSZIkSWqFwUdJkiRJkiRJrTD4KEmSJEmSJKkVBh8lSZIkSZIktcLgoyRJkiRJkqRWGHyUJEmSJEmS1AqDj5IkSZIkSZJaYfBRkiRJkiRJUisMPkqSJEmSJElqhcFHSZIkSZIkSa0w+ChJkiRJkiSpFQYfJUmSJEmSJLXC4KMkSZIkSZKkVhh8lCRJkiRJktQKg4+SJEmSJEmSWmHwUZIkSZIkSVIrDD5KkiRJkiRJaoXBR0mSJEmSJEmtMPgoSZIkSZIkqRUGHyVJkiRJkiS1wuCjJEmSJEmSpFYYfJQkSZIkSZLUCoOPkiRJkiRJklph8FGSJEmSJElSKww+SpIkSZIkSWqFwUdJkiRJkiRJrTD4KEmSJEmSJKkVBh8lSZIkSZIktcLgoyRJkiRJkqRWGHyUJEmSJEmS1AqDj5IkSZIkSZJaYfBRkiRJkiRJUisMPkqSJEmSJElqhcFHSZIkSZIkSa0w+ChJkiRJkiSpFQYfJUmSJEmSJLXC4KMkSZIkSZKkVhh8lCRJkiRJktQKg4+SJEmSJEmSWmHwUZIkSZIkSVIrDD5KkiRJkiRJaoXBR0mSJEmSJEmtMPgoSZIkSZIkqRV7fPAxIpZFxNER8ciIeH1EfDMiNkdE1m7HtzyG4xvnG+Z2szbHJkmSJEmSJI1qxbQHME0R8RHg7sC6aY9FkiRJkiRJWmr26OAjcFMWZuDxHGDHgNtuaXMgkiRJkiRJ0qj29OBj3Vbgh8D3gL2Bh09xLMdn5tlTPL8kSZIkSZI0b3t68PHdwO8oAccfZeZ2gIg4iekGHyVJkiRJkqRFb48OPmbmC6Y9BkmSJEmSJGmp2uO7XUuSJEmSJElqh8FHSZIkSZIkSa0w+ChJkiRJkiSpFXt0zccF7OURcX3gcGAdcAlwHvBN4PPAJzJz5/SGJ0mSJEmSJM3N4OPC9JDG8kHV7YbA3wBnRcTTMvMTEx+ZJEmSJEmSNCCDjwvXxcBllMzHA+icIn9N4OMR8fLMfO6oJ4iIn/R46MhRjylJkiRJkiTNsObjwvFn4PXAPYADM/OAzDwiMw+iBB/vD/xfY5/nRMRTJjxOSZIkSZIkaSBmPi4M3wOumplbuj2YmZcCH4uIjwPPBV5ae/ifI+Kjmfm7YU+amUd3W19lRF5/2ONJkiRJkiRJdWY+LgCZuaFX4LGxXWbmPwFvqa1eDTyjtcFJkiRJkiRJIzL4uDg9D7i8tnzCtAYiSZIkSZIk9WLwcRHKzD8Dp9ZWHR4Rh01rPJIkSZIkSVI3Bh8XrzMbywdPZRSSJEmSJElSDwYfF6/LG8trpzIKSZIkSZIkqQeDj4vXIY3lC6cyCkmSJEmSJKkHg4+L1+1q97cD505rIJIkSZIkSVI3Bh8XoYi4J3Ct2qr/y8zN0xqPJEmSJEmS1I3BxxZExBERkbXbKX223WvIYx8GvLWx+uThRylJkiRJkiS1y+Dj9D04Ik6NiPtGxKp+G0bEXYBvAVerrf4B8J42ByhJkiRJkiSNYsW0BzBNEXF/4FVdHlrfWH5fRDS7SwM8IzM/Ooah3L66XRIR/wf8EPgjsIHSxfoawF2BGzX2Ow84MTN3jWEMkiRJkiRJ0ljt0cFHYB/gyAG2u3Kf/cdpP+De1W0upwEPz8yzxzwGSZIkSZIkaSycdj193wXeCfwMyDm2TeAbwMOB22bmr1semyRJkiRJkjSyPTrzMTNPpoVmLVU2Ygy47Y+BRwNExH7ATYCrA1cC9gK2ApcAZwPfzsxLxz1eSZIkSZIkqQ17dPBxocnMS4CvTHsckiRJkiRJ0jg47VqSJEmSJElSKww+SpIkSZIkSWqFwUdJkiRJkiRJrTD4KEmSJEmSJKkVBh8lSZIkSZIktcLgoyRJkiRJkqRWGHyUJEmSJEmS1AqDj5IkSZIkSZJaYfBRkiRJkiRJUisMPkot+945F/GpH/yBizdvm/ZQJEmSJEmSJmrFtAcgLWV/uORyPnL6uQBcsnkbjzjuiOkOSJIkSZIkaYLMfJRadO4ll19x/6wLN7Erc4qjkSRJkiRJmiyDj1KLLtuy/Yr7W3fs4pLN2/tsLUmSJEmStLQYfJRatGHLjo7lP156eY8tJUmSJEmSlh6Dj1KLNlzemen4x0u3TGkkkiRJkiRJk2fwUWrRZbMyHw0+SpIkSZKkPYfBR6lFG7Z0Zj6e57RrSZIkSZK0BzH4KLVk566cVfPx4s3buXzbzimNSJIkSZIkabIMPkot+fOmrWSX9edd5tRrSZIkSZK0ZzD4KLXkgsu2dl1vx2tJkiRJkrSnMPgoteT8HhmO59l0RpIkSZIk7SEMPkotOb9n5qPBR0mSJEmStGcw+Ci1pJ75eNDeqzvW79zVrRqkJEmSJEnS0mLwUWrJBRt2Zz4eefDeLItyf8eu5MKN3bMiJUmSJEmSlhKDj1JLLqhlPh6wdiVXqmU/OvVakiRJkiTtCQw+Si05f8PuAOP6vVZy2L5rrlg+z47XkiRJkiRpD2DwUWpJveHMPmtWcti+e12xbOajJEmSJEnaExh8lFqwY+cu/ryxHnxc0ZH5aPBRkiRJkiTtCVZMewDSUvTnTduoN7Rev2Ylq1bsjvVv3LqDDVu2s37NyimMTpIkSZIkaTLMfJRacH6t2czqFctYtWIZ69esZO/Vu+P955n9KEmSJEmSljiDj1ILmvUeZzj1WpIkSZIk7UkMPkotqGc+rt9rd7ZjZ/DRjteSJEmSJGlpM/goteCCDd0zHw+147UkSZIkSdqDGHyUWnBBLfNxnzXdMx8v3LiV7Tt3TXRckiRJkiRJk2TwUWpBx7TrWubjlfZezYplAcCuhAtqtSElSZIkSZKWGoOPUgvqDWfW1zIfly8LDtnHuo+SJEmSJGnPYPBRasEFG+rTrld2PHZovenMZdZ9lCRJkiRJS5fBR2nMtu/cxZ83bbtieZ+9OoOPHR2vLzH4KEmSJEmSli6Dj9KYXbhxK5m7l+vTrgEOq3W8Pu+yy8n6xpIkSZIkSUuIwUdpzOr1HvdauZyVyzufZofWaj5u2b6Ly7bsmNjYJEmSJEmSJsngozRmnZ2uV8x6fK9Vy1lVC0hu3mbwUZIkSZIkLU0GH6Uxu+Cy3s1mZuy1avkV9y/ftrP1MUmSJEmSJE2DwUdpzC7YsHvadbfMRyjTsWdsNvgoSZIkSZKWKIOP0pjVp103O13PqGc+btlu8FGSJEmSJC1NBh+lMas3nDHzUZIkSZIk7ckMPkpj1tlwZoCaj2Y+SpIkSZKkJcrgozRm9ZqP+/TIfFy70oYzkiRJkiRp6TP4KI3Rth27uGjTtiuWB+p2beajJEmSJElaogw+SmP0p41bO5Z71nw0+ChJkiRJkvYABh+lMarXe9x/7UpWLO/+FNvLadeSJEmSJGkPYPBRGqMLasHHg9ev6bmdmY+SJEmSJGlPYPBRGqPzL9s97frgfVb33K6e+bh5245WxyRJkiRJkjQtBh+lMbpgw+7Mx0P26Z35uHbV7lqQW7fvYldmq+OSJEmSJEmahokEHyNi30mcR5q2eubjIQNmPiawxanXkiRJkiRpCZpU5uMfIuLdEXH7CZ1Pmop6w5l+mY+rVy4jass2nZEkSZIkSUvRpIKPewEPA74SEWdGxNMj4qAJnVuamAvqNR/7NJxZFsGalTadkSRJkiRJS9ukaz4GcG3gn4HfR8SHIuIeEx6D1JrzazUf+zWcgUbHazMfJUmSJEnSEjSp4OMrgT821q0E7g98JiLOjogXRMTVJjQeaey27tjJJZu3X7Hcb9o1dNZ9NPNRkiRJkiQtRRMJPmbmc4CrAycCnwRmIi1R3a4OvBA4KyL+JyJOjIjl3Y4lLVT1KdcAB+09eObjZjMfJUmSJEnSEjSxadeZuSszP5mZJ1KCjc8FftXYbDlwd+AjlGnZr4iIa09qjNJ8XFCbcn3gulWsWtH/6VXPfLTbtSRJkiRJWoomXfMRgMw8LzNfkZnXAe4IvB+YSRubyYY8BHgG8POI+EpEPDQi+qeSSVN0fr3ZzBxTrsHMR0mSJEmStPRNJfhYl5mnZubDgcOAJwNnNDYJ4PbAe4A/RMS/R8Qxkx2lNLfzL6s1m1k/d5x8rTUfJUmSJEnSEjf14OOMzLw0M9+QmccCNwPeClxWPTyTDbk/8CTg+xHxrYh4TESsm86IpU4XbNid+XjIHJ2uwW7XkiRJkiRp6Vswwce6zDw9Mx9PyYY8CTgfyOo2E4i8GfA24NyIeJ2dsjVtf6oFHw9eP8C0azMfJUmSJEnSErcgg48AEXEo8FTgecDBtYdyZpPq6z7AE4FfRMQ/RcTKiQ1Sqtm4ZccV9/fZa8Wc25v5KEmSJEmSlrq5IyQTFBHLgHsDjwXuSel+fcXD1dc/Au8DbgjctbZ+NfBs4OYRcc/M3DWRQUuVTdt2Bx/Xrhog+GjmoyRJkiRJWuIWRPAxIq4FPBp4JHDozOraJruAL1DqQH4qM3dW+10NeBzweGC/ap+7AE8A3jCJsUsz6h2r161e3mfLorPb9Y4+W0qSJEmSJC1OU5t2HRGrI+LhEfEV4EzgmZQajzM1HaFkOb4MODIz75mZH58JPAJk5u8y87nAtYEv1Q7/iIl8E1LNpq27A4jrhsx83L4z2bHLZF1JkiRJkrS0TDzzMSJuQplW/RBg35nVtU12AV+kkeXYT2b+OSJOAs6hTNU+apxjlgZRn3a9bvXcT63m1OzLt+1k/ZoFW4ZVkiRJkiRpaBMJPkbEvsDDKEHHG82sZnf3aihZju8E3p6Z5wx7jsz8Q0ScA1wTWDfvQUtD2rx1d5x87aq5p12vXB4sj2Bnlh5KJfhovyRJkiRJkrR0TCrz8Y+UhjCwO+hI9fULwNuATw6S5TiHDfPcXxrZxtq0670HyHyMCPZatfyK/Ww6I0mSJEmSlppJBR/X0JnleB7zyHLs44+UxjPSRO3YuYutO3bXbFw7QPARSt3HK4KP2ww+SpIkSZKkpWWSNR/HneU4+wSZ9xr3MaVBbG5kLa4bYNo1dHa8NvNRkiRJkiQtNZMKPr6c8Wc5SgtGvdM1zG4m00u94/VmMx8lSZIkSdISM5HgY2Y+bxLnkaZlU63ZzKrly1i1YrCu1WY+SpIkSZKkpWxS3a5vX929PDO/M4/jHAvsDZCZXx3H2KRx2Lxtd+bj2tWDTbmGRvDRzEdJkiRJkrTETGra9SmUmo+/Aq47j+O8AzimOtYk61VKfdU7Xa8bcMo1dE67NvNRkiRJkiQtNZMM4AW7u13P9zjSgrK5Nu163RCZj2vNfJQkSZIkSUvYYIXpxiMneC5pojbVp12b+ShJkiRJkgRMNvg4DjORmh19t5ImbNOImY/WfJQkSZIkSUvZYgs+HlZ93TjVUUgN9YYzo9Z83GzmoyRJkiRJWmIWTfAxIu4IHEiZvn3OlIcjdejMfBwi+FjLfNyybSeZVieQJEmSJElLx1gbzkTEMcCN+2yyPiL+aohDLgP2BW4APLi2/rThRye1p7Pm4xDTrmuZjzsz2bZzF6tXDL6/JEmSJEnSQjbubtf3A17Q47EADgbeOeKxZ7pcJ/CfIx5DasWmrbuDj3uPmPkIpe6jwUdJkiRJkrRUtDHtOubeZOjj1QOPL8zM7475HNK8bK41ixmm2/WKZctYtXz309CO15IkSZIkaSkZd+bjjF4ByGEDkzsozWXOpky1fmdmfmce45JasbGW+ThMt2so2Y/bLt8F2PFakiRJkiQtLWMNPmbmi4EXN9dHxC5K1uKvM/M64zyntBB0dLseYto1lLqPl16+HTDzUZIkSZIkLS2T7HY97unY0oJR73Y9TMMZ6Kz7aOajJEmSJElaStqadt00kw150YTOJ01UveHMuiFqPkJnx2szHyVJkiRJ0lIykeBjNR1bWrLqDWeGnnZdy3zcbOajJEmSJElaQiY57VpasjZtG73hzFozHyVJkiRJ0hJl8FEag/q067XDTru25qMkSZIkSVqiDD5K87Rtxy6278wrlveex7RrMx8lSZIkSdJSMraajxFRj5pkZq7o8dg4dBxfmqbNtSnXAGuHnHbd0XDGzEdJkiRJkrSEjDOAF0BWX4d5TFrUNm5tBB9XDhl8NPNRkiRJkiQtUeOedt0vuGjgUUtSvUP1mpXLWLF8uKdVPfOxmUUpSZIkSZK0mI0z8/FRIz4mLWr1ZjPrhmw2A53Bx63bd7Erk2VhrF6SJEmSJC1+Yws+Zua7RnlMWuw2bd2d+ThsvUfo7I6dlABkfSq2JEmSJEnSYmW3a2meNm2bX+bj6pXLOmoSOPVakiRJkiQtFQYfpXmqBwvXrR4++LgsgjUrbTojSZIkSZKWHoOP0jxtrE+7HnG6dEfH620GHyVJkiRJ0tIwzoYzYxURa4FrAyuBczLzT1MektTV5lrDmb1HyHyEzqYzZj5KkiRJkqSlYiKZjxGxd0Rcs7pdZY5trxwRHwQuBk4HvgWcFxFfi4hbTGK80jA2batnPo4YfKxlPm4281GSJEmSJC0Rk5p2/a/AL6vbP/baKCIOBU4DHkjJeIza7TbA1yPixLYHKw1j09Z6zccRp13XMh+3mPkoSZIkSZKWiEkFH+8LVzT0fWOf7d4EXLW6n43HkjJN/L0RcfXxDk8a3XwbzoCZj5IkSZIkaWlqPfgYEUcAh1KChz/LzF/22O5o4ER2Bx0vBP4euBfwHGBj9dhewIvaHLM0jE21hjPrRm04Y81HSZIkSZK0BE2i4cz1a/e/2We7R1RfA7gcOC4zz6rWfS4iTgO+XC0/KCKekJlbxjtUaXj1adej1nxca7drSZIkSZK0BE1i2nV9ivTP+mx3z+prAh+sBR7LysxTgFOqxbXAsWManzQvm7bZ7VqSJEmSJKmbSQQf96ndv7jbBhFxJeAGtVX/3eNYp9TuHzW/YUnjUa/RuHbUhjNmPkqSJEmSpCVoEtOuV9buN5vIzLgNuxvSbAdO7bHd72v395/nuBa0iFgG3Bo4EjgMuBQ4F/haZnYN4mo6Nta7XY847drMR0mSJEmStBRNIvi4oXb/wB7b3KH6msD3MvPyHtvVg5er5jswuCLIdxRws9rtRpTGNjPuWE37bl1ErACeCTwBuHKXTbZFxKeAp2fm2ZMYk/rbXG84M4Zu12Y+SpIkSZKkpWISwcdza/d71Wk8oXb/632OdUDt/saRR1SJiI8AdwfWzfdY4xARhwCfpgRAe1kFPAC4a0T8VWZ+YiKDU0/1mo9rx9DtetvOXezYtYsVyyZRFUGSJEmSJKk9kwg+nl59DeCEiDgwM/8882BE3IUytXjG//Y51nVq9/84hrHdlIUTeNwL+ASdgcdzgfcCv6Zkjd4TuH312D7AByLiTpnZr4u4WpSZHd2uR818bHbJvnzbTtavMfgoSZIkSZIWt9ajG5n5G+D7lCnT64BPRcQNImJ1RNwReCe7p1NfSP/g4y1r93855qFuBb4DvIUS8Ju0l9D5/X0YODIzn5WZb8/MV2bmHYCHUepiAqwBPhgRayY8VlW27tjFrloxgHUjNpxZuTxYHnHFslOvJUmSJEnSUjCp1KqXs7uhzC2BHwCbgS8BV6keS+C1mdk16hIRhwPHVItbgB+PYVzvBv6GkgG5PjNvkZmPp38AdOwi4qrAk2qrfgg8NDO3NrfNzPcDL6ituhrwxHZHqF7qWY8wesOZiOis+2jTGUmSJEmStARMJPiYmR+hZBTOBCCjdpvJG/s28Oo+h3nYzOGAb2fmjj7bDjquF1RZhadn5va592jN4ylZjDOeMcd4/pXOWppPbWNQmtumrZ1BwnrtxmHZ8VqSJEmSJC01Eysql5lPoATZzmk8tAV4M3CXzNzWbd+IWMXuzMAA/qetcU7J/Wr3zwG+0G/jKvD6ztqqq0ZEvyY1akm92cy6VctZtiz6bN2fHa8lSZIkSdJSM4mGM1fIzLcCb42IawCHUqZe/6xX0LFmf+DZteXPtTTEiat+FkfVVn0pM7PX9jVfBJ5XW74P8N1xjk1z21zvdD1is5kZZj5KkiRJkqSlZqLBxxlVE5rfDLH9+cC72hvRVN2osXzagPt9G9jB7t/hMX22VUs21qZdr1s1+pRr6Mx83GzmoyRJkiRJWgImNu1aPR3VWP7VIDtl5hbgD7VV1x/biDSwzbWGM+vmm/lowxlJkiRJkrTEGHycvms2ln87xL71bZvH0QRs2lbPfBzjtGszHyVJkiRJ0hJg8HH69mksXzTEvhfX7q+MiNVjGI+GsGlrvebj/KZdr7XhjCRJkiRJWmKmUvMxItZSah0eBewHrKN0sR5YZr5k/CObir0by1uG2PfyLsfaOujOEfGTHg8dOcQY9mgd3a5tOCNJkiRJktRhosHHiLgBpUPzfYH5ZuktleDjmsbyXJ2/65qBxr3mORYNaXNLDWfMfJQkSZIkSUvBxIKPEfF44LXVOWeyHJMhMx5r+y0VzUzHVV3W9dIM4DYzIfvKzKO7ra8yIm1gM4CN9WnXY6z5uNnMR0mSJEmStARMJPgYEScCb6wW64HDpNQ43DiJcSxQze99DYMHH5uZjnvyz3EqNtemXe89xm7XW7btJDOJGCU2L0mSJEmStDC0HnyMEj35t2pxJtPxv4C3At/OzGFqHC5FlzWW9wcuGXDf/Wr3t2fmwPUeNR71btfzbThTz3zcmcn2ncmqFQYfJUmSJEnS4jWJzMebA0ewO+PxMZn5zgmcd7H4TWP56l3W9XJ47f5Z4xmOhlHvdr1uvtOuGzUjN2/bwaoVq+Z1TEmSJEmSpGlaNoFz3Lh2/38NPM7y08bytQbZKSLWAFfucxxNQEfDmXlOu16xbBmrlu9+StrxWpIkSZIkLXaTCD4eULv/2Qmcb7H5QWP5uAH3uwWdmas/Gs9wNIxN2+qZj/Obdg12vJYkSZIkSUvLJIKPf67dv3gC51tUMvM3wM9rq+4Sg3UZuWtj+dPjG5UGVZ92vXaemY/QWffRzEdJkiRJkrTYTSL4eHbt/kETON9i9LHa/cOBu/XbOCJWAI+qrToX+G4L49Ic6g1n9p5nwxkw81GSJEmSJC0tkwg+nsLu7Mc7TuB8UxcRR0RE1m6nzLHLm4F6p+pXRcTKPts/HbhKbfm1mZm9NlZ7NtczH+fZcAbMfJQkSZIkSUtL68HHzNwOvBEI4G4RceO2z7nYZObvKD+jGccA74uI1c1tI+IhwItrq84F3tDuCNXNrl3Zkfk4327X0Jn5uNnMR0mSJEmStMjNP1oymJdSahQeB3wkIu6Ymb+d0Ll7ioj7A6/q8tD6xvL7IuLyLts9IzM/OqbhPB+4PXCzavlBwK0j4j3AWcD+wL2AO9T22Qr8ZWZuGdMYNIRmZuK6MUy7XmvmoyRJkiRJWkImEnzMzJ0RcU/gv4B7Aj+IiFcA78rM8ycxhh72AY4cYLsr99l/LDJzc0ScAHwGOLZafRXgWT122QA8MjO/Pq4xaDj1TtcA68bRcMaaj5IkSZIkaQmZSPAxIr5c3V0G7AL2BV4BvCIizgHOA4bJ3svMvPN4Rzl9mXleRNyKEnB8AnBol822UQKU/1B1ytaUbNq6Ozi4LGD1ivlXMegIPpr5KEmSJEmSFrlJTbs+Hqg3RElKDUiAIygdngcVjWONLDNPBk4ex7Eaxz2b3d/fsPtuB14aES8Hbg1cCziEkun4e+BrmXnRmIaqedhUazazbvUKIkb6lXfoaDhj5qMkSZIkSVrkJhV8hP7BuPlHbZaYzNwJfK26aQHaPOZmM2C3a0mSJEmStLRMKvj4rgmdR5qYeubj2jE0mwFrPkqSJEmSpKVlUg1nHjWJ80iTVG8400bm45btO9mVybIxTOeWJEmSJEmahvl3yJD2UJtrDWfWjSnzcW0tiJnA1u27xnJcSZIkSZKkaTD4KI1o49bxZz6uXrmsowDq5lp2pSRJkiRJ0mJj8FEaUT0wuHb1eIKPyyJYY9MZSZIkSZK0RBh8lEa0qdYQZu8xTbuGRtMZg4+SJEmSJGkRm1S361ki4l7AXYFbAlcF9gfWAr/KzOs2tl0J3KRa3JmZ35vkWKVuNte7XY9p2jV0Np2x47UkSZIkSVrMJh58jIiHAS8FDq+v7nEfgMzcHhHvBq5dHeMmmfnDVgcqzWFjveHMKjMfJUmSJEmSmiY27ToiVkTEfwHvpgQeo3aD0ty3nzfWtn14K4OUhlCv+bhuTDUfwcxHSZIkSZK0dEyy5uP7gAezO+C4Hfgs8GLgCdW6fgHID9Uev2d7w5QGU6/5OK6GM9DIfDT4KEmSJEmSFrGJTLuOiL8EHkQJHgbwEeDJmfnH2jZv6neMzDwvIr4PHAtcPyIOzMw/tzhsqa9NtZqPY512Xct83Oy0a0mSJEmStIhNKvPxxbX7b8nMB9UDj0M4vXb/BvMckzQvHcHHMWY+rjXzUZIkSZIkLRGtBx8j4vqURjEJ/A74+3kc7he1+0fOZ1zSfG3eVm8401LNRzMfJUmSJEnSIjaJzMdja/f/OzO3zuNYl9Tu7z+P40jzVs98XLu6pW7XZj5KkiRJkqRFbBLBx0Nq98+c57HqkZhV8zyWNC+bat2u926r27WZj5IkSZIkaRGbRPCx3sF6vhGaA2r3L57nsaSR7dyVbNm+64rlteNsOGPmoyRJkiRJWiImEXy8oHZ/vnUab1y7f/48jyWNrJ71CO3VfNy2cxc7du3qs7UkSZIkSdLCNYng409r9+8z6kEiYjVw99qq00YekTRPm7d2ZiSOt9t157HMfpQkSZIkSYtV68HHzDydkqUYwHUj4pEjHuqJwJUo07h/mpl/HNMQpaHVMx9XLg9WrRjfU2nl8mB5xBXL1n2UJEmSJEmL1SQyHwHeUX0N4E0Rcddhdq62f3lt1evGNTBpFB2drsc45RogIlhj3UdJkiRJkrQETCr4+EpK7ccE9gL+JyLeFBHX6bdTRBwQEa8APk3pbp3AL4D/bHm8Ul+batOux9npesZaO15LkiRJkqQlYPxRky4yc2NEnAh8iRJ8XA78LfC3EXEW8JPa5gdExJuBo4FbVdvOzEG9DDgxM43GaKo2b6tnPo6v0/UMO15LkiRJkqSlYFKZj2TmacAJdHa/DkoH7BMoWY0A+wN/A9yGzuDoH4F7ZuaZ7Y9W6m9jfdp1C5mPe5n5KEmSJEmSloCJBR8BMvMrwDHAycD22kPR2DRq63YC7wGOrQKY0tRt3lafdt1u5uNmMx8lSZIkSdIiNZFp13WZ+Sfg0RHxbOCBwO2AGwEHAvsBm4ELgZ8DXwE+nJnnTHqcUj9tNpyBxrRrMx8lSZIkSdIiNfHg44zMPB94Y3WTFpV6w5l1bdR8rE273mLmoyRJkiRJWqQmOu1aWirqDWfWtdHt2mnXkiRJkiRpCTD4KI1gU8vBRxvOSJIkSZKkpWCi064jIoBjq9uVgAOAfYBLgYsotR6/m5lnTHJc0rDq067XtjHtul7z0cxHSZIkSZK0SE0k+BgRtwCeAdyZEmyca/tLgC8C/5KZ32t3dNLw6g1n9jbzUZIkSZIkqatWp11HxCER8Wngm8D9gH2BqG5dd6lu+wMPAr4dER+PiIPaHKc0rHodxla6Xa/szHzMzLGfQ5IkSZIkqW2tBR8j4ijgNOCe7A421iMo0eVGY7sATgC+GRHXaWus0rA2bq3XfGx32vXOTLbvNPgoSZIkSZIWn1amXUfE1YCvAgdSAolJCSRuomRBfgM4G7gY2AisB/YDrgkcB9wKWMfuIOQ1gVMj4maZeW4bY5aG0dHtuo3Mx0Ydyc3bdrBqxaqxn0eSJEmSJKlNbdV8/E92Bx4D+BPwz8DbM3PDXDtHxD7A3wL/SGlMk8AhwH9QMimlqepoONNC5uOKZctYtXwZ23buAkrdx/3GfhZJkiRJkqR2jX3adUTcndJYZiZr8TvAsZn5b4MEHgEy87LM/BdKV+zvsntK9t0i4s7jHrM0rE0tZz5Co+O1TWckSZIkSdIi1EbNx6dUXwP4HXD3UadKZ+bvgXtUx5kJZj51vgOU5mtzLfNxXQvdrmF20xlJkiRJkqTFZqzBx4g4ALhLtZjAYzPzkvkcMzMvAh7L7qY0d4uI/eZzTGk+tu3YdcV0aGin4Qw0Mh8NPkqSJEmSpEVo3JmPd6TUkUzgR5n5pXEcNDO/CPyoWlwB3Gkcx5VGUW82A7C2rWnXK512LUmSJEmSFrdxBx9vVbv/7jEfu368W/XcSmrZpkYW4rpVZj5KkiRJkiR1M+7g4/Vq97815mOfVrt//TEfWxrY5q27Mx9Xr1jGiuVtlE6FtbXMx81mPkqSJEmSpEVo3FGTI2r3vzfmY59eu3/4mI8tDWxjLfjYVrMZMPNRkiRJkiQtfuMOPh5cfb08M7eM88CZeTmwmdJ05uA5Npdas3lbvdN1O1OuAdZY81GSJEmSJC1y4w4+7k1pNnPJmI87Y+a461s6vjSnTfXMx5aazQCsNfNRkiRJkiQtcuMOPq6uvm4e83FnXF59XdXS8aU5bap1u17bUrMZsNu1JEmSJEla/MYdfGyn88ZsMaHzSLNs2lqfdm3NR0mSJEmSpF4mFSyUlozN2yYz7bqe+bhl+052ZbZ2LkmSJEmSpDYYfJSGtLGW+bi2xYYza2uBzQS2bt/V2rkkSZIkSZLa0Fba1vqI+Ks2jtvCMaWhbK41nNm7xWnXq1cuIyiBR7DuoyRJkiRJWnzaipwcDLyzpWNLU7WpVn9xbYvTrpdFsGbl8iuCjvXp3pIkSZIkSYtBe5GTdprCWPROU7dpa73mY3vTrqE0nZkJPpr5KEmSJEmSFps2go9tdqK2y7WmrqPhTIvTrqGz6YwdryVJkiRJ0mIz7sjJo8Z8PGnB2VRrOLOuxYYzUDIfZ5j5KEmSJEmSFpuxBh8z813jPJ60EG2qZT62WfMRzHyUJEmSJEmL27JpD0BabDZNqNs1NDIfDT5KkiRJkqRFxuCjNKTNHd2uW552vdJp15IkSZIkafEy+CgNaePWyTWcqQc3N5v5KEmSJEmSFhmDj9IQMrMjCDjJbtf1LtuSJEmSJEmLgcFHaQhbd+xi5668Ynldy9Ou16/ZHdzcsMXgoyRJkiRJWlwMPkpDqDebAVjbcubj+jUrr7i/YcsOMrPP1pIkSZIkSQuLwUdpCM26i2tXtpv5uM9eu4OP23bu6qg3KUmSJEmStNAZfJSGsKlWd3HtquUsWxatnm/tquXUT3HBhq2tnk+SJEmSJGmcDD5KQ6hPu167qt0p1wDLIjqmXp9/2ZbWzylJkiRJkjQuBh+lIWzaunva9d6r251yPaPedOaCy8x8lCRJkiRJi4fBR2kIm7dNNvMROpvOXLDBzEdJkiRJkrR4GHyUhrCxlvm4bkKZj/vUMh/PN/NRkiRJkiQtIgYfpSFMJ/OxNu3ahjOSJEmSJGkRMfgoDaGz5uNkgo/72HBGkiRJkiQtUgYfpSF0drueRsMZg4+SJEmSJGnxMPgoDWFTbdr1ugllPnY2nNlKZk7kvJIkSZIkSfNl8FEawuZpNJzZa3fwcfO2nWysZV9KkiRJkiQtZAYfpSFsnELDmbWrlrMsdi/bdEaSJEmSJC0WBh+lIWyuZR2um1DNx2URHVOvbTojSZIkSZIWC4OP0hA2batPu55M5iM0m86Y+ShJkiRJkhYHg4/SEOrdricbfKw3nTHzUZIkSZIkLQ4GH6UhbK5lPq6d0LRrgH1qmY/nm/koSZIkSZIWCYOP0hDqmY97T2natTUfJUmSJEnSYmHwURpCPfg4qW7XAPt0TLs281GSJEmSJC0OBh+lAe3alWzeXm84M7lp1x01H818lCRJkiRJi4TBR2lAW3bsJHP38tS6XW/YStYHIkmSJEmStEAZfJQGtLE25Rpg3SSnXe+1O/Nx87ads8YiSZIkSZK0EBl8lAa0eevuKdfLAtasnNzTZ+2q5SyL3ct2vJYkSZIkSYuBwUdpQJu27c42XLdqBRHRZ+vxWhbRWfdxg3UfJUmSJEnSwmfwURrQplrm49oJNpuZ0VH30cxHSZIkSZK0CBh8lAbUzHycNDMfJUmSJEnSYmPwURpQvebjJDtdz9inlvlozUdJkiRJkrQYGHyUBrSp1mF67arpTrs+/zIzHyVJkiRJ0sJn8FEaUMe066lkPtanXZv5KEmSJEmSFj6Dj9KANm+b7rTrjpqPZj5KkiRJkqRFwOCjNKCNW+sNZ6bc7XrDVjJz4mOQJEmSJEkahsFHaUCbO2o+TmHa9V67Mx83b9vZEQyVJEmSJElaiAw+SgPaVJt2vffqyWc+rl21nBXL4oplO15LkiRJkqSFzuCjNKCObtdTqPm4LIKD1q++YvmCDdZ9lCRJkiRJC5vBR2lA9czHadR8BDh4nzVX3L/AzEdJkiRJkrTAGXyUBlSv+TiNbtcAB9cyH8+347UkSZIkSVrgDD5KA9o45YYzAIfsU592beajJEmSJEla2Aw+SgPaXJ92PYWGMwAHr9897drMR0mSJEmStNAZfJQGtHnb9Kddm/koSZIkSZIWE4OP0oDq067XTWnadWfDGTMfJUmSJEnSwmbwURrAzl3Jlu27rlheO61u1x0NZ7aSmVMZhyRJkiRJ0iAMPkoDqE+5Bth7atOud2c+Xr59Z0c2piRJkiRJ0kJj8FEawKatOzuW106p4cwBa1exYllcsXz+ZdZ9lCRJkiRJC5fBR2kAm2qZjyuWBauWT+eps2xZcND6etMZ6z5KkiRJkqSFy+CjNIDNtczHdatXEBF9tm5XZ9MZMx8lSZIkSdLCZfBRGkBnp+vpTLme0dl0xsxHSZIkSZK0cBl8lAZQbzizdkrNZmYcsk992rWZj5IkSZIkaeEy+CgNYNO2zmnX03TI+t3Trs18lCRJkiRJC5nBR2kAmxbStGszHyVJkiRJ0iJh8FEaQD34uHbVdDMfOxvOmPkoSZIkSZIWLoOP0gA216Zd7716ITWc2UpmTnE0kiRJkiRJvU03hWuBioijgWOAKwM7gXOB72bmb6Y6ME1NR+bjtGs+1jIfL9++k0s2b2f/daumOCJJkiRJkqTuDD7WRMQDgedTAo/dHv8G8NzMPKWFcx8PfGXE3W+emd8d32jUtGnbwqn5eOC6Vey710ouvXw7AGeev4FbXfPAqY5JkiRJkiSpG6ddAxGxPCLeCXyIHoHHyq2B/42Il05mZFooNm9dON2uI4LrHrr+iuUzz9swxdFIkiRJkiT1ZuZj8RrgpNryZuB9wBnAKuCWwAOAlZSA7fMi4qLMfE2LYzoH2DHnVoVdR1q2saPb9fSfNtc7dD3f/s1FAPzc4KMkSZIkSVqgph9FmbKIuDfwd7VVPwXukZm/a2x3I+B/KHUgAf41Ir6UmT9qaWjHZ+bZLR1bQ6o3nFk75YYzANc5ZHfm4y/ON/goSZIkSZIWpj162nVELANeXlu1GTihGXgEyMwfAA8CdlWrmvtqCavXfNx7ytOuoWQ+zvjFeRvseC1JkiRJkhakPTr4CNyZzhqPr8vMs3ptnJnfoNSFnHGfiLhWW4PTwtHR7XoBTLu+Ti34uGHrDs695PIpjkaSJEmSJKm7PT34eL/G8n8MsM/bG8snjmcoWsg21RvOTLnbNcA+a1Zylf32umLZpjOSJEmSJGkh2tODj/eu3f91Zv56gH2+RmeDl/uMd0haiDbXpl1Pu9v1jHrHa5vOSJIkSZKkhWiPDT5GxH7A1WurThtkv8zcBnyvtuqYXttq6ejIfFwADWegM/ho5qMkSf+/vfuOj+sq8z/+fWbUJcuSe4m705wGCYnTkw0JLQtLC7AhhLIsgVAWtoT9hYXfssvSFn6EXTphwyYQ6kJCykIIgfQKpOEUd8d27LhILurl+f1xp9y51sgz1ozmSvN5v17z8j1nzr33kXQsXT06BQAAAHFUtclHSUdHymuKODc8QrLdzOaUIJ6oT5vZo2bWYWb9ZvaCmT1uZt80s9ebWTwyYFWgf3BY/UPDmXIc1nyUIpvOsOM1AAAAAACIoWpOPi6NlDcVcW60bfRapfCXkk6Q1CapVtJMScdJeo+k/5H0rJn9RRnui4ie/qGcchynXa/dsV8DoQQpAAAAAABAHFRz8rE1Ut5dxLkdkfKUEVuNXYekjZJ2SopmlpZKusHM/q1M90bK/tB6j5LUFIMNZyRp6YwW1SRMkjQw5Fq3o6vCEQEAAAAAAOSq5uRjS6TcO2KrkfUc5FqHapek/5T0CknT3X2auy9295mSpkl6vaR7I+dcaWZ/cyg3M7M/jfSStGwsH8Rk092XTT7W1SRUm4zHf5u6moSWzmzOlJ/etreC0QAAAAAAABwoHlmUymiIlPuLOLcvUm4cYyxSsInNYe7+IXf/lbvnjMR09z3u/nNJZ0n6eOTcz5nZghLEgBF0haZdt8RkynXakXOyA3jZdAYAAAAAAMRNNScfoyMd64o4tz5Sjo6ELJq773P3g46+9MCnJH0jEs8Vh3DPY0Z6KXdDnarXFRr5GJcp12lHseM1AAAAAACIsWpOPu6PlKMjIUcTHekYvdZ4+CflJj1fXYEYqkI4+dgck52u046cnU0+Pk3yEQAAAAAAxEw1Jx+jC+S1F3FuW6Q87lkfd98l6c5Q1SIzmzvecVSD7tC06+b6eI18DO94vaWzR/t6ByoYDQAAAAAAQK5qTj6uj5QXFnHuokh53RhjOVTPRMqzKhLFJNcV2u26OWZrPh7W3pizDuWz2ysxCBcAAAAAAGBk1Zx8XBUpLy/i3PBu0B3uvq0E8RyK6FqTTRWJYpKL85qPZqYjZmc3W2fdRwAAAAAAECdVm3x0905Jm0JVpxVynpnVSTopVPVECcMq1uxIeWdFopjkuvrC067jNfJRyp16/cy26GoCAAAAAAAAlVO1yceUW0PHy8xsaQHnnKXczWluLm1IRTkrdDwgaUulApnMuvvju+GMxKYzAAAAAAAgvqo9+fjzSPmvCzgn2uaG0oRSHDN7pXKnit/r7t2ViGWy2x8a+dgUsw1nJOnIOa2Z42e275O7VzAaAAAAAACArGpPPt4u6clQ+YNmtiRfYzM7TdJFoapb3H11nraLzcxDr9+Nct3GYoJO7Wr9zUj1d4u5BgoX95GPR4WmXXd2D+iFfX0VjAYAAAAAACCrqpOP7j4s6cpQVbOkm8xsQbStmR0v6SfKfs6GJX2sRKG82czuNLPXpNaUzMvMzpf0oKRwjI9Juq5EsSAi7ms+tjfXadaU+kyZTWcAAAAAAEBcxC+TMs7c/SYz+5qky1NVx0h6ysy+L+lRSbWSTpX0xtRx2kfd/bEShnJ26tVpZvdKelzS85L2KdjFeomkCySdEDlvm6TXphKpKIPwbtfNMdvtOu3IOVMyIx6f2bZPZx8xs8IRAQAAAAAAkHxM+5CkKZLelio3S3pPnrYu6bPu/oUyxdIm6cLU62AekHSJu28oUyxQ7rTrphiOfJSCqdd3rw42O2fTGQAAAAAAEBdVPe06zd2H3P1SSW9W7hqQUQ9IOt/drxylzaF4RNI1kp5SkNwcjUu6T9Ilks5097UljgURXf3ZadctMdxwRpKOCO14/cz2vRWMBAAAAAAAICuew7gqxN1/LOnHZnaspOMlzZM0JGmrpIfdfV0R19ogyQps+6Skd0mSmbVJerGkhZJmSGqU1CepU9IGSQ+5+55C48DYhaddN8VwwxlJOiq04/Xq7fs1NOxKJgrqfgAAAAAAAGUTz0xKhaWSgaONgCznvTsl/bYS98bIctd8jOd/mcNntyhh0rBLfYPD2rCrS8tmtlQ6LAAAAAAAUOWYdg2Mwt3V3R/e7Tqe064bapNaPL05U35yC4NjAQAAAABA5ZF8BEbRNzisweHsMpzNMd1wRpJetLAtc/zbp1+oXCAAAAAAAAApJB+BUYRHPUpSU108Rz5K0gVHz84c3/H0CxoYGq5gNAAAAAAAACQfgVGF13uU4rvhjCSdfcRM1dUE/6X39g7q4fW7KxwRAAAAAACodiQfgVF09WeTj421yVjvIN1cX6Mzlk3PlG9btb2C0QAAAAAAAJB8BEbV1Rf/zWbCLlgxJ3P861Xb5e6jtAYAAAAAACgvko/AKPb2DmSOW2K82Uza+UfPyhxv6ezRU8/vq2A0AAAAAACg2pF8BEbR2d2fOW5rqqtgJIWZ1dqgFy1oy5R/zdRrAAAAAABQQSQfgVF0dGVHPrY31VYwksJdsCK76/Wvn9pWwUgAAAAAAEC1I/kIjCI88rG9Of4jHyXpZaHk45Nb9mprZ08FowEAAAAAANWM5CMwio7u8MjHiZF8XD6rRUtmNGfKtz/F1GsAAAAAAFAZJB+BUXSERz5OkGnXZpY79Zp1HwEAAAAAQIWQfARG0Rka+TgRNpxJCycfH1i3K2fXbgAAAAAAgPFC8hEYRe7Ix4mTfDxxYbump9aoHBhy/e6ZHRWOCAAAAAAAVCOSj8AoOrom3rRrSUomTOcdNStTZuo1AAAAAACohJpKBwDEWUfMpl1f/+CmgtvW1yQzx7f9aZuuvX+DahK5f2+4eOXCksUGAAAAAAAQxchHII/egSH1DAxlyu3NE2fkoxTsel2bNElS3+Cw1u/sqnBEAAAAAACg2pB8BPIIbzYjTaw1HyWpriah5TNbMuUnNu+pYDQAAAAAAKAakXwE8ghvNtNQm1BDbXKU1vF0zPypmeM/PtepvT3seg0AAAAAAMYPyUcgj4m603XY8fOnqrUhWNp1aNh192p2vQYAAAAAAOOH5COQR2fMNps5FDXJhM46fGam/NCG3drfN1jBiAAAAAAAQDUh+QjkER75OG2CbTYTdvLiaWquD0Y/Dgy57l2zs8IRAQAAAACAakHyEchjMox8lIKNZ85aPiNTvn/dLnX3M/oRAAAAAACUH8lHII+OrvCajxN35KMkrVwyTY2pDXP6B4d139pdFY4IAAAAAABUA5KPQB4doZGPE3XDmbT62qTOWD49U75v7U71DgxVMCIAAAAAAFANSD4CeXSG1nycyNOu005bOkP1NcF/+d6BYT2wjtGPAAAAAACgvEg+Anns7p48064lqbEuqdOWZUc/3rNmJ2s/AgAAAACAsiL5COTROYmmXaedsWyGapMmSeruH9L1D26qcEQAAAAAAGAyI/kI5NGRM+164o98lKTm+hqtXJId/fj1363N2VgHAAAAAACglEg+AiMYGnbt6Zl8Ix8l6azDs6Mfd3X1619vXlXhiAAAAAAAwGRF8hEYwd6eAblny5Mp+TiloVYXHD07U/7ZH7fojqe3VzAiAAAAAAAwWZF8BEYQnnKdMGlKQ00Foym905fP0IL2xkz5yp89qb29A6OcAQAAAAAAUDySj8AIOkKbzbQ11SmRsApGU3oJM73+xMNUlwy+BWzb26vP3Pp0haMCAAAAAACTDclHYASdoZGP7ZNks5mo2a0N+uB5yzPlHzy0Sfet2VnBiAAAAAAAwGRD8hEYQXjk42Ra7zHqvecu04q5rZnyR3/2uLr7BysYEQAAAAAAmExIPgIjCI98bJvEycfaZEKff+PxSqamlT+3u0f//qtnKhwVAAAAAACYLEg+AiPoqIJp12nHzp+q956zNFP+7n0bdC/TrwEAAAAAQAmQfARGkDPtunnyjnxM++B5h2v5rBZJkrv0Nz/8o7bv7a1wVAAAAAAAYKIj+QiMoKMrPO16co98lKSG2qS+/JYXqa4m+Jawc3+/Pnj9HzU4NFzhyAAAAAAAwERG8hEYQe6068k/8lGSjpk3VZ98zTGZ8kMbdusLtz1bwYgAAAAAAMBER/IRGEFnzm7Xk3/kY9pbTl6g1714fqb8jTvX6jdPba9gRAAAAAAAYCIj+QiMoKNKdruOMjP92+uO1eGp9R8l6W9//Jie291dwagAAAAAAMBERfIRiHD33A1nqij5KElNdTX6+iUnqrE2KUna0zOgD1z/B/UNDlU4MgAAAAAAMNGQfAQiegaG1D+Y3WilmqZdpy2fNUWfef1xmfJjm/fokzetqmBEAAAAAABgIqqpdABA3IRHPUqTe9r19Q9uGvX9UxZP00MbdmfadvUNauWS6XnbX7xyYUnjAwAAAAAAExsjH4GIjq7seo8t9TWqq6ne/yYXHj9XC9obM+WbHtuq9Tu7KhgRAAAAAACYSKo3qwLkEd7puq0Kp1yH1SYTeuvKRZrSEAySHnbp+gc3qjO0IQ8AAAAAAEA+JB+BiPBO19W22cxIWhtrdcnKRUomTJLU1T+k7z2wMWddTAAAAAAAgJGQfAQiwqP6qn3kY9qCaU167YvmZ8pb9/TqZ3/cLHevYFQAAAAAACDuSD4CEbu7stOuGfmYddKidp2+LLvZzOOb9+jOZ3dUMCIAAAAAABB3JB+BiNxp14x8DHvlsXO1dGZzpnzbqu16fHNn5QICAAAAAACxRvIRiMidds3Ix7BkwnTxyQs1rTn7efnp7zdrAztgAwAAAACAEZB8BCI6usPTrhn5GNVUX6N3nLZYjbVJSdLgsOu6BzZq576+CkcGAAAAAADihuQjEBEe+djezMjHkcyYUq9LTs3ugN0zMKTv3r9Bu/aTgAQAAAAAAFkkH4GI8MhHpl3nt2RGs9540mGZ8u6ufr372kfUOzBUwagAAAAAAECckHwEIthwpnAnHNaml6+YnSn/cVOnPvzDRzU07BWMCgAAAAAAxAXJRyBkcGhY+3oHM+V2Rj4e1NlHzNTJi6dlyr/80zZd8dPHNUwCEgAAAACAqkfyEQjp7BnIKbPm48GZmV5zwjwdMbslU/c/f9isf7rxSbmTgAQAAAAAoJqRfARCwpvN1CZNzXXJCkYzcSQTpotPWaSVS7IjIK9/cJP+5eZVJCABAAAAAKhiJB+BkOhmM2ZWwWgmlrqahL7zjpN14sK2TN01927QZ3/5NAlIAAAAAACqFMlHIKSji81mxqKlvkbffdcpOv6wqZm6b965TlfdvrqCUQEAAAAAgEoh+QiEhHe6bmOzmUPS2lCra991io6aMyVT9+XfrNYXfvUMIyABAAAAAKgyJB+BkPC0a0Y+Hrq2pjp9790rtXxWdhOar/x2jT52w5MaYhdsAAAAAACqBslHICQ88rGdkY9jMqOlXte/e6WOnJ0dAXn9g5v0wR/8QX2DQxWMDAAAAAAAjBeSj0BIZ1fuhjMYm1mtDfrRZafqpEXtmbpbn9imd333Ye3vG6xgZAAAAAAAYDyQfARCckc+Mu26FNqa6nTdX52ic4+cmam7d80uXfztB7Rrf18FIwMAAAAAAOVWU+kAgDjpzFnzkZGPxbr+wU1533vpUbPV0dWvxzbvkSQ9vnmPLvjSXbr0tEWaNaVhxHMuXrmwLHECAAAAAIDxwchHICR3t2tGPpZSMmG66CULdNqy6Zm63V39+sada7Xmhf0VjAwAAAAAAJQLyUcgJGe362ZGPpZawkx/ftxcvfyYOZm63oFhffe+9Xpw/a4KRgYAAAAAAMqB5COQ4u7qZLfrsjMznXPETF18ykLVJk2SNOzSjY9u1S2Pb9Wwe4UjBAAAAAAApULyEUjZ3zeoweFs4osNZ8rr2PlT9Z6zlqm1Ibv07L1rd+m6+zeqb2CogpEBAAAAAIBSIfkIpIQ3m5GkqY0kH8ttfnuj3nfucs2bmt1w5pnt+/TNu9blrL8JAAAAAAAmJpKPQEo42dXaUKOaJP89xsPUxlq95+xlWjG3NVO3bW+vvva7tfrDpo4KRgYAAAAAAMaK7AqQsrsrtN4jm82Mq7qahC5euVBnHz4zU9fVN6i3fOsB/eKxrRWMDAAAAAAAjAXJRyAlPO26jc1mxl3CTK84do7ecOJ8JS3YiKZ/cFgf+sEfddXtz8rZiAYAAAAAgAmH5COQ0pGz0zXrPVbKSYum6Z1nLlZjbTJTd9Xtq3XZdb/Xvt6BUc4EAAAAAABxQ/IRSOkIjXxsZ+RjRS2d0aL3nbtMS2c0Z+puW7Vdf/GVe7V6+74KRgYAAAAAAIpB8hFIeWFvb+Z4Gms+VtyMlnr9/PIzdM4R2XUg1+3s0l989V7d8vjzFYwMAAAAAAAUiuQjkLL6hf2Z46Uzm0dpifEytalW//WOk/Wh85Zn6rr7h/T+6/+gT9/6lAaHhisYHQAAAAAAOBiSj4Akd8+Zznv4rCkVjAZhyYTpb192pK6+9CWa0lCTqf/WXet00Tfv1/qdXRWMDgAAAAAAjIbkIyBpx74+7e0dzJQPn9VSwWgwkvNXzNZNHzhTR87OJob/uKlTr/ry3brugY3shg0AAAAAQAyRfASUO+V6Rkud2lnzMZYWz2jWz99/ut5w4mGZup6BIX38hif1jmse1vbQup0AAAAAAKDySD4CElOuJ5Cmuhp98U0n6BuXnKj2ptpM/Z3P7tDLr7pLN/xxC6MgAQAAAACICZKPgHJHPh4+mynXE8Erjp2rX33kbJ131KxMXWf3gD78o0d10Tfu1xOb91QwOgAAAAAAIEk1B28CTH45yUfWe4yN6x/cdNA2Lz1qlqY21OqWJ55Xf2r360c2dug1X7lHJy5q18tWzNaUhmCE5MUrF5Y1XgAAAAAAkIvkI6pedKfr5Uy7nlDMTCcvmaZls1p06xPPa9XzeyVJLun3Gzv05JY9OveImTp12fTKBgoAAAAAQBUi+Yiqt6urXx3dA5ky064npmnNdbrk1EVa88J+3fz4Vr2wr0+S1Dc4rF+t2q671+xUd/+QLj1tUWYkJAAAAAAAKC/WfETVW709O+V6WnOdZrTUVzAajNXyWS364HmH69UnzFNjbTJT390/pH//1TM683O/1VW3P6s9oYQzAAAAAAAoD5KPqHprXghPuWbU42SQTJhOWzpdf3fBETrr8BmqS2a/1e3pGdBVt6/WmZ+7Q5/536e0fW9vBSMFAAAAAGByI/mIqsdmM5NXU32NXnnsXP3Dy4/UuUfOVEt9dqWJfX2D+uad63Tm5+7Q3//kMT0bWvcTAAAAAACUBslHVL3wtGuSj5NTc32NXrZiju796Hn68PmHq7Uhm4QcGHL99Peb9bIv3aV3ffdhPbBul9y9gtECAAAAADB5sOEMql7OyMfZ7HQ9mU1tqtWHzz9C7z5rqX740CZ95571en5Pdtr1HU+/oDuefkEnHDZV7zl7mV5x7BwlE1bBiAEAAAAAmNhIPqKqdXT1a+f+vkyZkY+T2/UPbsocN9XV6PJzl+vxzZ26e/VObQut/fjY5j16//V/0LTmOp25fIZOXNiuupoDB4pfvHLhuMQNAAAAAMBERfIRVS086nFqY61mTmGn62qSTJhevLBdL1rQptUv7Nfdq3do7Y6uzPu7u/r1i8e26vantuvM5TO0csl0NdYlR7kiAAAAAAAII/mIqrY6tNP14bNaZMYU22pkZjpi9hQdMXuKtnT26O7VO/TE5j1Kr/zY3T+k21Zt153P7tCpS6frjOUzcjavAQAAAAAAI+O3Z1S1nM1mZjPlGtL8tka95eSFevmKft2zdqce2bBbA0NBGrJvcFh3PrtD967ZqZMXT9M5R87U/LbGCkcMAAAAAEB8sds1qtqa0LTr5bPYbAZZ7c11evXx8/QPLz9Kf3bkTDXUZr9dDg677l+3S+d8/re64qePad2O/aNcCQAAAACA6sXIR1S16LRrIKqlvkYXrJijsw6fqQfX79Y9a3aqq29QUpCE/PEjm/WT32/Wq46bq8vPXaZj5k2tcMQAAAAAAMQHyUdUrT09A9q+N7TTNdOuMYqG2qTOOWKmTl82XY9s7NDdz+5QZ8+AJMlduuXx53XL48/rjOXT9bZTF+v8o2epJsngcgAAAABAdSP5iKoVnnI9pb5Gc1obKhgNJoraZEKnLZ2uUxZPU2NdUl/73RqtC+2Qfe+aXbp3zS7Nm9qgt566SG8+eYFmtLCLOgAAAACgOjEsB1Vr9fbslOvls9npGsVJJkxvPOkw/foj5+jrbz1Rx85vzXl/655e/fuvntHpn7lDH7j+D/rNU9s1MDRcoWgBAAAAAKgMRj6iaq0OjXxkvUccqmTC9Mrj5uoVx87RIxs7dO39G/W/TzyvweFgh+z+oWHd/Pjzuvnx59XeVKsLj5+r175ovk5a1E7CGwAAAAAw6ZF8RNXKTT6y0zWKd/2Dmw6oO23pdB0zr1UPb9ith9bv1r7ewcx7Hd0D+t4Dm/S9BzapralWK+a2asXcVi2a3qxkwnTxyoXjGT4AAAAAAGVH8hFVa01k2jVQKq0NtXrpUbN17hGz9My2fXp0c6eefn5vZjSkJHV2D+i+tbt039pdaqxN6ui5UzStuU5nHj5DLfV8awYAAAAATA78houqtK93QFv39GbKTLtGOSQTphXzWrViXqt6B4b0p6179OhznVq3o0seatczMKQ/bOrUe7/3e9UkTCctatc5R87U2YfP1Iq5rUokmJ4NAAAAAJiYSD6iKq0N7U7cXJfU/LbGCkaDatBQm9RJi6bppEXTtLd3QE89v1dPPb9Xa3d0aSg0InJw2PXg+t16cP1uff6Xz2hGS73OWD5dpy4NXounN7FWJAAAAABgwiD5iKqUs9P1LHa6xvhqbajVyiXTtXLJdPUODOnZ7fu06vm92rirW3t6BnLa7tzfpxsf3aobH90qSZrT2qBTl07TKUum6+TF7Vo2s4WRkQAAAACA2CL5iKq0JrTZzHI2m0EFNdQmdfxhbTr+sDa9+eQFemxzp+56dofufHaHHnuuU8Oe237b3l7d8OhW3ZBKRrY11eqkhe16yeJpesnidh03f6oaapMV+EgAAAAAADgQyUdUpWdDIx8PZ7MZxMSPHn5OkjRrSoMuOmmBLjxurtbu6NL6nfu1bkeXXtjXd8A5nd0D+s3TL+g3T78gKVhn8rC2Ri2a3qRLT1uskxa1q725blw/DgAAAAAA0kg+jsDMjpF0vKR5koYkbZH0iLuvH+c4EpJOl7RM0lxJe1Kx3O3uHeMZy2Syp2dAj2zMfvrYbAZx1VRXo+PmT9Vx86dKkvb3DWr9zi6t27FfG3d1a/veXkUGRmpo2LVxd7c27u7WXat3SgqWFnjJomB05MmL27VwGutGAgAAAADGB8nHEDN7o6SPK0g8jvT+fZI+5u6/K3McNZI+KulyBQnQqH4zu0nS37v7hnLGMhn91z3rta93UJI0pb5GJy+ZVuGIgMK01OcmI3sHhrRpd7c27urShl3d2tzRrYGhaDoyWGZgzQv79cPUyMoZLfU6eXG7Tl06XSuXTtMRs6awbiQAAAAAoCxIPkoys6SkqyW94yBNT5f0GzP7tLt/vEyxzJZ0s6SXjNKsTtIbJF1gZpe6+43liGUy2tM9oP+6JzuA9V1nLlFrQ20FIwIOXUNtUkfMnqIjZgfrlg4Nu7Z29gQjH3d1afvePu3cf+BU7Z37+/S/T27T/z65TZLU3lSrkxdP08ql0/XihW1aMbeVdSMBAAAAACVB8jHwJeUmHrslfV/SowoSfSsVJPtqJSUk/ZOZ7Xb3L5UyCDNrlHSjchOPWyR9T9JaSdMlvVLS2an3WiX90MzOc/f7SxnLZPWde9ZpX19q1GNDjd515pIKRwSUTjJhWjCtSQumNenM5TP0l6cs0MZd3XpkY4ce2bBbj2zsyNlsKa2je0C3rdqu21ZtlyTVJk0r5rbqRQva9KKFbTpi9hQtndGixjoSkgAAAACA4lR98tHMLpT0wVDVKkmvcPfnIu1OkHSrstOgv2Bmt7v7EyUM518UJDrTfirpEncPD136rJldLOm7CpKhDZJ+ZGZHuHtvCWOZdDq7+/Vf927IlN995lJNbWTUIyavHzyU/TaW3lG7u29QG3d3a/3OLq3f2aWtnT0HrBs5MOR6bPMePbZ5j/77/o2Z+vltjVo2q0XLZjZrQXuT5kxt0OzWes2a0qBZrfWqryE5CQAAAADIVdXJx9SGLp8OVXVLenU08ShJ7v6YmV0k6W4Fox/T5766RLEcJukDoarHJV3s7gMjxHK9mS2U9JlU1QJJ75f0xVLEMlldffd67U+NemxtqNE7z1xc2YCACmiqr9HRc1t19NxWScG6kRt2dWn9ji5t2t2tLZ09Ghw+cN1ISdrS2aMtnT2669kdI74/tbFWbU21amusVWtjbaY8tbFWbY11mtpYq6mpcmtDraY01KilvkYtDTWqTSbK9jEDAAAAACqnqpOPkl6q3M1l/sPd1+Vr7O73mdlPJL05VfXnZrbc3deUIJb3KRjFmHbFSInHkC8oSFbOT5U/LJKPee3u6tc192bXenzP2UtZ6xFQsG7kUXNaddScIBk5NOzatrdXz+0ONrDZ2tmrnfv78iYkw/b0DGhPz4A2HrTlgeprEjnJyJb6GrXU16qlPqnGuho11ibVUJtQY21SjXVJNdQmU3VJNdYlcsuhNk11SRKbAAAAAFBB1Z58fF2kfHUB53xb2eSjJL1WQSKwlLFslHTbaI3dfdDMrpH0T6mqw8zsJe7+SAlimXS+ffc6dfUPSZLammr19tMXVzYgIKaSCdP8tkbNb2tUsMysNOyuzu4B7djXqx37+rRjf5/29Axob8+g9vYOqDv1f2ss+gaH1be/Xzv394/5WlEt9TVqa6rVtOY6tTXVaVpTbfBvc53aQ8dtTbVqb6pTe1Md61sCAAAAQIlUe/LxwtDxWndfW8A5d0vqVXaU4p9rjMlHM1si6ehQ1e3ufvBhRtKvlU0+pmMh+Rixa3+f/vu+DZnyX5+1VFMY9QgULGGmac1Bgu7IOQe+Pzg0rL29g+rqG1TPwJB6+ofUMzCk7v4h9abK3Zn6QfX0DwXJxsHhcYl/f9+g9vcNanNHT8Hn1Nck1N4USkg2B0nKtsagPKUhO0IzGLFZq+b6pKak/q1htCUAAAAASKri5KOZtUlaGKp6oJDz3L3fzH4v6YxU1fGjtS/QCZFyQbFIekjSoLJfx1LEMul86+51mZFZ7Yx6BEquJpnIJCeLMeyu/lQSsncglZAcGFJv6t++wWH1Dg5pcMjVPzSsgcFhDQwNa2DIU/8Gx/3p48FhDQy7BgaHD9hEp1h9g8PatrdX2/Ye2j5ejbXJ0PTx7HTyKfU1aq6PJi6Duik5U86D48bapMxsjB8NAAAAAFRO1SYflTvSUJKKWbdxrbLJx3Yzm+Pu28Y7FnfvNbOtyiZRV4whhklp5/4+XXtfdgW6y85Zppb6au72QHwkzNSQWqexlDvPu7uGhl0DQ66+wWAEZnf/kLr6B4PjvsFMuSdc3z+k/hKNxuwZCEZ/7tjXN6brmEkNNak1LGsSaqhLZsupNTDrU+tc1tUkVJdMqCZhqq1JqDaZUF3SVJsMjmtrEqpNWOY4/F5N0lSXbpdMqK4meC+ZMNUkgn/Tr5rIMclRAAAAAKOp5izM0kh5UxHnRtsulTSW5ONYY0knH6PXqXr/fd8G9QwEox6nN9fp0tMWVTgiAOVmZqpJmmqSUmNdUm1NhZ87ODSs7oF0wnJQ3X2paeOhBGX3wFB2ZGZ6xObgkAaGxjre8kDu2URmXCVMqkkklEho1ERlupxIJSvTSct06tIseAV1lq1TtkG4bfpcS9UnzGQ28r+JTNmUDMWZjq8maTmJ1pqc+BOh9w+sz9aFzs1Xn/r40x9rIhV7ULYRPx5JSiRC7RR8PEodj3id1OfNFfQhueRyuafrPDM6OL3ISxBbcH7ClBNrwkzJTNwkmwEAAFCcak4+tkbKu4s4tyNSnhKTWGrNrN7dxzbUZhJ537nBSMdv3rVOl52zVE111dzlARxMTTKh1mRCrYewLuzQcDCNvHcwO4U8naAM6odHTFr2DQxnjnsHgn8L2Fw8NoZd6h8aloYkaXzW8UTlRJORmeRuKlmZTvQmEqFjMyUSoeNQvYUSnenkZ/5EaKocOk6k2tsIx+lzw23zGW2lbR9lIYd0Qjd7jVSS17PnZpO+oevlSQRH63ISx56NJXzfzNcm8nUK6iynPNJ7kX9SbQ7848BIbdIJ85G+7tGvRbiPZL+GwR8E0sfpr1n485v5nB1Qp4LaKafdoV8n53KpymJiiCb8M21D10/H5zl12bY550ZiyNc/sudkzz9Y27xx5dwrG0e0v+Zcq5i4Ivca6WsY/qNUug+m6zXCH67Sf7TJHIf7f+QPWdnjkesVvVbe61vkGtk/FkWvW6himmc/99HvLz7C1+DAfhFtE/6elb5ubmy53y+in6fwe/m+N1n4ZEW/BnnOidw/G0/+9tn7HRjz6PfIvn/A98O83y8Pcq0ivqj5mmb/BxTavrzXH7ltnmsUHUuR1y/yOvlOGKm2mM/XzCn1umDF7Hx3rSrVnIlpiZSLWdgrumtB9FqVjqWg5KOZ/SnPW0etXbtWxxxzTBFhxNuwS1f9QPpycT/nx2xPz8D43hDApBBNQmR+gfARfjlIt0m/n7lG6JfFTL3nlLPHB/7CO+aFMwEAAIAq1liX0GHtRUzDirG1a9dK0oJDPb+ak48NkXJ/EedGk3uNkygWSRru6+vrWrVq1XMluFY1W5b6t5Bd1FFd6BvIh76BfOgbyIe+gXzoG8iHvoF86BslNCBp1fOVjqJkFkjqPtSTqzn5GB1dWMw2rfWRcnT0YSliKXT04yHH4u6TZ2hjDKVHlvJ5RhR9A/nQN5APfQP50DeQD30D+dA3kA99A+WSqHQAFbQ/Uo6OPhxNdHRh9FoTORYAAAAAAACgJKo5+bg3Um4v4ty2SHnf2EIpWSwDbDYDAAAAAACAuKjm5OP6SHlhEecuipTXxSSWscYBAAAAAAAAlEw1Jx9XRcrLizh3Wei4w923VSIWM2uQNG+U6wAAAAAAAAAVU7XJR3fvlLQpVHVaIeeZWZ2kk0JVT5QgnMci5YJikXSKcjcNKkUsAAAAAAAAQElU827XknSrpPemjpeZ2VJ3P9jU5bOUuyHMzWMNwt3Xm9nTko5KVZ1vZubufpBTL4iUxxwLSocdwpAPfQP50DeQD30D+dA3kA99A/nQN5APfQPlUrUjH1N+Hin/dQHnRNvcUJpQcmJZJOllozU2sxpJ7wxVbZH0SIliAQAAAAAAAMas2pOPt0t6MlT+oJktydfYzE6TdFGo6hZ3X52n7WIz89DrdweJ5euSwjtVf97Makdp//eS5ofKVxUwUhIAAAAAAAAYN1WdfHT3YUlXhqqaJd1kZguibc3seEk/UfZzNizpYyWM5TlJXw1VHS/p+2ZWP0Isfynpk6GqLZK+UqpYAAAAAAAAgFIwBstJZvZVSZeHqrokfV/So5JqJZ0q6Y2p47R/cPcvjHLNxZLWh6rudPdzDxJHk6Q7Jb0kVL1F0nWS1klql/QqSeeE3u+TdL673zPatQEAAAAAAIDxRvJRkpklJV0j6W0FNHdJn3X3K0drdCjJx9R5cyTdIunEAmLZJ+nt7h5duxIAAAAAAACouKqedp3m7kPufqmkNyt3DcioBxSMMhw18TjGWLYpGGn5CUnb8jTrV7BBzQkkHgEAAAAAABBXjHwcgZkdq2DNxXmShiRtlfSwu68b5ziSkk6XtFzSbAUjHTdLutvdd49nLAAAAAAAAECxSD4CIWZ2jHITz1skPeLu60c9sfRxJBQknpdJmitpTyqWu929YzxjQaDSfcPM6iQdLWmFpDmSmiTtlbQ9Fce4/nEEWZXuG4ivuPUNM2tV8LNlnqRZkvZLeiEV16Pu3lWJuKpRXPqGmS1TsNTPXElTJPVI2iXpcUlPuPvgeMaD+OBZFFE8iwIYC5KPgCQze6Okjyv4RWAk90n6mLv/rsxx1Ej6qIINkOaN0KRf0k2S/t7dN5QzFgQq2TfMbL6Cza5eJelMBQ95+ayR9DVJX3P3vlLHggPF5ftGPmb2fklfiVR/0t3/uQLhVJW49Q0zO0vBz5YLJNXlaTakYHmZj7n7neMRVzWKQ99Izax5n6T3SzpqlKY7Jf23pE8z46Z8Ukm+oxVsOJl+nSCpMdTsz8bx+wXPojERh77Bs2g8xaFvFIJnUYSRfERVSz2AXy3pHQU0H1bwAP7xMsUyW9LNyt3tPJ+9ki519xvLEQsq3zfM7GWSfinJijz1T5Le5O6rShULclW6bxTCzA6TtErBSKYwHvjKKG59w8yaFDz0v0OFfy/5B3f/QrliqlZx6RtmNkvBxoaFPGukvSDpDe5+T6njqXZm9j+SXi6p+SBNxyWJwLNofMShb/AsGk9x6BuF4FkUUTWVDgCosC8p9xeBbknfl/SogtEhKyW9QVKtgg2a/snMdrv7l0oZhJk1SrpRuQ97WyR9T9JaSdMlvVLS2an3WiX90MzOc/f7SxkLMirdN5qU+7A3LOkxSXdL2iipQ1K7gg2q/kLZ0UzHSLrDzM509zUligW5Kt03CvF1Hfiwh/KLTd8ws2YFSaZzQtU9kn6jYITjdklJBVPnXiTpPAU/W1AeFe8bqSmTv1buqMs+Sb9Q0Cd2S2qRdJyCkU7TUm1mSfpfM1tJMqHkTtLBEwjjgmfR2IlD3+BZNJ7i0DcKwbMocrk7L15V+ZJ0oSQPvf4kacEI7U5Q8PCVbjck6bgSx/LvkVh+Iql+hHYXK5jukm63SVJDpT+Xk+0Vh74h6bWpa65TMP1p7ihtFyqYqheO+a5Kfx4n4ysOfaOAGN8Suu+qSLz/XOnP4WR9xa1vSLo1Es+1kmaN0r5W0uskvaLSn8vJ9opL35B0RSSORyUtydN2iqQfRdr/utKfy8n2krQh9PntlfSQgl/Yr4t87s8dh1h4Fo3RKw59QzyLxvIVh75RQIw8i/I64JUQUIVS62R8OlTVLenV7v5ctK27PybpIgV/7ZOCEQmfjrYbQyyHSfpAqOpxSRf7CGuluPv1kj4RqlqgYM0mlEiM+sYLki6TdKS7f87dn8/X0N03KZh+8Uyo+iwzOzvPKTgEMeobo8U4XdKXU8VeSR8q9z0Rv75hZn+lYIRS2ufd/VJ3fyHfOe4+4O4/d/dfljKWahezvvH20HFPKo71IzV0932S3qrgmSTtpWY20hqAOHTXSnqPgpFMU9z9FHd/n4IRyuOGZ9FYikPf4Fk0nuLQN/LiWRT5kHxEtXqpcqcd/YePskObu9+n4C/AaX9uZstLFMv7JDWEyle4+8Ao7b+gYGRE2odLFAcCsegb7n6fu3/rIH0h3H6fpE9Gqv98rHEgRyz6xkF8ScEUSUn6lILF31F+sekbZjZFwc+JtAck/Z9SXBuHJBZ9w8waFOxQm3bzSAnQSCyDkr4dvozyb5SDQ+Dun3D3b7v7Hwr9eV8mPIvGTBz6Bs+i8RSHvnEQPItiRCQfUa1eFylfXcA5346UX1uaUHJi2SjpttEap34ZuCZUdZiZFbNwPEYXp75RrNsj5WUViWLyinXfSC0M/7ZUcZWkz5frXjhAnPrGJZLaQuUr3H04T1uUX1z6xvRIudBfBldHytNGbIWJjmdRlArPolWMZ1GMhuQjqtWFoeO17r62gHPuVjB0PG3Mf8kzsyWSjg5V3e4eLJRxEL+OlPmrYunEom8cov2R8kRYjHoiiW3fSG0u8s1U0SVdFtO/hk9Wceob7wkdP+Pud5foujg0cekbnQq+N6QV+vOhJVLOO3UfExPPoigxnkWrFM+iOBiSj6g6ZtamYFHktAcKOc/d+yX9PlRViqlHJ0TKBcWiYGHhwRLHUvVi1jcOxZJIeVtFopiEJkDf+JSkxanjq939njLdBxFx6htmNkPBztVpt471mjh0ceob7t6lYJfatPMKPPWloeP0xgaYXHgWRSnxLFq9eBbFqEg+ohodHSkXsw5FeMRCu5nNqUQs7t4raWuoakW+tihKnPrGoXh9pHx/BWKYrGLbN8zsFGUX896uYEdKjJ849Y1TIuX7pWDxdzP7iJndY2bPm1lf6t/7zOxTZnb4GO+LkcWpb0jSf4aOjzWzUTcJMbOTJb0rVPUtd99bgjgQLzyLopR4Fq1CPIuiECQfUY2WRsqbijg32jZ6rUrFMtY4EIhT3yiKmbVIujxU1S/pxvGMYZKLZd8ws1pJ31H25/lH3L2jVNdHQeLUN14cKT9tZm+Q9LSk/yfpDElzJNWl/j1N0sckPWVmXzOz+jHeH7ni1DekYI2+8M+F/0x93Y8KNzKzOWZ2haTfSkr3iYckXVmCGBA/PIuiJHgWrU48i6JQJB9RjVoj5d1FnBv9RjolJrHU8ktjScSpbxTri5LmhsrfcHemupROXPvGP0o6NnV8m7v/oITXRmHi1DdmRsrnKtg5eUaq7JJ2SHpe0lCoXVLBbre/MbPGMcaArDj1DaXW8XuTpKsUTJc1BV/3p8xsj5mtN7N0//icgrXaBiR9XdJLU1O3MfnwLIpS4Vm0OvEsioKQfEQ1ii6e3jtiq5H1HORaEzkWTNCvh5ldqtxNJjZJ+vh43b9KxK5vmNnRCkatpe/xvlJcF0WLU99oi5S/qCDB1CfpnyXNd/dZ7j5Pwe7Hlys30XCGgkQTSiNOfUNSsJ6ku39EwS+Kd4bealWwVteMUN0mSa9198vdPbqJBCaP2PVTTDw8i1YnnkVRDJKPqEYNkXJ/Eef2RcpjHSESp1gwAb8eZnaOpG+HqgYkvYV1uUouVn3DzEzB1z09yuRf3H3dWK+LQxKnvhH9xb9WwfeEV7n7J939+fQb7r7H3b8u6UxJu0LnvD211h/GLk59Q5JkZgkz+4ikuySdc5DmCyXdYma/NjOm1E5eseunmFh4Fq1OPIuiWCQfUY2if9GtK+Lc6HSS6F98J3IsmGBfDzM7SdIvlI3TJb3T3Vncu/Ti1jcuVzBKTZKeUDDCDZURp74x0oilL7r7HflOcPenJP1tpPrDY4wDgTj1DZlZg6SbFaz/OStVfbuk1yqYKlknqV1BUvLbyk7NP1/SI2Z24lhjQCzFqp9iYuFZtKrxLIqikHxENYpOHYr+xXc00b/ojnUaUpxiwQT6epjZcZJ+pdy1mi539++X875VLDZ9w8wWSPpMquiSLnP3gbFcE2MSm74haV+k7JL+o4DzrlewO2Xa+WOMA4E49Q1J+rKkV4bKV7r7Be5+o7tvc/cBd+9097vc/T2SXqZsYqpd0s9SG0pgcolbP8UEwbNo9eJZFIeC5COqUXQKQHsR57ZFytFf9IpVqlgG3D069QXFi1PfyCu1M+ntCtZsS/uwu3+jXPdErPrG15XdfOIbjC6ouDj1jWgsT4enWufj7oOS7glVzTKzw8YYC2LUN1Lrcv11qOoX7v6ZfO0lKTVi9mOhqkWSLhtLHIglnkVRNJ5Fqx7PoigayUdUo/WR8sIizl0UKY91XYtSxcL6GqURp74xIjM7XNIdyk6Zk6R/dPcvl+N+yIhF3zCz10i6MFXcJun/HOq1UDKx6BspayPlTUWcuzFSju6cjeLFqW+8RcHmQ2lfKfC8byp3DcDXjzEOxA/PoigKz6LVjWdRHKqaSgcAVMCqSHl5EecuCx13uPu2MsRy50gNw1LrNs0b5To4NHHqGwdILfh/h4K1udI+4e6fK/W9cIC49I3wpg9Nkn4frPedV/Tn/IfM7JJQ+VPu/t0xxIP49A1J+lOkXMyutdG2xUy9xMji1DeOj5QfKeQkd+8ys6dD5x8zxjgQPzyLomA8i0I8i+IQkXxE1XH3TjPbpOxfdk8r5Dwzq5N0UqjqiRKE81ikfJqk7xRw3inK/f9biliqXsz6RvQeiyT9VlJ4KuSn3P1fS30vHCimfaNVuessFaJduVPq2koWTZWKWd94UsEmIclUeVoR50bb7hqxFQoWs77RHCkXszZfV+iY3YwnH55FURCeRTECnkVRMKZdo1rdGjpelvor3sGcpdyRIDePNQh3Xy/p6VDV+XaQPx2lXBApjzkWZMSib4Sl1l67Q7lToT7n7h8v5X1wULHrG4iNWPQNd9+j3BFLx5tZoc96Lw4dD0jaPNZ4ICkmfUNSR6Q8p4hzwyOcSEpPMjyLohA8iwIYK5KPqFY/j5T/esRWo7e5oTSh5MSySMHuknmZWY2kd4aqtqjA6VMoSJz6hsxsroKHvfAvrP/P3f+xVPdAwSreN9z9Kne3Ql+SlkQu8clIm6vGEg8yKt43Qn4aOp6qg/xMkSQzWyLp5FDVA+7eXaJ4ql1c+saaSDmaOBpRam23xaGqZ0sQC+KHZ1HkxbMowngWxaEi+YhqdbuC6WlpH0z98jUiMztN0kWhqlvcfXWetovNzEOv3x0klq9LCu8O+Hkzqx2l/d9Lmh8qX+XufpB7oHCx6RtmNjMVz+Gh6v9w97872AeBsohN30DsxKlvXCdpe6j82dQ03tF8UbnPhP99kPYoXFz6xi8j5SvNbMqILXNF13H7VQHnoMJ4FkU+PIsiH55FUW4kH1GV3H1Y0pWhqmZJN5nZgmhbMzte0k+U/f8yLOljJYzlOUlfDVUdL+n7ZlY/Qix/KemToaotKnzHShQgLn3DzNol/VrSilD119z9b0pxfRQvLn0D8ROnvuHu+yX931DVCZJ+lvqeEo2l3sy+Kul1oepnJV1bqniqXVz6hrvfLenhUNUySbemplIewMyazOxq5faNvZK+XYp4EC88i2IkPIsCKCU2nEHVcvebzOxrki5PVR0j6Skz+76kRyXVSjpV0htTx2kfdffo4txj9XFJZ0t6Sap8kaTTzew6SesULMr7KknnhM7pk/QWdy9mN1MUICZ94wMKkgZhrzCz6NS50Wx293NLFA8Um76BGIpZ3/iWgp8Xf5kqXyhpjZn9WNLjkgYVjGJ5k4Iplmn7Jb3B3QdKHE9Vi1HfuEzSXZJaUuUzFfSLX0h6UMF6js0KEk9vkDQ9cv7fuPvOEsZT9czs9ZI+P8Jb0VGp3zeznhHaXeHuPytRODyLxkhM+gbPojEUk74BFI3kI6rdhxR8o35bqtws6T152rqkz7r7F0odhLt3m9mrJd0i6cRU9XxJ+dZS2Sfp7e5+T6ljQUal+0ZyhLpCNioI43t8eVS6byC+YtE33N3N7B0KRtC9OVU9TdJ7Rzlti6TXufuTo7TBoat433D3P5rZhZJ+qOwmMvUKkkwX5T1R6pX0EXf/binjgaRgl9hlBbSbN8r5JcGzaOzEoW/wLBpPcegbQNGYdo2q5u5D7n6pgl/ORvuF6wFJ57v7laO0GWss2xSMfPiEpG15mvUrWBT8BHePLmKPEopT30C80DeQT5z6hrv3u/tbFIxufHSUpnsUjKA4wd0fHqUdxiAufcPd75J0rKR/U/5njbRuSddIerG7f6Mc8SBeeBYFAJSLsTYwkGVmxyqYbjRP0pCkrZIedvd14xxHUtLpkpZLmq3gr8ubJd3t7rvHMxYE4tI3ED/0DeQTp75hZkdIenEqljoFU2xXSXrI3QfHO55qF4e+YWYm6WhJL5I0U8HIzB5JuxX0jUfdvS/vBTCp8SwKACglko8AAAAAAAAAyoJp1wAAAAAAAADKguQjAAAAAAAAgLIg+QgAAAAAAACgLEg+AgAAAAAAACgLko8AAAAAAAAAyoLkIwAAAAAAAICyIPkIAAAAAAAAoCxIPgIAAAAAAAAoC5KPAAAAAAAAAMqC5CMAAAAAAACAsiD5CAAAAAAAAKAsSD4CAAAAAAAAKAuSjwAAAAAAAADKguQjAAAAAAAAgLIg+QgAAAAAAACMwMwSZnaMmb3dzP7TzO43s24z89Dr3ErHmWZmGyKxHcrrd6WMqaaUFwMAAAAAAAAmAzP7H0kvl9Rc6VjGWWcpL0byEQAAAAAAADjQSZp4iccNkgaLPGeepMZQ+Qcli0YkHwEAAAAAAICD6ZP0uKTfS2qRdEllwxmZu59bTHszq5e0Rdnk4y5JN5QyJpKPAAAAAAAAwIGulfScgoTjE+4+IElm9g7FNPl4CF4raXqofJ2795XyBiQfAQAAAAAAgAh3/8R43cvMTNKJklZImiXJJG2X9Ad3/1MZb/3uSPk7pb4ByUcAAAAAAACgAsxsiqSPKkgCzs7TZrWk/+vuJV2L0cwWS3ppqOpBd3+ylPeQpESpLwgAAAAAAABgdGZ2qqTVkj6mPInHlMMlXW9mPzaz2hKG8C4FIyzTri7htTMY+QgAAAAAAACMIzP7M0k3S2oKVT+TqlurYMfqIyW9SdKC1PsXSXJJby7B/ROS3hGq6pL0o7FedyQkHwEAAAAAAIBxYmazJP1A2cRjr6T3S7rG3T3S9uOSviTpslTVm8zsZne/boxhvEzZpKYk/cjd943xmiNi2jUAAAAAAAAwfj6r7DTrYUmvc/f/iiYeJcnde9z9vZL+J1T9r6mRi2MR3WimLFOuJZKPAAAAAAAAwLgwszmS3hqqutrdf1nAqR+SNJA6XiTpVWOIYaak14SqVrn7/Yd6vYMh+QgAAAAAAACMjzdKqguVv1TISe6+VdLtoaoLxhDDpZLCG9d8ZwzXOiiSjwAAAAAAAMD4OCt0vM7dny7i3IdCxyvHEMO7Qsf9kq4dw7UOiuQjAAAAAAAAMD5OCB3/qchzt4eODzuUm5vZaZJWhKpudPedh3KtQrHbNQAAAAAAADA+poeOX21mB2wyU6D2Qzxv3DaaSWPkIwAAAAAAADA+2kp0naZiTzCzFklvClVtVO46kmXByEcAAAAAAABgfHRLak0dd0jaPY73foukllD5GncfLvdNST4CAAAAAAAA42OnssnHn7j7ZeN4778KHQ9LumY8bsq0awAAAAAAAGB8hHe3Pma8bmpmx0g6NVR1m7tvGo97k3wEAAAAAAAAxsdvQ8enmtmMcbrvX0XK3xmn+5J8BAAAAAAAAMbJTyUNpo6Tkv6h3Dc0szpJbwtV7ZB0Y7nvm0byEQAAAAAAABgH7r5B0g9CVX9rZi8r5hoWqCvilL+QFB5hea27DxRzz7Eg+QgAAAAAAACMnyskPZ86rpF0k5n9nZk1jHaSmc01sw8qWDfyxCLuV7Ep15Jk7j6e9wMAAAAAAABiz8xeL+nzI7w1RdKsUHmrpJ4R2l3h7j/Lc+3TJP1S2Z2vpWAn7F9JelTSbgXTstskHaEg2fhiSZZqe5q7P1DAx7BQ0nplByDe5+5nHOy8UqoZz5sBAAAAAAAAE0SrpGUFtJs3yvkjcvf7zexUSTcoSC5KwdTot6ZeBzNUQBtJeqdyZz5fXeB5JcO0awAAAAAAAGCcuftTko6V9F5Jqwo4ZZWkL0p6sbs/fLDGZmYKko9p+yT9+BBCHROmXQMAAAAAAAAVZmbzJZ0qabakdkn9kjokrZX0pLvvqGB4h4zkIwAAAAAAAICyYNo1AAAAAAAAgLIg+QgAAAAAAACgLEg+AgAAAAAAACgLko8AAAAAAAAAyoLkIwAAAAAAAICyIPkIAAAAAAAAoCxIPgIAAAAAAAAoC5KPAAAAAAAAAMqC5CMAAAAAAACAsiD5CAAAAAAAAKAsSD4CAAAAAAAAKAuSjwAAAAAAAADKguQjAAAAAAAAgLIg+QgAAAAAAACgLEg+AgAAAAAAACgLko8AAAAAAAAAyoLkIwAAAAAAAICyIPkIAAAAAAAAoCxIPgIAAAAAAAAoC5KPAAAAAAAAAMri/wMZEmFf1HC9MAAAAABJRU5ErkJggg==\n",
-      "text/plain": [
-       "<Figure size 1500x750 with 1 Axes>"
-      ]
-     },
-     "metadata": {
-      "needs_background": "light"
-     },
-     "output_type": "display_data"
-    }
-   ],
-   "source": [
-    "gene_detection_counts = [i for i in gene_detection_counts_dict.values()]\n",
-    "import seaborn as sns\n",
-    "import matplotlib.pyplot as plt\n",
-    "plt.figure(figsize=(10,5), dpi=150)\n",
-    "plt.rcParams.update({'font.size': 18})\n",
-    "count_plot = sns.distplot(gene_detection_counts).set_title(f\"# Cells Expressing Each\\nProtein-Coding or miRNA Gene\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 47,
-   "id": "missing-bradley",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "27454"
-      ]
-     },
-     "execution_count": 47,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "len(gene_detection_counts)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 55,
-   "id": "perfect-signal",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "25424"
-      ]
-     },
-     "execution_count": 55,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "len([i for i in gene_detection_counts if i > 0])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 56,
-   "id": "faced-theory",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "22735"
-      ]
-     },
-     "execution_count": 56,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "len([i for i in gene_detection_counts if i > 100])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 57,
-   "id": "tough-workplace",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "21167"
-      ]
-     },
-     "execution_count": 57,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "len([i for i in gene_detection_counts if i > 1000])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 49,
-   "id": "cooperative-camcorder",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "173152.0299000284"
-      ]
-     },
-     "execution_count": 49,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "gene_detection_event_digest = crick.tdigest.TDigest()\n",
-    "gene_detection_event_digest.update(gene_detection_counts)\n",
-    "gene_detection_event_digest.quantile(0.5)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.11"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

examples/pretraining_new_model/pretrain_geneformer_w_deepspeed.py

@@ -1,167 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-
-# run with:
-# deepspeed --num_gpus=12 --num_nodes=3 pretrain_geneformer_w_deepspeed.py --deepspeed ds_config.json
-
-import datetime
-
-# imports
-import os
-
-os.environ["NCCL_DEBUG"] = "INFO"
-os.environ["OMPI_MCA_opal_cuda_support"] = "true"
-os.environ["CONDA_OVERRIDE_GLIBC"] = "2.56"
-
-import pickle
-import random
-import subprocess
-
-import numpy as np
-import pytz
-import torch
-from datasets import load_from_disk
-from transformers import BertConfig, BertForMaskedLM, TrainingArguments
-
-from geneformer import GeneformerPretrainer
-
-seed_num = 0
-random.seed(seed_num)
-np.random.seed(seed_num)
-seed_val = 42
-torch.manual_seed(seed_val)
-torch.cuda.manual_seed_all(seed_val)
-
-# set local time/directories
-timezone = pytz.timezone("US/Eastern")
-rootdir = "/parent_ouput_directory"
-
-# set model parameters
-# model type
-model_type = "bert"
-# max input size
-max_input_size = 2**11  # 2048
-# number of layers
-num_layers = 6
-# number of attention heads
-num_attn_heads = 4
-# number of embedding dimensions
-num_embed_dim = 256
-# intermediate size
-intermed_size = num_embed_dim * 2
-# activation function
-activ_fn = "relu"
-# initializer range, layer norm, dropout
-initializer_range = 0.02
-layer_norm_eps = 1e-12
-attention_probs_dropout_prob = 0.02
-hidden_dropout_prob = 0.02
-
-
-# set training parameters
-# total number of examples in Genecorpus-30M after QC filtering:
-num_examples = 27_406_208
-# number gpus
-num_gpus = 12
-# batch size for training and eval
-geneformer_batch_size = 12
-# max learning rate
-max_lr = 1e-3
-# learning schedule
-lr_schedule_fn = "linear"
-# warmup steps
-warmup_steps = 10_000
-# number of epochs
-epochs = 3
-# optimizer
-optimizer = "adamw"
-# weight_decay
-weight_decay = 0.001
-
-
-# output directories
-current_date = datetime.datetime.now(tz=timezone)
-datestamp = f"{str(current_date.year)[-2:]}{current_date.month:02d}{current_date.day:02d}_{current_date.strftime('%X').replace(':','')}"
-run_name = f"{datestamp}_geneformer_30M_L{num_layers}_emb{num_embed_dim}_SL{max_input_size}_E{epochs}_B{geneformer_batch_size}_LR{max_lr}_LS{lr_schedule_fn}_WU{warmup_steps}_O{optimizer}_DS{num_gpus}"
-training_output_dir = f"{rootdir}/models/{run_name}/"
-logging_dir = f"{rootdir}/runs/{run_name}/"
-model_output_dir = os.path.join(training_output_dir, "models/")
-
-
-# ensure not overwriting previously saved model
-model_output_file = os.path.join(model_output_dir, "pytorch_model.bin")
-if os.path.isfile(model_output_file) is True:
-    raise Exception("Model already saved to this directory.")
-
-
-# make training and model output directories
-subprocess.call(f"mkdir {training_output_dir}", shell=True)
-subprocess.call(f"mkdir {model_output_dir}", shell=True)
-
-
-# load gene_ensembl_id:token dictionary (e.g. https://huggingface.co/datasets/ctheodoris/Genecorpus-30M/blob/main/token_dictionary.pkl)
-with open("token_dictionary.pkl", "rb") as fp:
-    token_dictionary = pickle.load(fp)
-
-# model configuration
-config = {
-    "hidden_size": num_embed_dim,
-    "num_hidden_layers": num_layers,
-    "initializer_range": initializer_range,
-    "layer_norm_eps": layer_norm_eps,
-    "attention_probs_dropout_prob": attention_probs_dropout_prob,
-    "hidden_dropout_prob": hidden_dropout_prob,
-    "intermediate_size": intermed_size,
-    "hidden_act": activ_fn,
-    "max_position_embeddings": max_input_size,
-    "model_type": model_type,
-    "num_attention_heads": num_attn_heads,
-    "pad_token_id": token_dictionary.get("<pad>"),
-    "vocab_size": len(token_dictionary),  # genes+2 for <mask> and <pad> tokens
-}
-
-config = BertConfig(**config)
-model = BertForMaskedLM(config)
-model = model.train()
-
-# define the training arguments
-training_args = {
-    "learning_rate": max_lr,
-    "do_train": True,
-    "do_eval": False,
-    "group_by_length": True,
-    "length_column_name": "length",
-    "disable_tqdm": False,
-    "lr_scheduler_type": lr_schedule_fn,
-    "warmup_steps": warmup_steps,
-    "weight_decay": weight_decay,
-    "per_device_train_batch_size": geneformer_batch_size,
-    "num_train_epochs": epochs,
-    "save_strategy": "steps",
-    "save_steps": np.floor(
-        num_examples / geneformer_batch_size / 8
-    ),  # 8 saves per epoch
-    "logging_steps": 1000,
-    "output_dir": training_output_dir,
-    "logging_dir": logging_dir,
-}
-training_args = TrainingArguments(**training_args)
-
-print("Starting training.")
-
-# define the trainer
-trainer = GeneformerPretrainer(
-    model=model,
-    args=training_args,
-    # pretraining corpus (e.g. https://huggingface.co/datasets/ctheodoris/Genecorpus-30M/tree/main/genecorpus_30M_2048.dataset)
-    train_dataset=load_from_disk("genecorpus_30M_2048.dataset"),
-    # file of lengths of each example cell (e.g. https://huggingface.co/datasets/ctheodoris/Genecorpus-30M/blob/main/genecorpus_30M_2048_lengths.pkl)
-    example_lengths_file="genecorpus_30M_2048_lengths.pkl",
-    token_dictionary=token_dictionary,
-)
-
-# train
-trainer.train()
-
-# save model
-trainer.save_model(model_output_dir)

examples/tokenizing_scRNAseq_data.ipynb

@@ -1,91 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "a91bca46-c056-4784-8c6c-b0f5d3f33496",
-   "metadata": {
-    "tags": []
-   },
-   "source": [
-    "## Tokenizing .loom or .h5ad single cell RNA-seq data to rank value encoding .dataset format"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "1fe86f48-5578-47df-b373-58c21ec170ab",
-   "metadata": {},
-   "source": [
-    "#### Input data is a directory with .loom or .h5ad files containing raw counts from single cell RNAseq data, including all genes detected in the transcriptome without feature selection. The input file type is specified by the argument file_format in the tokenize_data function.\n",
-    "\n",
-    "#### The discussion below references the .loom file format, but the analagous labels are required for .h5ad files, just that they will be column instead of row attributes and vice versa due to the transposed format of the two file types.\n",
-    "\n",
-    "#### Genes should be labeled with Ensembl IDs (loom row attribute \"ensembl_id\"), which provide a unique identifer for conversion to tokens. Other forms of gene annotations (e.g. gene names) can be converted to Ensembl IDs via Ensembl Biomart. Cells should be labeled with the total read count in the cell (loom column attribute \"n_counts\") to be used for normalization.\n",
-    "\n",
-    "#### No cell metadata is required, but custom cell attributes may be passed onto the tokenized dataset by providing a dictionary of custom attributes to be added, which is formatted as loom_col_attr_name : desired_dataset_col_attr_name. For example, if the original .loom dataset has column attributes \"cell_type\" and \"organ_major\" and one would like to retain these attributes as labels in the tokenized dataset with the new names \"cell_type\" and \"organ\", respectively, the following custom attribute dictionary should be provided: {\"cell_type\": \"cell_type\", \"organ_major\": \"organ\"}. \n",
-    "\n",
-    "#### Additionally, if the original .loom file contains a cell column attribute called \"filter_pass\", this column will be used as a binary indicator of whether to include these cells in the tokenized data. All cells with \"1\" in this attribute will be tokenized, whereas the others will be excluded. One may use this column to indicate QC filtering or other criteria for selection for inclusion in the final tokenized dataset.\n",
-    "\n",
-    "#### If one's data is in other formats besides .loom or .h5ad, one can use the relevant tools (such as Anndata tools) to convert the file to a .loom or .h5ad format prior to running the transcriptome tokenizer."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "32c69493-4e5a-4b07-8dc1-958ff2ee7d0b",
-   "metadata": {},
-   "source": [
-    "**********************************************************************************************************\n",
-    "#### OF NOTE: PLEASE ENSURE THE CORRECT TOKEN DICTIONARY AND GENE MEDIAN FILE IS USED FOR THE CORRECT MODEL.\n",
-    "#### 95M: current defaults; 30M: https://huggingface.co/ctheodoris/Geneformer/tree/main/geneformer/gene_dictionaries_30m\n",
-    "\n",
-    "#### ADDITIONALLY:\n",
-    "#### The 95M model series require the special_token argument to be set to True and model_input_size to be 4096. (current defaults)\n",
-    "#### The 30M model series require the special_token argument to be set to False and the model_input_size to be 2048."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "080fdd9c-0c48-4d5d-a254-52b6c53cdf78",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from geneformer import TranscriptomeTokenizer"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "37205758-aa52-4443-a383-0638519ee8a9",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "tk = TranscriptomeTokenizer({\"cell_type\": \"cell_type\", \"organ_major\": \"organ\"}, nproc=16)\n",
-    "tk.tokenize_data(\"loom_data_directory\", \n",
-    "                 \"output_directory\", \n",
-    "                 \"output_prefix\", \n",
-    "                 file_format=\"loom\")"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.15"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

geneformer/__init__.py

@@ -1,34 +0,0 @@
-# ruff: noqa: F401
-import warnings
-from pathlib import Path
-
-warnings.filterwarnings("ignore", message=".*The 'nopython' keyword.*")  # noqa # isort:skip
-
-GENE_MEDIAN_FILE = Path(__file__).parent / "gene_median_dictionary_gc95M.pkl"
-TOKEN_DICTIONARY_FILE = Path(__file__).parent / "token_dictionary_gc95M.pkl"
-ENSEMBL_DICTIONARY_FILE = Path(__file__).parent / "gene_name_id_dict_gc95M.pkl"
-ENSEMBL_MAPPING_FILE = Path(__file__).parent / "ensembl_mapping_dict_gc95M.pkl"
-
-from . import (
-    collator_for_classification,
-    emb_extractor,
-    in_silico_perturber,
-    in_silico_perturber_stats,
-    pretrainer,
-    tokenizer,
-)
-from .collator_for_classification import (
-    DataCollatorForCellClassification,
-    DataCollatorForGeneClassification,
-)
-from .emb_extractor import EmbExtractor, get_embs
-from .in_silico_perturber import InSilicoPerturber
-from .in_silico_perturber_stats import InSilicoPerturberStats
-from .pretrainer import GeneformerPretrainer
-from .tokenizer import TranscriptomeTokenizer
-
-from . import classifier  # noqa # isort:skip
-from .classifier import Classifier  # noqa # isort:skip
-
-from . import mtl_classifier  # noqa # isort:skip
-from .mtl_classifier import MTLClassifier  # noqa # isort:skip

geneformer/classifier.py

@@ -1,1563 +0,0 @@
-"""
-Geneformer classifier.
-
-**Input data:**
-
-| Cell state classifier:
-| Single-cell transcriptomes as Geneformer rank value encodings with cell state labels in Geneformer .dataset format (generated from single-cell RNAseq data by tokenizer.py)
-
-| Gene classifier:
-| Dictionary in format {Gene_label: list(genes)} for gene labels and single-cell transcriptomes as Geneformer rank value encodings in Geneformer .dataset format (generated from single-cell RNAseq data by tokenizer.py)
-
-**Usage:**
-
-.. code-block :: python
-
-    >>> from geneformer import Classifier
-    >>> cc = Classifier(classifier="cell",  # example of cell state classifier
-    ...                 cell_state_dict={"state_key": "disease", "states": "all"},
-    ...                 filter_data={"cell_type":["Cardiomyocyte1","Cardiomyocyte2","Cardiomyocyte3"]},
-    ...                 training_args=training_args,
-    ...                 freeze_layers = 2,
-    ...                 num_crossval_splits = 1,
-    ...                 forward_batch_size=200,
-    ...                 nproc=16)
-    >>> cc.prepare_data(input_data_file="path/to/input_data",
-    ...                 output_directory="path/to/output_directory",
-    ...                 output_prefix="output_prefix")
-    >>> all_metrics = cc.validate(model_directory="path/to/model",
-    ...                           prepared_input_data_file=f"path/to/output_directory/{output_prefix}_labeled.dataset",
-    ...                           id_class_dict_file=f"path/to/output_directory/{output_prefix}_id_class_dict.pkl",
-    ...                           output_directory="path/to/output_directory",
-    ...                           output_prefix="output_prefix",
-    ...                           predict_eval=True)
-    >>> cc.plot_conf_mat(conf_mat_dict={"Geneformer": all_metrics["conf_matrix"]},
-    ...                  output_directory="path/to/output_directory",
-    ...                  output_prefix="output_prefix",
-    ...                  custom_class_order=["healthy","disease1","disease2"])
-    >>> cc.plot_predictions(predictions_file=f"path/to/output_directory/datestamp_geneformer_cellClassifier_{output_prefix}/ksplit1/predictions.pkl",
-    ...                     id_class_dict_file=f"path/to/output_directory/{output_prefix}_id_class_dict.pkl",
-    ...                     title="disease",
-    ...                     output_directory="path/to/output_directory",
-    ...                     output_prefix="output_prefix",
-    ...                     custom_class_order=["healthy","disease1","disease2"])
-"""
-
-import datetime
-import logging
-import os
-import pickle
-import subprocess
-from pathlib import Path
-
-import numpy as np
-import pandas as pd
-import seaborn as sns
-from tqdm.auto import tqdm, trange
-from transformers import Trainer
-from transformers.training_args import TrainingArguments
-
-from . import (
-    TOKEN_DICTIONARY_FILE,
-    DataCollatorForCellClassification,
-    DataCollatorForGeneClassification,
-)
-from . import classifier_utils as cu
-from . import evaluation_utils as eu
-from . import perturber_utils as pu
-
-sns.set()
-
-
-logger = logging.getLogger(__name__)
-
-
-class Classifier:
-    valid_option_dict = {
-        "classifier": {"cell", "gene"},
-        "quantize": {bool, dict},
-        "cell_state_dict": {None, dict},
-        "gene_class_dict": {None, dict},
-        "filter_data": {None, dict},
-        "rare_threshold": {int, float},
-        "max_ncells": {None, int},
-        "max_ncells_per_class": {None, int},
-        "training_args": {None, dict},
-        "freeze_layers": {int},
-        "num_crossval_splits": {0, 1, 5},
-        "split_sizes": {None, dict},
-        "no_eval": {bool},
-        "stratify_splits_col": {None, str},
-        "forward_batch_size": {int},
-        "token_dictionary_file": {None, str},
-        "nproc": {int},
-        "ngpu": {int},
-    }
-
-    def __init__(
-        self,
-        classifier=None,
-        quantize=False,
-        cell_state_dict=None,
-        gene_class_dict=None,
-        filter_data=None,
-        rare_threshold=0,
-        max_ncells=None,
-        max_ncells_per_class=None,
-        training_args=None,
-        ray_config=None,
-        freeze_layers=0,
-        num_crossval_splits=1,
-        split_sizes={"train": 0.8, "valid": 0.1, "test": 0.1},
-        stratify_splits_col=None,
-        no_eval=False,
-        forward_batch_size=100,
-        token_dictionary_file=None,
-        nproc=4,
-        ngpu=1,
-    ):
-        """
-        Initialize Geneformer classifier.
-
-        **Parameters:**
-
-        classifier : {"cell", "gene"}
-            | Whether to fine-tune a cell state or gene classifier.
-        quantize : bool, dict
-            | Whether to fine-tune a quantized model.
-            | If True and no config provided, will use default.
-            | Will use custom config if provided.
-            | Configs should be provided as dictionary of BitsAndBytesConfig (transformers) and LoraConfig (peft).
-            | For example: {"bnb_config": BitsAndBytesConfig(...),
-            |               "peft_config": LoraConfig(...)}
-        cell_state_dict : None, dict
-            | Cell states to fine-tune model to distinguish.
-            | Two-item dictionary with keys: state_key and states
-            | state_key: key specifying name of column in .dataset that defines the states to model
-            | states: list of values in the state_key column that specifies the states to model
-            | Alternatively, instead of a list of states, can specify "all" to use all states in that state key from input data.
-            | Of note, if using "all", states will be defined after data is filtered.
-            | Must have at least 2 states to model.
-            | For example: {"state_key": "disease",
-            |               "states": ["nf", "hcm", "dcm"]}
-            |               or
-            |               {"state_key": "disease",
-            |               "states": "all"}
-        gene_class_dict : None, dict
-            | Gene classes to fine-tune model to distinguish.
-            | Dictionary in format: {Gene_label_A: list(geneA1, geneA2, ...),
-            |                        Gene_label_B: list(geneB1, geneB2, ...)}
-            | Gene values should be Ensembl IDs.
-        filter_data : None, dict
-            | Default is to fine-tune with all input data.
-            | Otherwise, dictionary specifying .dataset column name and list of values to filter by.
-        rare_threshold : float
-            | Threshold below which rare cell states should be removed.
-            | For example, setting to 0.05 will remove cell states representing
-            | < 5% of the total cells from the cell state classifier's possible classes.
-        max_ncells : None, int
-            | Maximum number of cells to use for fine-tuning.
-            | Default is to fine-tune with all input data.
-        max_ncells_per_class : None, int
-            | Maximum number of cells per cell class to use for fine-tuning.
-            | Of note, will be applied after max_ncells above.
-            | (Only valid for cell classification.)
-        training_args : None, dict
-            | Training arguments for fine-tuning.
-            | If None, defaults will be inferred for 6 layer Geneformer.
-            | Otherwise, will use the Hugging Face defaults:
-            | https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments
-            | Note: Hyperparameter tuning is highly recommended, rather than using defaults.
-        ray_config : None, dict
-            | Training argument ranges for tuning hyperparameters with Ray.
-        freeze_layers : int
-            | Number of layers to freeze from fine-tuning.
-            | 0: no layers will be frozen; 2: first two layers will be frozen; etc.
-        num_crossval_splits : {0, 1, 5}
-            | 0: train on all data without splitting
-            | 1: split data into train and eval sets by designated split_sizes["valid"]
-            | 5: split data into 5 folds of train and eval sets by designated split_sizes["valid"]
-        split_sizes : None, dict
-            | Dictionary of proportion of data to hold out for train, validation, and test sets
-            | {"train": 0.8, "valid": 0.1, "test": 0.1} if intending 80/10/10 train/valid/test split
-        stratify_splits_col : None, str
-            | Name of column in .dataset to be used for stratified splitting.
-            | Proportion of each class in this column will be the same in the splits as in the original dataset.
-        no_eval : bool
-            | If True, will skip eval step and use all data for training.
-            | Otherwise, will perform eval during training.
-        forward_batch_size : int
-            | Batch size for forward pass (for evaluation, not training).
-        token_dictionary_file : None, str
-            | Default is to use token dictionary file from Geneformer
-            | Otherwise, will load custom gene token dictionary.
-        nproc : int
-            | Number of CPU processes to use.
-        ngpu : int
-            | Number of GPUs available.
-
-        """
-
-        self.classifier = classifier
-        if self.classifier == "cell":
-            self.model_type = "CellClassifier"
-        elif self.classifier == "gene":
-            self.model_type = "GeneClassifier"
-        self.quantize = quantize
-        self.cell_state_dict = cell_state_dict
-        self.gene_class_dict = gene_class_dict
-        self.filter_data = filter_data
-        self.rare_threshold = rare_threshold
-        self.max_ncells = max_ncells
-        self.max_ncells_per_class = max_ncells_per_class
-        self.training_args = training_args
-        self.ray_config = ray_config
-        self.freeze_layers = freeze_layers
-        self.num_crossval_splits = num_crossval_splits
-        self.split_sizes = split_sizes
-        self.train_size = self.split_sizes["train"]
-        self.valid_size = self.split_sizes["valid"]
-        self.oos_test_size = self.split_sizes["test"]
-        self.eval_size = self.valid_size / (self.train_size + self.valid_size)
-        self.stratify_splits_col = stratify_splits_col
-        self.no_eval = no_eval
-        self.forward_batch_size = forward_batch_size
-        self.token_dictionary_file = token_dictionary_file
-        self.nproc = nproc
-        self.ngpu = ngpu
-
-        if self.training_args is None:
-            logger.warning(
-                "Hyperparameter tuning is highly recommended for optimal results. "
-                "No training_args provided; using default hyperparameters."
-            )
-
-        self.validate_options()
-
-        if self.filter_data is None:
-            self.filter_data = dict()
-
-        if self.classifier == "cell":
-            if self.cell_state_dict["states"] != "all":
-                self.filter_data[
-                    self.cell_state_dict["state_key"]
-                ] = self.cell_state_dict["states"]
-
-        # load token dictionary (Ensembl IDs:token)
-        if self.token_dictionary_file is None:
-            self.token_dictionary_file = TOKEN_DICTIONARY_FILE
-        with open(self.token_dictionary_file, "rb") as f:
-            self.gene_token_dict = pickle.load(f)
-
-        self.token_gene_dict = {v: k for k, v in self.gene_token_dict.items()}
-
-        # filter genes for gene classification for those in token dictionary
-        if self.classifier == "gene":
-            all_gene_class_values = set(pu.flatten_list(self.gene_class_dict.values()))
-            missing_genes = [
-                gene
-                for gene in all_gene_class_values
-                if gene not in self.gene_token_dict.keys()
-            ]
-            if len(missing_genes) == len(all_gene_class_values):
-                logger.error(
-                    "None of the provided genes to classify are in token dictionary."
-                )
-                raise
-            elif len(missing_genes) > 0:
-                logger.warning(
-                    f"Genes to classify {missing_genes} are not in token dictionary."
-                )
-            self.gene_class_dict = {
-                k: list(set([self.gene_token_dict.get(gene) for gene in v]))
-                for k, v in self.gene_class_dict.items()
-            }
-            empty_classes = []
-            for k, v in self.gene_class_dict.items():
-                if len(v) == 0:
-                    empty_classes += [k]
-            if len(empty_classes) > 0:
-                logger.error(
-                    f"Class(es) {empty_classes} did not contain any genes in the token dictionary."
-                )
-                raise
-
-    def validate_options(self):
-        # confirm arguments are within valid options and compatible with each other
-        for attr_name, valid_options in self.valid_option_dict.items():
-            attr_value = self.__dict__[attr_name]
-            if not isinstance(attr_value, (list, dict)):
-                if attr_value in valid_options:
-                    continue
-            valid_type = False
-            for option in valid_options:
-                if (option in [int, float, list, dict, bool, str]) and isinstance(
-                    attr_value, option
-                ):
-                    valid_type = True
-                    break
-            if valid_type:
-                continue
-            logger.error(
-                f"Invalid option for {attr_name}. "
-                f"Valid options for {attr_name}: {valid_options}"
-            )
-            raise
-
-        if self.filter_data is not None:
-            for key, value in self.filter_data.items():
-                if not isinstance(value, list):
-                    self.filter_data[key] = [value]
-                    logger.warning(
-                        "Values in filter_data dict must be lists. "
-                        f"Changing {key} value to list ([{value}])."
-                    )
-
-        if self.classifier == "cell":
-            if set(self.cell_state_dict.keys()) != set(["state_key", "states"]):
-                logger.error(
-                    "Invalid keys for cell_state_dict. "
-                    "The cell_state_dict should have only 2 keys: state_key and states"
-                )
-                raise
-
-            if self.cell_state_dict["states"] != "all":
-                if not isinstance(self.cell_state_dict["states"], list):
-                    logger.error(
-                        "States in cell_state_dict should be list of states to model."
-                    )
-                    raise
-                if len(self.cell_state_dict["states"]) < 2:
-                    logger.error(
-                        "States in cell_state_dict should contain at least 2 states to classify."
-                    )
-                    raise
-
-        if self.classifier == "gene":
-            if len(self.gene_class_dict.keys()) < 2:
-                logger.error(
-                    "Gene_class_dict should contain at least 2 gene classes to classify."
-                )
-                raise
-        if sum(self.split_sizes.values()) != 1:
-            logger.error("Train, validation, and test proportions should sum to 1.")
-            raise
-
-    def prepare_data(
-        self,
-        input_data_file,
-        output_directory,
-        output_prefix,
-        split_id_dict=None,
-        test_size=None,
-        attr_to_split=None,
-        attr_to_balance=None,
-        max_trials=100,
-        pval_threshold=0.1,
-    ):
-        """
-        Prepare data for cell state or gene classification.
-
-        **Parameters**
-
-        input_data_file : Path
-            | Path to directory containing .dataset input
-        output_directory : Path
-            | Path to directory where prepared data will be saved
-        output_prefix : str
-            | Prefix for output file
-        split_id_dict : None, dict
-            | Dictionary of IDs for train and test splits
-            | Three-item dictionary with keys: attr_key, train, test
-            | attr_key: key specifying name of column in .dataset that contains the IDs for the data splits
-            | train: list of IDs in the attr_key column to include in the train split
-            | test: list of IDs in the attr_key column to include in the test split
-            | For example: {"attr_key": "individual",
-            |               "train": ["patient1", "patient2", "patient3", "patient4"],
-            |               "test": ["patient5", "patient6"]}
-        test_size : None, float
-            | Proportion of data to be saved separately and held out for test set
-            | (e.g. 0.2 if intending hold out 20%)
-            | If None, will inherit from split_sizes["test"] from Classifier
-            | The training set will be further split to train / validation in self.validate
-            | Note: only available for CellClassifiers
-        attr_to_split : None, str
-            | Key for attribute on which to split data while balancing potential confounders
-            | e.g. "patient_id" for splitting by patient while balancing other characteristics
-            | Note: only available for CellClassifiers
-        attr_to_balance : None, list
-            | List of attribute keys on which to balance data while splitting on attr_to_split
-            | e.g. ["age", "sex"] for balancing these characteristics while splitting by patient
-            | Note: only available for CellClassifiers
-        max_trials : None, int
-            | Maximum number of trials of random splitting to try to achieve balanced other attributes
-            | If no split is found without significant (p<0.05) differences in other attributes, will select best
-            | Note: only available for CellClassifiers
-        pval_threshold : None, float
-            | P-value threshold to use for attribute balancing across splits
-            | E.g. if set to 0.1, will accept trial if p >= 0.1 for all attributes in attr_to_balance
-        """
-
-        if test_size is None:
-            test_size = self.oos_test_size
-
-        # prepare data and labels for classification
-        data = pu.load_and_filter(self.filter_data, self.nproc, input_data_file)
-
-        if self.classifier == "cell":
-            if "label" in data.features:
-                logger.error(
-                    "Column name 'label' must be reserved for class IDs. Please rename column."
-                )
-                raise
-        elif self.classifier == "gene":
-            if "labels" in data.features:
-                logger.error(
-                    "Column name 'labels' must be reserved for class IDs. Please rename column."
-                )
-                raise
-
-        if (attr_to_split is not None) and (attr_to_balance is None):
-            logger.error(
-                "Splitting by attribute while balancing confounders requires both attr_to_split and attr_to_balance to be defined."
-            )
-            raise
-
-        if not isinstance(attr_to_balance, list):
-            attr_to_balance = [attr_to_balance]
-
-        if self.classifier == "cell":
-            # remove cell states representing < rare_threshold of cells
-            data = cu.remove_rare(
-                data, self.rare_threshold, self.cell_state_dict["state_key"], self.nproc
-            )
-            # downsample max cells and max per class
-            data = cu.downsample_and_shuffle(
-                data, self.max_ncells, self.max_ncells_per_class, self.cell_state_dict
-            )
-            # rename cell state column to "label"
-            data = cu.rename_cols(data, self.cell_state_dict["state_key"])
-
-        # convert classes to numerical labels and save as id_class_dict
-        # of note, will label all genes in gene_class_dict
-        # if (cross-)validating, genes will be relabeled in column "labels" for each split
-        # at the time of training with Classifier.validate
-        data, id_class_dict = cu.label_classes(
-            self.classifier, data, self.gene_class_dict, self.nproc
-        )
-
-        # save id_class_dict for future reference
-        id_class_output_path = (
-            Path(output_directory) / f"{output_prefix}_id_class_dict"
-        ).with_suffix(".pkl")
-        with open(id_class_output_path, "wb") as f:
-            pickle.dump(id_class_dict, f)
-
-        if split_id_dict is not None:
-            data_dict = dict()
-            data_dict["train"] = pu.filter_by_dict(
-                data, {split_id_dict["attr_key"]: split_id_dict["train"]}, self.nproc
-            )
-            data_dict["test"] = pu.filter_by_dict(
-                data, {split_id_dict["attr_key"]: split_id_dict["test"]}, self.nproc
-            )
-            train_data_output_path = (
-                Path(output_directory) / f"{output_prefix}_labeled_train"
-            ).with_suffix(".dataset")
-            test_data_output_path = (
-                Path(output_directory) / f"{output_prefix}_labeled_test"
-            ).with_suffix(".dataset")
-            data_dict["train"].save_to_disk(str(train_data_output_path))
-            data_dict["test"].save_to_disk(str(test_data_output_path))
-        elif (test_size is not None) and (self.classifier == "cell"):
-            if 1 > test_size > 0:
-                if attr_to_split is None:
-                    data_dict = data.train_test_split(
-                        test_size=test_size,
-                        stratify_by_column=self.stratify_splits_col,
-                        seed=42,
-                    )
-                    train_data_output_path = (
-                        Path(output_directory) / f"{output_prefix}_labeled_train"
-                    ).with_suffix(".dataset")
-                    test_data_output_path = (
-                        Path(output_directory) / f"{output_prefix}_labeled_test"
-                    ).with_suffix(".dataset")
-                    data_dict["train"].save_to_disk(str(train_data_output_path))
-                    data_dict["test"].save_to_disk(str(test_data_output_path))
-                else:
-                    data_dict, balance_df = cu.balance_attr_splits(
-                        data,
-                        attr_to_split,
-                        attr_to_balance,
-                        test_size,
-                        max_trials,
-                        pval_threshold,
-                        self.cell_state_dict["state_key"],
-                        self.nproc,
-                    )
-                    balance_df.to_csv(
-                        f"{output_directory}/{output_prefix}_train_test_balance_df.csv"
-                    )
-                    train_data_output_path = (
-                        Path(output_directory) / f"{output_prefix}_labeled_train"
-                    ).with_suffix(".dataset")
-                    test_data_output_path = (
-                        Path(output_directory) / f"{output_prefix}_labeled_test"
-                    ).with_suffix(".dataset")
-                    data_dict["train"].save_to_disk(str(train_data_output_path))
-                    data_dict["test"].save_to_disk(str(test_data_output_path))
-            else:
-                data_output_path = (
-                    Path(output_directory) / f"{output_prefix}_labeled"
-                ).with_suffix(".dataset")
-                data.save_to_disk(str(data_output_path))
-                print(data_output_path)
-        else:
-            data_output_path = (
-                Path(output_directory) / f"{output_prefix}_labeled"
-            ).with_suffix(".dataset")
-            data.save_to_disk(str(data_output_path))
-
-    def train_all_data(
-        self,
-        model_directory,
-        prepared_input_data_file,
-        id_class_dict_file,
-        output_directory,
-        output_prefix,
-        save_eval_output=True,
-        gene_balance=False,
-    ):
-        """
-        Train cell state or gene classifier using all data.
-
-        **Parameters**
-
-        model_directory : Path
-            | Path to directory containing model
-        prepared_input_data_file : Path
-            | Path to directory containing _labeled.dataset previously prepared by Classifier.prepare_data
-        id_class_dict_file : Path
-            | Path to _id_class_dict.pkl previously prepared by Classifier.prepare_data
-            | (dictionary of format: numerical IDs: class_labels)
-        output_directory : Path
-            | Path to directory where model and eval data will be saved
-        output_prefix : str
-            | Prefix for output files
-        save_eval_output : bool
-            | Whether to save cross-fold eval output
-            | Saves as pickle file of dictionary of eval metrics
-        gene_balance : None, bool
-            | Whether to automatically balance genes in training set.
-            | Only available for binary gene classifications.
-
-        **Output**
-
-        Returns trainer after fine-tuning with all data.
-
-        """
-
-        if (gene_balance is True) and (len(self.gene_class_dict.values()) != 2):
-            logger.error(
-                "Automatically balancing gene sets for training is only available for binary gene classifications."
-            )
-            raise
-
-        ##### Load data and prepare output directory #####
-        # load numerical id to class dictionary (id:class)
-        with open(id_class_dict_file, "rb") as f:
-            id_class_dict = pickle.load(f)
-        class_id_dict = {v: k for k, v in id_class_dict.items()}
-
-        # load previously filtered and prepared data
-        data = pu.load_and_filter(None, self.nproc, prepared_input_data_file)
-        data = data.shuffle(seed=42)  # reshuffle in case users provide unshuffled data
-
-        # define output directory path
-        current_date = datetime.datetime.now()
-        datestamp = f"{str(current_date.year)[-2:]}{current_date.month:02d}{current_date.day:02d}"
-        if output_directory[-1:] != "/":  # add slash for dir if not present
-            output_directory = output_directory + "/"
-        output_dir = f"{output_directory}{datestamp}_geneformer_{self.classifier}Classifier_{output_prefix}/"
-        subprocess.call(f"mkdir {output_dir}", shell=True)
-
-        # get number of classes for classifier
-        num_classes = cu.get_num_classes(id_class_dict)
-
-        if self.classifier == "gene":
-            targets = pu.flatten_list(self.gene_class_dict.values())
-            labels = pu.flatten_list(
-                [
-                    [class_id_dict[label]] * len(targets)
-                    for label, targets in self.gene_class_dict.items()
-                ]
-            )
-            assert len(targets) == len(labels)
-            data = cu.prep_gene_classifier_all_data(
-                data, targets, labels, self.max_ncells, self.nproc, gene_balance
-            )
-
-        trainer = self.train_classifier(
-            model_directory, num_classes, data, None, output_dir
-        )
-
-        return trainer
-
-    def validate(
-        self,
-        model_directory,
-        prepared_input_data_file,
-        id_class_dict_file,
-        output_directory,
-        output_prefix,
-        split_id_dict=None,
-        attr_to_split=None,
-        attr_to_balance=None,
-        gene_balance=False,
-        max_trials=100,
-        pval_threshold=0.1,
-        save_eval_output=True,
-        predict_eval=True,
-        predict_trainer=False,
-        n_hyperopt_trials=0,
-        save_gene_split_datasets=True,
-        debug_gene_split_datasets=False,
-    ):
-        """
-        (Cross-)validate cell state or gene classifier.
-
-        **Parameters**
-
-        model_directory : Path
-            | Path to directory containing model
-        prepared_input_data_file : Path
-            | Path to directory containing _labeled.dataset previously prepared by Classifier.prepare_data
-        id_class_dict_file : Path
-            | Path to _id_class_dict.pkl previously prepared by Classifier.prepare_data
-            | (dictionary of format: numerical IDs: class_labels)
-        output_directory : Path
-            | Path to directory where model and eval data will be saved
-        output_prefix : str
-            | Prefix for output files
-        split_id_dict : None, dict
-            | Dictionary of IDs for train and eval splits
-            | Three-item dictionary with keys: attr_key, train, eval
-            | attr_key: key specifying name of column in .dataset that contains the IDs for the data splits
-            | train: list of IDs in the attr_key column to include in the train split
-            | eval: list of IDs in the attr_key column to include in the eval split
-            | For example: {"attr_key": "individual",
-            |               "train": ["patient1", "patient2", "patient3", "patient4"],
-            |               "eval": ["patient5", "patient6"]}
-            | Note: only available for CellClassifiers with 1-fold split (self.classifier="cell"; self.num_crossval_splits=1)
-        attr_to_split : None, str
-            | Key for attribute on which to split data while balancing potential confounders
-            | e.g. "patient_id" for splitting by patient while balancing other characteristics
-            | Note: only available for CellClassifiers with 1-fold split (self.classifier="cell"; self.num_crossval_splits=1)
-        attr_to_balance : None, list
-            | List of attribute keys on which to balance data while splitting on attr_to_split
-            | e.g. ["age", "sex"] for balancing these characteristics while splitting by patient
-        gene_balance : None, bool
-            | Whether to automatically balance genes in training set.
-            | Only available for binary gene classifications.
-        max_trials : None, int
-            | Maximum number of trials of random splitting to try to achieve balanced other attribute
-            | If no split is found without significant (p < pval_threshold) differences in other attributes, will select best
-        pval_threshold : None, float
-            | P-value threshold to use for attribute balancing across splits
-            | E.g. if set to 0.1, will accept trial if p >= 0.1 for all attributes in attr_to_balance
-        save_eval_output : bool
-            | Whether to save cross-fold eval output
-            | Saves as pickle file of dictionary of eval metrics
-        predict_eval : bool
-            | Whether or not to save eval predictions
-            | Saves as a pickle file of self.evaluate predictions
-        predict_trainer : bool
-            | Whether or not to save eval predictions from trainer
-            | Saves as a pickle file of trainer predictions
-        n_hyperopt_trials : int
-            | Number of trials to run for hyperparameter optimization
-            | If 0, will not optimize hyperparameters
-        save_gene_split_datasets : bool
-            | Whether or not to save train, valid, and test gene-labeled datasets
-        """
-        if self.num_crossval_splits == 0:
-            logger.error("num_crossval_splits must be 1 or 5 to validate.")
-            raise
-
-        if (gene_balance is True) and (len(self.gene_class_dict.values()) != 2):
-            logger.error(
-                "Automatically balancing gene sets for training is only available for binary gene classifications."
-            )
-            raise
-
-        # ensure number of genes in each class is > 5 if validating model
-        if self.classifier == "gene":
-            insuff_classes = [k for k, v in self.gene_class_dict.items() if len(v) < 5]
-            if (self.num_crossval_splits > 0) and (len(insuff_classes) > 0):
-                logger.error(
-                    f"Insufficient # of members in class(es) {insuff_classes} to (cross-)validate."
-                )
-                raise
-
-        ##### Load data and prepare output directory #####
-        # load numerical id to class dictionary (id:class)
-        with open(id_class_dict_file, "rb") as f:
-            id_class_dict = pickle.load(f)
-        class_id_dict = {v: k for k, v in id_class_dict.items()}
-
-        # load previously filtered and prepared data
-        data = pu.load_and_filter(None, self.nproc, prepared_input_data_file)
-        data = data.shuffle(seed=42)  # reshuffle in case users provide unshuffled data
-
-        # define output directory path
-        current_date = datetime.datetime.now()
-        datestamp = f"{str(current_date.year)[-2:]}{current_date.month:02d}{current_date.day:02d}"
-        if output_directory[-1:] != "/":  # add slash for dir if not present
-            output_directory = output_directory + "/"
-        output_dir = f"{output_directory}{datestamp}_geneformer_{self.classifier}Classifier_{output_prefix}/"
-        subprocess.call(f"mkdir {output_dir}", shell=True)
-
-        # get number of classes for classifier
-        num_classes = cu.get_num_classes(id_class_dict)
-
-        ##### (Cross-)validate the model #####
-        results = []
-        all_conf_mat = np.zeros((num_classes, num_classes))
-        iteration_num = 1
-        if self.classifier == "cell":
-            for i in trange(self.num_crossval_splits):
-                print(
-                    f"****** Validation split: {iteration_num}/{self.num_crossval_splits} ******\n"
-                )
-                ksplit_output_dir = os.path.join(output_dir, f"ksplit{iteration_num}")
-                if self.num_crossval_splits == 1:
-                    # single 1-eval_size:eval_size split
-                    if split_id_dict is not None:
-                        data_dict = dict()
-                        data_dict["train"] = pu.filter_by_dict(
-                            data,
-                            {split_id_dict["attr_key"]: split_id_dict["train"]},
-                            self.nproc,
-                        )
-                        data_dict["test"] = pu.filter_by_dict(
-                            data,
-                            {split_id_dict["attr_key"]: split_id_dict["eval"]},
-                            self.nproc,
-                        )
-                    elif attr_to_split is not None:
-                        data_dict, balance_df = cu.balance_attr_splits(
-                            data,
-                            attr_to_split,
-                            attr_to_balance,
-                            self.eval_size,
-                            max_trials,
-                            pval_threshold,
-                            self.cell_state_dict["state_key"],
-                            self.nproc,
-                        )
-
-                        balance_df.to_csv(
-                            f"{output_dir}/{output_prefix}_train_valid_balance_df.csv"
-                        )
-                    else:
-                        data_dict = data.train_test_split(
-                            test_size=self.eval_size,
-                            stratify_by_column=self.stratify_splits_col,
-                            seed=42,
-                        )
-                    train_data = data_dict["train"]
-                    eval_data = data_dict["test"]
-                else:
-                    # 5-fold cross-validate
-                    num_cells = len(data)
-                    fifth_cells = int(np.floor(num_cells * 0.2))
-                    num_eval = min((self.eval_size * num_cells), fifth_cells)
-                    start = i * fifth_cells
-                    end = start + num_eval
-                    eval_indices = [j for j in range(start, end)]
-                    train_indices = [
-                        j for j in range(num_cells) if j not in eval_indices
-                    ]
-                    eval_data = data.select(eval_indices)
-                    train_data = data.select(train_indices)
-                if n_hyperopt_trials == 0:
-                    trainer = self.train_classifier(
-                        model_directory,
-                        num_classes,
-                        train_data,
-                        eval_data,
-                        ksplit_output_dir,
-                        predict_trainer,
-                    )
-                else:
-                    trainer = self.hyperopt_classifier(
-                        model_directory,
-                        num_classes,
-                        train_data,
-                        eval_data,
-                        ksplit_output_dir,
-                        n_trials=n_hyperopt_trials,
-                    )
-                    if iteration_num == self.num_crossval_splits:
-                        return
-                    else:
-                        iteration_num = iteration_num + 1
-                        continue
-
-                result = self.evaluate_model(
-                    trainer.model,
-                    num_classes,
-                    id_class_dict,
-                    eval_data,
-                    predict_eval,
-                    ksplit_output_dir,
-                    output_prefix,
-                )
-                results += [result]
-                all_conf_mat = all_conf_mat + result["conf_mat"]
-                iteration_num = iteration_num + 1
-
-        elif self.classifier == "gene":
-            # set up (cross-)validation splits
-            targets = pu.flatten_list(self.gene_class_dict.values())
-            labels = pu.flatten_list(
-                [
-                    [class_id_dict[label]] * len(targets)
-                    for label, targets in self.gene_class_dict.items()
-                ]
-            )
-            assert len(targets) == len(labels)
-            n_splits = int(1 / (1 - self.train_size))
-            skf = cu.StratifiedKFold3(n_splits=n_splits, random_state=0, shuffle=True)
-            # (Cross-)validate
-            test_ratio = self.oos_test_size / (self.eval_size + self.oos_test_size)
-            for train_index, eval_index, test_index in tqdm(
-                skf.split(targets, labels, test_ratio)
-            ):
-                print(
-                    f"****** Validation split: {iteration_num}/{self.num_crossval_splits} ******\n"
-                )
-                ksplit_output_dir = os.path.join(output_dir, f"ksplit{iteration_num}")
-                # filter data for examples containing classes for this split
-                # subsample to max_ncells and relabel data in column "labels"
-                train_data, eval_data = cu.prep_gene_classifier_train_eval_split(
-                    data,
-                    targets,
-                    labels,
-                    train_index,
-                    eval_index,
-                    self.max_ncells,
-                    iteration_num,
-                    self.nproc,
-                    gene_balance,
-                )
-
-                if save_gene_split_datasets is True:
-                    for split_name in ["train", "valid"]:
-                        labeled_dataset_output_path = (
-                            Path(output_dir)
-                            / f"{output_prefix}_{split_name}_gene_labeled_ksplit{iteration_num}"
-                        ).with_suffix(".dataset")
-                        if split_name == "train":
-                            train_data.save_to_disk(str(labeled_dataset_output_path))
-                        elif split_name == "valid":
-                            eval_data.save_to_disk(str(labeled_dataset_output_path))
-
-                if self.oos_test_size > 0:
-                    test_data = cu.prep_gene_classifier_split(
-                        data,
-                        targets,
-                        labels,
-                        test_index,
-                        "test",
-                        self.max_ncells,
-                        iteration_num,
-                        self.nproc,
-                    )
-                    if save_gene_split_datasets is True:
-                        test_labeled_dataset_output_path = (
-                            Path(output_dir)
-                            / f"{output_prefix}_test_gene_labeled_ksplit{iteration_num}"
-                        ).with_suffix(".dataset")
-                        test_data.save_to_disk(str(test_labeled_dataset_output_path))
-                if debug_gene_split_datasets is True:
-                    logger.error(
-                        "Exiting after saving gene split datasets given debug_gene_split_datasets = True."
-                    )
-                    raise
-                if n_hyperopt_trials == 0:
-                    trainer = self.train_classifier(
-                        model_directory,
-                        num_classes,
-                        train_data,
-                        eval_data,
-                        ksplit_output_dir,
-                        predict_trainer,
-                    )
-                    result = self.evaluate_model(
-                        trainer.model,
-                        num_classes,
-                        id_class_dict,
-                        eval_data,
-                        predict_eval,
-                        ksplit_output_dir,
-                        output_prefix,
-                    )
-                else:
-                    trainer = self.hyperopt_classifier(
-                        model_directory,
-                        num_classes,
-                        train_data,
-                        eval_data,
-                        ksplit_output_dir,
-                        n_trials=n_hyperopt_trials,
-                    )
-
-                    model = cu.load_best_model(
-                        ksplit_output_dir, self.model_type, num_classes
-                    )
-
-                    if self.oos_test_size > 0:
-                        result = self.evaluate_model(
-                            model,
-                            num_classes,
-                            id_class_dict,
-                            test_data,
-                            predict_eval,
-                            ksplit_output_dir,
-                            output_prefix,
-                        )
-                    else:
-                        if iteration_num == self.num_crossval_splits:
-                            return
-                        else:
-                            iteration_num = iteration_num + 1
-                            continue
-                results += [result]
-                all_conf_mat = all_conf_mat + result["conf_mat"]
-                # break after 1 or 5 splits, each with train/eval proportions dictated by eval_size
-                if iteration_num == self.num_crossval_splits:
-                    break
-                iteration_num = iteration_num + 1
-
-        all_conf_mat_df = pd.DataFrame(
-            all_conf_mat, columns=id_class_dict.values(), index=id_class_dict.values()
-        )
-        all_metrics = {
-            "conf_matrix": all_conf_mat_df,
-            "macro_f1": [result["macro_f1"] for result in results],
-            "acc": [result["acc"] for result in results],
-        }
-        all_roc_metrics = None  # roc metrics not reported for multiclass
-        if num_classes == 2:
-            mean_fpr = np.linspace(0, 1, 100)
-            all_tpr = [result["roc_metrics"]["interp_tpr"] for result in results]
-            all_roc_auc = [result["roc_metrics"]["auc"] for result in results]
-            all_tpr_wt = [result["roc_metrics"]["tpr_wt"] for result in results]
-            mean_tpr, roc_auc, roc_auc_sd = eu.get_cross_valid_roc_metrics(
-                all_tpr, all_roc_auc, all_tpr_wt
-            )
-            all_roc_metrics = {
-                "mean_tpr": mean_tpr,
-                "mean_fpr": mean_fpr,
-                "all_roc_auc": all_roc_auc,
-                "roc_auc": roc_auc,
-                "roc_auc_sd": roc_auc_sd,
-            }
-        all_metrics["all_roc_metrics"] = all_roc_metrics
-        if save_eval_output is True:
-            eval_metrics_output_path = (
-                Path(output_dir) / f"{output_prefix}_eval_metrics_dict"
-            ).with_suffix(".pkl")
-            with open(eval_metrics_output_path, "wb") as f:
-                pickle.dump(all_metrics, f)
-
-        return all_metrics
-
-    def hyperopt_classifier(
-        self,
-        model_directory,
-        num_classes,
-        train_data,
-        eval_data,
-        output_directory,
-        n_trials=100,
-    ):
-        """
-        Fine-tune model for cell state or gene classification.
-
-        **Parameters**
-
-        model_directory : Path
-            | Path to directory containing model
-        num_classes : int
-            | Number of classes for classifier
-        train_data : Dataset
-            | Loaded training .dataset input
-            | For cell classifier, labels in column "label".
-            | For gene classifier, labels in column "labels".
-        eval_data : None, Dataset
-            | (Optional) Loaded evaluation .dataset input
-            | For cell classifier, labels in column "label".
-            | For gene classifier, labels in column "labels".
-        output_directory : Path
-            | Path to directory where fine-tuned model will be saved
-        n_trials : int
-            | Number of trials to run for hyperparameter optimization
-        """
-
-        # initiate runtime environment for raytune
-        import ray
-        from ray import tune
-        from ray.tune.search.hyperopt import HyperOptSearch
-
-        ray.shutdown()  # engage new ray session
-        ray.init()
-
-        ##### Validate and prepare data #####
-        train_data, eval_data = cu.validate_and_clean_cols(
-            train_data, eval_data, self.classifier
-        )
-
-        if (self.no_eval is True) and (eval_data is not None):
-            logger.warning(
-                "no_eval set to True; hyperparameter optimization requires eval, proceeding with eval"
-            )
-
-        # ensure not overwriting previously saved model
-        saved_model_test = os.path.join(output_directory, "pytorch_model.bin")
-        if os.path.isfile(saved_model_test) is True:
-            logger.error("Model already saved to this designated output directory.")
-            raise
-        # make output directory
-        subprocess.call(f"mkdir {output_directory}", shell=True)
-
-        ##### Load model and training args #####
-        model = pu.load_model(
-            self.model_type,
-            num_classes,
-            model_directory,
-            "train",
-            quantize=self.quantize,
-        )
-        def_training_args, def_freeze_layers = cu.get_default_train_args(
-            model, self.classifier, train_data, output_directory
-        )
-        del model
-
-        if self.training_args is not None:
-            def_training_args.update(self.training_args)
-        logging_steps = round(
-            len(train_data) / def_training_args["per_device_train_batch_size"] / 10
-        )
-        def_training_args["logging_steps"] = logging_steps
-        def_training_args["output_dir"] = output_directory
-        if eval_data is None:
-            def_training_args["evaluation_strategy"] = "no"
-            def_training_args["load_best_model_at_end"] = False
-        def_training_args.update(
-            {"save_strategy": "epoch", "save_total_limit": 1}
-        )  # only save last model for each run
-        training_args_init = TrainingArguments(**def_training_args)
-
-        ##### Fine-tune the model #####
-        # define the data collator
-        if self.classifier == "cell":
-            data_collator = DataCollatorForCellClassification(
-                token_dictionary=self.gene_token_dict
-            )
-        elif self.classifier == "gene":
-            data_collator = DataCollatorForGeneClassification(
-                token_dictionary=self.gene_token_dict
-            )
-
-        # define function to initiate model
-        def model_init():
-            model = pu.load_model(
-                self.model_type,
-                num_classes,
-                model_directory,
-                "train",
-                quantize=self.quantize,
-            )
-
-            if self.freeze_layers is not None:
-                def_freeze_layers = self.freeze_layers
-
-            if def_freeze_layers > 0:
-                modules_to_freeze = model.bert.encoder.layer[:def_freeze_layers]
-                for module in modules_to_freeze:
-                    for param in module.parameters():
-                        param.requires_grad = False
-
-            if self.quantize is False:
-                model = model.to("cuda:0")
-            return model
-
-        # create the trainer
-        trainer = Trainer(
-            model_init=model_init,
-            args=training_args_init,
-            data_collator=data_collator,
-            train_dataset=train_data,
-            eval_dataset=eval_data,
-            compute_metrics=cu.compute_metrics,
-        )
-
-        # specify raytune hyperparameter search space
-        if self.ray_config is None:
-            logger.warning(
-                "No ray_config provided. Proceeding with default, but ranges may need adjustment depending on model."
-            )
-            def_ray_config = {
-                "num_train_epochs": tune.choice([1]),
-                "learning_rate": tune.loguniform(1e-6, 1e-3),
-                "weight_decay": tune.uniform(0.0, 0.3),
-                "lr_scheduler_type": tune.choice(["linear", "cosine", "polynomial"]),
-                "warmup_steps": tune.uniform(100, 2000),
-                "seed": tune.uniform(0, 100),
-                "per_device_train_batch_size": tune.choice(
-                    [def_training_args["per_device_train_batch_size"]]
-                ),
-            }
-
-        hyperopt_search = HyperOptSearch(metric="eval_macro_f1", mode="max")
-
-        # optimize hyperparameters
-        trainer.hyperparameter_search(
-            direction="maximize",
-            backend="ray",
-            resources_per_trial={"cpu": int(self.nproc / self.ngpu), "gpu": 1},
-            hp_space=lambda _: def_ray_config
-            if self.ray_config is None
-            else self.ray_config,
-            search_alg=hyperopt_search,
-            n_trials=n_trials,  # number of trials
-            progress_reporter=tune.CLIReporter(
-                max_report_frequency=600,
-                sort_by_metric=True,
-                max_progress_rows=n_trials,
-                mode="max",
-                metric="eval_macro_f1",
-                metric_columns=["loss", "eval_loss", "eval_accuracy", "eval_macro_f1"],
-            ),
-            storage_path=output_directory,
-        )
-
-        return trainer
-
-    def train_classifier(
-        self,
-        model_directory,
-        num_classes,
-        train_data,
-        eval_data,
-        output_directory,
-        predict=False,
-    ):
-        """
-        Fine-tune model for cell state or gene classification.
-
-        **Parameters**
-
-        model_directory : Path
-            | Path to directory containing model
-        num_classes : int
-            | Number of classes for classifier
-        train_data : Dataset
-            | Loaded training .dataset input
-            | For cell classifier, labels in column "label".
-            | For gene classifier, labels in column "labels".
-        eval_data : None, Dataset
-            | (Optional) Loaded evaluation .dataset input
-            | For cell classifier, labels in column "label".
-            | For gene classifier, labels in column "labels".
-        output_directory : Path
-            | Path to directory where fine-tuned model will be saved
-        predict : bool
-            | Whether or not to save eval predictions from trainer
-        """
-
-        ##### Validate and prepare data #####
-        train_data, eval_data = cu.validate_and_clean_cols(
-            train_data, eval_data, self.classifier
-        )
-
-        if (self.no_eval is True) and (eval_data is not None):
-            logger.warning(
-                "no_eval set to True; model will be trained without evaluation."
-            )
-            eval_data = None
-
-        if (self.classifier == "gene") and (predict is True):
-            logger.warning(
-                "Predictions during training not currently available for gene classifiers; setting predict to False."
-            )
-            predict = False
-
-        # ensure not overwriting previously saved model
-        saved_model_test = os.path.join(output_directory, "pytorch_model.bin")
-        if os.path.isfile(saved_model_test) is True:
-            logger.error("Model already saved to this designated output directory.")
-            raise
-        # make output directory
-        subprocess.call(f"mkdir {output_directory}", shell=True)
-
-        ##### Load model and training args #####
-        model = pu.load_model(
-            self.model_type,
-            num_classes,
-            model_directory,
-            "train",
-            quantize=self.quantize,
-        )
-
-        def_training_args, def_freeze_layers = cu.get_default_train_args(
-            model, self.classifier, train_data, output_directory
-        )
-
-        if self.training_args is not None:
-            def_training_args.update(self.training_args)
-        logging_steps = round(
-            len(train_data) / def_training_args["per_device_train_batch_size"] / 10
-        )
-        def_training_args["logging_steps"] = logging_steps
-        def_training_args["output_dir"] = output_directory
-        if eval_data is None:
-            def_training_args["evaluation_strategy"] = "no"
-            def_training_args["load_best_model_at_end"] = False
-        training_args_init = TrainingArguments(**def_training_args)
-
-        if self.freeze_layers is not None:
-            def_freeze_layers = self.freeze_layers
-
-        if def_freeze_layers > 0:
-            modules_to_freeze = model.bert.encoder.layer[:def_freeze_layers]
-            for module in modules_to_freeze:
-                for param in module.parameters():
-                    param.requires_grad = False
-
-        ##### Fine-tune the model #####
-        # define the data collator
-        if self.classifier == "cell":
-            data_collator = DataCollatorForCellClassification(
-                token_dictionary=self.gene_token_dict
-            )
-        elif self.classifier == "gene":
-            data_collator = DataCollatorForGeneClassification(
-                token_dictionary=self.gene_token_dict
-            )
-
-        # create the trainer
-        trainer = Trainer(
-            model=model,
-            args=training_args_init,
-            data_collator=data_collator,
-            train_dataset=train_data,
-            eval_dataset=eval_data,
-            compute_metrics=cu.compute_metrics,
-        )
-
-        # train the classifier
-        trainer.train()
-        trainer.save_model(output_directory)
-        if predict is True:
-            # make eval predictions and save predictions and metrics
-            predictions = trainer.predict(eval_data)
-            prediction_output_path = f"{output_directory}/predictions.pkl"
-            with open(prediction_output_path, "wb") as f:
-                pickle.dump(predictions, f)
-            trainer.save_metrics("eval", predictions.metrics)
-        return trainer
-
-    def evaluate_model(
-        self,
-        model,
-        num_classes,
-        id_class_dict,
-        eval_data,
-        predict=False,
-        output_directory=None,
-        output_prefix=None,
-    ):
-        """
-        Evaluate the fine-tuned model.
-
-        **Parameters**
-
-        model : nn.Module
-            | Loaded fine-tuned model (e.g. trainer.model)
-        num_classes : int
-            | Number of classes for classifier
-        id_class_dict : dict
-            | Loaded _id_class_dict.pkl previously prepared by Classifier.prepare_data
-            | (dictionary of format: numerical IDs: class_labels)
-        eval_data : Dataset
-            | Loaded evaluation .dataset input
-        predict : bool
-            | Whether or not to save eval predictions
-        output_directory : Path
-            | Path to directory where eval data will be saved
-        output_prefix : str
-            | Prefix for output files
-        """
-
-        ##### Evaluate the model #####
-        labels = id_class_dict.keys()
-        y_pred, y_true, logits_list = eu.classifier_predict(
-            model, self.classifier, eval_data, self.forward_batch_size
-        )
-        conf_mat, macro_f1, acc, roc_metrics = eu.get_metrics(
-            y_pred, y_true, logits_list, num_classes, labels
-        )
-        if predict is True:
-            pred_dict = {
-                "pred_ids": y_pred,
-                "label_ids": y_true,
-                "predictions": logits_list,
-            }
-            pred_dict_output_path = (
-                Path(output_directory) / f"{output_prefix}_pred_dict"
-            ).with_suffix(".pkl")
-            with open(pred_dict_output_path, "wb") as f:
-                pickle.dump(pred_dict, f)
-        return {
-            "conf_mat": conf_mat,
-            "macro_f1": macro_f1,
-            "acc": acc,
-            "roc_metrics": roc_metrics,
-        }
-
-    def evaluate_saved_model(
-        self,
-        model_directory,
-        id_class_dict_file,
-        test_data_file,
-        output_directory,
-        output_prefix,
-        predict=True,
-    ):
-        """
-        Evaluate the fine-tuned model.
-
-        **Parameters**
-
-        model_directory : Path
-            | Path to directory containing model
-        id_class_dict_file : Path
-            | Path to _id_class_dict.pkl previously prepared by Classifier.prepare_data
-            | (dictionary of format: numerical IDs: class_labels)
-        test_data_file : Path
-            | Path to directory containing test .dataset
-        output_directory : Path
-            | Path to directory where eval data will be saved
-        output_prefix : str
-            | Prefix for output files
-        predict : bool
-            | Whether or not to save eval predictions
-        """
-
-        # load numerical id to class dictionary (id:class)
-        with open(id_class_dict_file, "rb") as f:
-            id_class_dict = pickle.load(f)
-
-        # get number of classes for classifier
-        num_classes = cu.get_num_classes(id_class_dict)
-
-        # load previously filtered and prepared data
-        test_data = pu.load_and_filter(None, self.nproc, test_data_file)
-
-        # load previously fine-tuned model
-        model = pu.load_model(
-            self.model_type,
-            num_classes,
-            model_directory,
-            "eval",
-            quantize=self.quantize,
-        )
-
-        # evaluate the model
-        result = self.evaluate_model(
-            model,
-            num_classes,
-            id_class_dict,
-            test_data,
-            predict=predict,
-            output_directory=output_directory,
-            output_prefix=output_prefix,
-        )
-
-        all_conf_mat_df = pd.DataFrame(
-            result["conf_mat"],
-            columns=id_class_dict.values(),
-            index=id_class_dict.values(),
-        )
-        all_metrics = {
-            "conf_matrix": all_conf_mat_df,
-            "macro_f1": result["macro_f1"],
-            "acc": result["acc"],
-        }
-        all_roc_metrics = None  # roc metrics not reported for multiclass
-
-        if num_classes == 2:
-            mean_fpr = np.linspace(0, 1, 100)
-            mean_tpr = result["roc_metrics"]["interp_tpr"]
-            all_roc_auc = result["roc_metrics"]["auc"]
-            all_roc_metrics = {
-                "mean_tpr": mean_tpr,
-                "mean_fpr": mean_fpr,
-                "all_roc_auc": all_roc_auc,
-            }
-        all_metrics["all_roc_metrics"] = all_roc_metrics
-        test_metrics_output_path = (
-            Path(output_directory) / f"{output_prefix}_test_metrics_dict"
-        ).with_suffix(".pkl")
-        with open(test_metrics_output_path, "wb") as f:
-            pickle.dump(all_metrics, f)
-
-        return all_metrics
-
-    def plot_conf_mat(
-        self,
-        conf_mat_dict,
-        output_directory,
-        output_prefix,
-        custom_class_order=None,
-    ):
-        """
-        Plot confusion matrix results of evaluating the fine-tuned model.
-
-        **Parameters**
-
-        conf_mat_dict : dict
-            | Dictionary of model_name : confusion_matrix_DataFrame
-            | (all_metrics["conf_matrix"] from self.validate)
-        output_directory : Path
-            | Path to directory where plots will be saved
-        output_prefix : str
-            | Prefix for output file
-        custom_class_order : None, list
-            | List of classes in custom order for plots.
-            | Same order will be used for all models.
-        """
-
-        for model_name in conf_mat_dict.keys():
-            eu.plot_confusion_matrix(
-                conf_mat_dict[model_name],
-                model_name,
-                output_directory,
-                output_prefix,
-                custom_class_order,
-            )
-
-    def plot_roc(
-        self,
-        roc_metric_dict,
-        model_style_dict,
-        title,
-        output_directory,
-        output_prefix,
-    ):
-        """
-        Plot ROC curve results of evaluating the fine-tuned model.
-
-        **Parameters**
-
-        roc_metric_dict : dict
-            | Dictionary of model_name : roc_metrics
-            | (all_metrics["all_roc_metrics"] from self.validate)
-        model_style_dict : dict[dict]
-            | Dictionary of model_name : dictionary of style_attribute : style
-            | where style includes color and linestyle
-            | e.g. {'Model_A': {'color': 'black', 'linestyle': '-'}, 'Model_B': ...}
-        title : str
-            | Title of plot (e.g. 'Dosage-sensitive vs -insensitive factors')
-        output_directory : Path
-            | Path to directory where plots will be saved
-        output_prefix : str
-            | Prefix for output file
-        """
-
-        eu.plot_ROC(
-            roc_metric_dict, model_style_dict, title, output_directory, output_prefix
-        )
-
-    def plot_predictions(
-        self,
-        predictions_file,
-        id_class_dict_file,
-        title,
-        output_directory,
-        output_prefix,
-        custom_class_order=None,
-        kwargs_dict=None,
-    ):
-        """
-        Plot prediction results of evaluating the fine-tuned model.
-
-        **Parameters**
-
-        predictions_file : path
-            | Path of model predictions output to plot
-            | (saved output from self.validate if predict_eval=True)
-            | (or saved output from self.evaluate_saved_model)
-        id_class_dict_file : Path
-            | Path to _id_class_dict.pkl previously prepared by Classifier.prepare_data
-            | (dictionary of format: numerical IDs: class_labels)
-        title : str
-            | Title for legend containing class labels.
-        output_directory : Path
-            | Path to directory where plots will be saved
-        output_prefix : str
-            | Prefix for output file
-        custom_class_order : None, list
-            | List of classes in custom order for plots.
-            | Same order will be used for all models.
-        kwargs_dict : None, dict
-            | Dictionary of kwargs to pass to plotting function.
-        """
-        # load predictions
-        with open(predictions_file, "rb") as f:
-            predictions = pickle.load(f)
-
-        # load numerical id to class dictionary (id:class)
-        with open(id_class_dict_file, "rb") as f:
-            id_class_dict = pickle.load(f)
-
-        if isinstance(predictions, dict):
-            if all(
-                [
-                    key in predictions.keys()
-                    for key in ["pred_ids", "label_ids", "predictions"]
-                ]
-            ):
-                # format is output from self.evaluate_saved_model
-                predictions_logits = np.array(predictions["predictions"])
-                true_ids = predictions["label_ids"]
-        else:
-            # format is output from self.validate if predict_eval=True
-            predictions_logits = predictions.predictions
-            true_ids = predictions.label_ids
-
-        num_classes = len(id_class_dict.keys())
-        num_predict_classes = predictions_logits.shape[1]
-        assert num_classes == num_predict_classes
-        classes = id_class_dict.values()
-        true_labels = [id_class_dict[idx] for idx in true_ids]
-        predictions_df = pd.DataFrame(predictions_logits, columns=classes)
-        if custom_class_order is not None:
-            predictions_df = predictions_df.reindex(columns=custom_class_order)
-        predictions_df["true"] = true_labels
-        custom_dict = dict(zip(classes, [i for i in range(len(classes))]))
-        if custom_class_order is not None:
-            custom_dict = dict(
-                zip(custom_class_order, [i for i in range(len(custom_class_order))])
-            )
-        predictions_df = predictions_df.sort_values(
-            by=["true"], key=lambda x: x.map(custom_dict)
-        )
-
-        eu.plot_predictions(
-            predictions_df, title, output_directory, output_prefix, kwargs_dict
-        )

geneformer/classifier_utils.py

@@ -1,648 +0,0 @@
-import json
-import logging
-import os
-import random
-from collections import Counter, defaultdict
-
-import numpy as np
-import pandas as pd
-from scipy.stats import chisquare, ranksums
-from sklearn.metrics import accuracy_score, f1_score
-from sklearn.model_selection import StratifiedKFold, train_test_split
-
-from . import perturber_utils as pu
-
-logger = logging.getLogger(__name__)
-
-
-def downsample_and_shuffle(data, max_ncells, max_ncells_per_class, cell_state_dict):
-    data = data.shuffle(seed=42)
-    num_cells = len(data)
-    # if max number of cells is defined, then subsample to this max number
-    if max_ncells is not None:
-        if num_cells > max_ncells:
-            data = data.select([i for i in range(max_ncells)])
-    if max_ncells_per_class is not None:
-        class_labels = data[cell_state_dict["state_key"]]
-        random.seed(42)
-        subsample_indices = subsample_by_class(class_labels, max_ncells_per_class)
-        data = data.select(subsample_indices)
-    return data
-
-
-# subsample labels to maximum number N per class and return indices
-def subsample_by_class(labels, N):
-    label_indices = defaultdict(list)
-    # Gather indices for each label
-    for idx, label in enumerate(labels):
-        label_indices[label].append(idx)
-    selected_indices = []
-    # Select up to N indices for each label
-    for label, indices in label_indices.items():
-        if len(indices) > N:
-            selected_indices.extend(random.sample(indices, N))
-        else:
-            selected_indices.extend(indices)
-    return selected_indices
-
-
-def rename_cols(data, state_key):
-    data = data.rename_column(state_key, "label")
-    return data
-
-
-def validate_and_clean_cols(train_data, eval_data, classifier):
-    # validate that data has expected label column and remove others
-    if classifier == "cell":
-        label_col = "label"
-    elif classifier == "gene":
-        label_col = "labels"
-
-    cols_to_keep = [label_col] + ["input_ids", "length"]
-    if label_col not in train_data.column_names:
-        logger.error(f"train_data must contain column {label_col} with class labels.")
-        raise
-    else:
-        train_data = remove_cols(train_data, cols_to_keep)
-
-    if eval_data is not None:
-        if label_col not in eval_data.column_names:
-            logger.error(
-                f"eval_data must contain column {label_col} with class labels."
-            )
-            raise
-        else:
-            eval_data = remove_cols(eval_data, cols_to_keep)
-    return train_data, eval_data
-
-
-def remove_cols(data, cols_to_keep):
-    other_cols = list(data.features.keys())
-    other_cols = [ele for ele in other_cols if ele not in cols_to_keep]
-    data = data.remove_columns(other_cols)
-    return data
-
-
-def remove_rare(data, rare_threshold, label, nproc):
-    if rare_threshold > 0:
-        total_cells = len(data)
-        label_counter = Counter(data[label])
-        nonrare_label_dict = {
-            label: [k for k, v in label_counter if (v / total_cells) > rare_threshold]
-        }
-        data = pu.filter_by_dict(data, nonrare_label_dict, nproc)
-    return data
-
-
-def label_classes(classifier, data, gene_class_dict, nproc):
-    if classifier == "cell":
-        label_set = set(data["label"])
-    elif classifier == "gene":
-        # remove cells without any of the target genes
-        def if_contains_label(example):
-            a = pu.flatten_list(gene_class_dict.values())
-            b = example["input_ids"]
-            return not set(a).isdisjoint(b)
-
-        data = data.filter(if_contains_label, num_proc=nproc)
-        label_set = gene_class_dict.keys()
-
-        if len(data) == 0:
-            logger.error(
-                "No cells remain after filtering for target genes. Check target gene list."
-            )
-            raise
-
-    class_id_dict = dict(zip(label_set, [i for i in range(len(label_set))]))
-    id_class_dict = {v: k for k, v in class_id_dict.items()}
-
-    def classes_to_ids(example):
-        if classifier == "cell":
-            example["label"] = class_id_dict[example["label"]]
-        elif classifier == "gene":
-            example["labels"] = label_gene_classes(
-                example, class_id_dict, gene_class_dict
-            )
-        return example
-
-    data = data.map(classes_to_ids, num_proc=nproc)
-    return data, id_class_dict
-
-
-def label_gene_classes(example, class_id_dict, gene_class_dict):
-    return [
-        class_id_dict.get(gene_class_dict.get(token_id, -100), -100)
-        for token_id in example["input_ids"]
-    ]
-
-
-def prep_gene_classifier_train_eval_split(
-    data,
-    targets,
-    labels,
-    train_index,
-    eval_index,
-    max_ncells,
-    iteration_num,
-    num_proc,
-    balance=False,
-):
-    # generate cross-validation splits
-    train_data = prep_gene_classifier_split(
-        data,
-        targets,
-        labels,
-        train_index,
-        "train",
-        max_ncells,
-        iteration_num,
-        num_proc,
-        balance,
-    )
-    eval_data = prep_gene_classifier_split(
-        data,
-        targets,
-        labels,
-        eval_index,
-        "eval",
-        max_ncells,
-        iteration_num,
-        num_proc,
-        balance,
-    )
-    return train_data, eval_data
-
-
-def prep_gene_classifier_split(
-    data,
-    targets,
-    labels,
-    index,
-    subset_name,
-    max_ncells,
-    iteration_num,
-    num_proc,
-    balance=False,
-):
-    # generate cross-validation splits
-    targets = np.array(targets)
-    labels = np.array(labels)
-    targets_subset = targets[index]
-    labels_subset = labels[index]
-    label_dict_subset = dict(zip(targets_subset, labels_subset))
-
-    # function to filter by whether contains train or eval labels
-    def if_contains_subset_label(example):
-        a = targets_subset
-        b = example["input_ids"]
-        return not set(a).isdisjoint(b)
-
-    # filter dataset for examples containing classes for this split
-    logger.info(f"Filtering data for {subset_name} genes in split {iteration_num}")
-    subset_data = data.filter(if_contains_subset_label, num_proc=num_proc)
-    logger.info(
-        f"Filtered {round((1-len(subset_data)/len(data))*100)}%; {len(subset_data)} remain\n"
-    )
-
-    # balance gene subsets if train
-    if (subset_name == "train") and (balance is True):
-        subset_data, label_dict_subset = balance_gene_split(
-            subset_data, label_dict_subset, num_proc
-        )
-
-    # subsample to max_ncells
-    subset_data = downsample_and_shuffle(subset_data, max_ncells, None, None)
-
-    # relabel genes for this split
-    def subset_classes_to_ids(example):
-        example["labels"] = [
-            label_dict_subset.get(token_id, -100) for token_id in example["input_ids"]
-        ]
-        return example
-
-    subset_data = subset_data.map(subset_classes_to_ids, num_proc=num_proc)
-
-    return subset_data
-
-
-def prep_gene_classifier_all_data(
-    data, targets, labels, max_ncells, num_proc, balance=False
-):
-    targets = np.array(targets)
-    labels = np.array(labels)
-    label_dict_train = dict(zip(targets, labels))
-
-    # function to filter by whether contains train labels
-    def if_contains_train_label(example):
-        a = targets
-        b = example["input_ids"]
-        return not set(a).isdisjoint(b)
-
-    # filter dataset for examples containing classes for this split
-    logger.info("Filtering training data for genes to classify.")
-    train_data = data.filter(if_contains_train_label, num_proc=num_proc)
-    logger.info(
-        f"Filtered {round((1-len(train_data)/len(data))*100)}%; {len(train_data)} remain\n"
-    )
-
-    if balance is True:
-        train_data, label_dict_train = balance_gene_split(
-            train_data, label_dict_train, num_proc
-        )
-
-    # subsample to max_ncells
-    train_data = downsample_and_shuffle(train_data, max_ncells, None, None)
-
-    # relabel genes for this split
-    def train_classes_to_ids(example):
-        example["labels"] = [
-            label_dict_train.get(token_id, -100) for token_id in example["input_ids"]
-        ]
-        return example
-
-    train_data = train_data.map(train_classes_to_ids, num_proc=num_proc)
-
-    return train_data
-
-
-def balance_gene_split(subset_data, label_dict_subset, num_proc):
-    # count occurrence of genes in each label category
-    label0_counts, label1_counts = count_genes_for_balancing(
-        subset_data, label_dict_subset, num_proc
-    )
-    label_ratio_0to1 = label0_counts / label1_counts
-
-    if 8 / 10 <= label_ratio_0to1 <= 10 / 8:
-        # gene sets already balanced
-        logger.info(
-            "Gene sets were already balanced within 0.8-1.25 fold and did not require balancing.\n"
-        )
-        return subset_data, label_dict_subset
-    else:
-        label_ratio_0to1_orig = label_ratio_0to1 + 0
-        label_dict_subset_orig = label_dict_subset.copy()
-        # balance gene sets
-        max_ntrials = 25
-        boost = 1
-        if label_ratio_0to1 > 10 / 8:
-            # downsample label 0
-            for i in range(max_ntrials):
-                label0 = 0
-                label0_genes = [k for k, v in label_dict_subset.items() if v == label0]
-                label0_ngenes = len(label0_genes)
-                label0_nremove = max(
-                    1,
-                    int(
-                        np.floor(
-                            label0_ngenes - label0_ngenes / (label_ratio_0to1 * boost)
-                        )
-                    ),
-                )
-                random.seed(i)
-                label0_remove_genes = random.sample(label0_genes, label0_nremove)
-                label_dict_subset_new = {
-                    k: v
-                    for k, v in label_dict_subset.items()
-                    if k not in label0_remove_genes
-                }
-                label0_counts, label1_counts = count_genes_for_balancing(
-                    subset_data, label_dict_subset_new, num_proc
-                )
-                label_ratio_0to1 = label0_counts / label1_counts
-                if 8 / 10 <= label_ratio_0to1 <= 10 / 8:
-                    # if gene sets now balanced, return new filtered data and new label_dict_subset
-                    return filter_data_balanced_genes(
-                        subset_data, label_dict_subset_new, num_proc
-                    )
-                elif label_ratio_0to1 > 10 / 8:
-                    boost = boost * 1.1
-                elif label_ratio_0to1 < 8 / 10:
-                    boost = boost * 0.9
-        else:
-            # downsample label 1
-            for i in range(max_ntrials):
-                label1 = 1
-                label1_genes = [k for k, v in label_dict_subset.items() if v == label1]
-                label1_ngenes = len(label1_genes)
-                label1_nremove = max(
-                    1,
-                    int(
-                        np.floor(
-                            label1_ngenes
-                            - label1_ngenes / ((1 / label_ratio_0to1) * boost)
-                        )
-                    ),
-                )
-                random.seed(i)
-                label1_remove_genes = random.sample(label1_genes, label1_nremove)
-                label_dict_subset_new = {
-                    k: v
-                    for k, v in label_dict_subset.items()
-                    if k not in label1_remove_genes
-                }
-                label0_counts, label1_counts = count_genes_for_balancing(
-                    subset_data, label_dict_subset_new, num_proc
-                )
-                label_ratio_0to1 = label0_counts / label1_counts
-                if 8 / 10 <= label_ratio_0to1 <= 10 / 8:
-                    # if gene sets now balanced, return new filtered data and new label_dict_subset
-                    return filter_data_balanced_genes(
-                        subset_data, label_dict_subset_new, num_proc
-                    )
-                elif label_ratio_0to1 < 8 / 10:
-                    boost = boost * 1.1
-                elif label_ratio_0to1 > 10 / 8:
-                    boost = boost * 0.9
-
-        assert i + 1 == max_ntrials
-        if (label_ratio_0to1 <= label_ratio_0to1_orig < 8 / 10) or (
-            10 / 8 > label_ratio_0to1_orig >= label_ratio_0to1
-        ):
-            label_ratio_0to1 = label_ratio_0to1_orig
-            label_dict_subset_new = label_dict_subset_orig
-        logger.warning(
-            f"Gene sets were not able to be balanced within 0.8-1.25 fold after {max_ntrials} trials. Imbalance level: {label_ratio_0to1}\n"
-        )
-        return filter_data_balanced_genes(subset_data, label_dict_subset_new, num_proc)
-
-
-def count_genes_for_balancing(subset_data, label_dict_subset, num_proc):
-    def count_targets(example):
-        labels = [
-            label_dict_subset.get(token_id, -100) for token_id in example["input_ids"]
-        ]
-        counter_labels = Counter(labels)
-        # get count of labels 0 or 1, or if absent, return 0
-        example["labels_counts"] = [counter_labels.get(0, 0), counter_labels.get(1, 0)]
-        return example
-
-    subset_data = subset_data.map(count_targets, num_proc=num_proc)
-
-    label0_counts = sum([counts[0] for counts in subset_data["labels_counts"]])
-    label1_counts = sum([counts[1] for counts in subset_data["labels_counts"]])
-
-    subset_data = subset_data.remove_columns("labels_counts")
-
-    return label0_counts, label1_counts
-
-
-def filter_data_balanced_genes(subset_data, label_dict_subset, num_proc):
-    # function to filter by whether contains labels
-    def if_contains_subset_label(example):
-        a = list(label_dict_subset.keys())
-        b = example["input_ids"]
-        return not set(a).isdisjoint(b)
-
-    # filter dataset for examples containing classes for this split
-    logger.info("Filtering data for balanced genes")
-    subset_data_len_orig = len(subset_data)
-    subset_data = subset_data.filter(if_contains_subset_label, num_proc=num_proc)
-    logger.info(
-        f"Filtered {round((1-len(subset_data)/subset_data_len_orig)*100)}%; {len(subset_data)} remain\n"
-    )
-
-    return subset_data, label_dict_subset
-
-
-def balance_attr_splits(
-    data,
-    attr_to_split,
-    attr_to_balance,
-    eval_size,
-    max_trials,
-    pval_threshold,
-    state_key,
-    nproc,
-):
-    metadata_df = pd.DataFrame({"split_attr_ids": data[attr_to_split]})
-    for attr in attr_to_balance:
-        if attr == state_key:
-            metadata_df[attr] = data["label"]
-        else:
-            metadata_df[attr] = data[attr]
-    metadata_df = metadata_df.drop_duplicates()
-
-    split_attr_ids = list(metadata_df["split_attr_ids"])
-    assert len(split_attr_ids) == len(set(split_attr_ids))
-    eval_num = round(len(split_attr_ids) * eval_size)
-    colnames = (
-        ["trial_num", "train_ids", "eval_ids"]
-        + pu.flatten_list(
-            [
-                [
-                    f"{attr}_train_mean_or_counts",
-                    f"{attr}_eval_mean_or_counts",
-                    f"{attr}_pval",
-                ]
-                for attr in attr_to_balance
-            ]
-        )
-        + ["mean_pval"]
-    )
-    balance_df = pd.DataFrame(columns=colnames)
-    data_dict = dict()
-    trial_num = 1
-    for i in range(max_trials):
-        if not all(
-            count > 1 for count in list(Counter(metadata_df[state_key]).values())
-        ):
-            logger.error(
-                f"Cannot balance by {attr_to_split} while retaining at least 1 occurrence of each {state_key} class in both data splits. "
-            )
-            raise
-        eval_base = []
-        for state in set(metadata_df[state_key]):
-            eval_base += list(
-                metadata_df.loc[
-                    metadata_df[state_key][metadata_df[state_key].eq(state)]
-                    .sample(1, random_state=i)
-                    .index
-                ]["split_attr_ids"]
-            )
-        non_eval_base = [idx for idx in split_attr_ids if idx not in eval_base]
-        random.seed(i)
-        eval_ids = random.sample(non_eval_base, eval_num - len(eval_base)) + eval_base
-        train_ids = [idx for idx in split_attr_ids if idx not in eval_ids]
-        df_vals = [trial_num, train_ids, eval_ids]
-        pvals = []
-        for attr in attr_to_balance:
-            train_attr = list(
-                metadata_df[metadata_df["split_attr_ids"].isin(train_ids)][attr]
-            )
-            eval_attr = list(
-                metadata_df[metadata_df["split_attr_ids"].isin(eval_ids)][attr]
-            )
-            if attr == state_key:
-                # ensure IDs are interpreted as categorical
-                train_attr = [str(item) for item in train_attr]
-                eval_attr = [str(item) for item in eval_attr]
-            if all(isinstance(item, (int, float)) for item in train_attr + eval_attr):
-                train_attr_mean = np.nanmean(train_attr)
-                eval_attr_mean = np.nanmean(eval_attr)
-                pval = ranksums(train_attr, eval_attr, nan_policy="omit").pvalue
-                df_vals += [train_attr_mean, eval_attr_mean, pval]
-            elif all(isinstance(item, (str)) for item in train_attr + eval_attr):
-                obs_counts = Counter(train_attr)
-                exp_counts = Counter(eval_attr)
-                all_categ = set(obs_counts.keys()).union(set(exp_counts.keys()))
-                obs = [obs_counts[cat] for cat in all_categ]
-                exp = [
-                    exp_counts[cat] * sum(obs) / sum(exp_counts.values())
-                    for cat in all_categ
-                ]
-                pval = chisquare(f_obs=obs, f_exp=exp).pvalue
-                train_attr_counts = str(obs_counts).strip("Counter(").strip(")")
-                eval_attr_counts = str(exp_counts).strip("Counter(").strip(")")
-                df_vals += [train_attr_counts, eval_attr_counts, pval]
-            else:
-                logger.error(
-                    f"Inconsistent data types in attribute {attr}. "
-                    "Cannot infer if continuous or categorical. "
-                    "Must be all numeric (continuous) or all strings (categorical) to balance."
-                )
-                raise
-            pvals += [pval]
-
-        df_vals += [np.nanmean(pvals)]
-        balance_df_i = pd.DataFrame(df_vals, index=colnames).T
-        balance_df = pd.concat([balance_df, balance_df_i], ignore_index=True)
-        valid_pvals = [
-            pval_i
-            for pval_i in pvals
-            if isinstance(pval_i, (int, float)) and not np.isnan(pval_i)
-        ]
-        if all(i >= pval_threshold for i in valid_pvals):
-            data_dict["train"] = pu.filter_by_dict(
-                data, {attr_to_split: balance_df_i["train_ids"][0]}, nproc
-            )
-            data_dict["test"] = pu.filter_by_dict(
-                data, {attr_to_split: balance_df_i["eval_ids"][0]}, nproc
-            )
-            return data_dict, balance_df
-        trial_num = trial_num + 1
-    balance_max_df = balance_df.iloc[balance_df["mean_pval"].idxmax(), :]
-    data_dict["train"] = pu.filter_by_dict(
-        data, {attr_to_split: balance_df_i["train_ids"][0]}, nproc
-    )
-    data_dict["test"] = pu.filter_by_dict(
-        data, {attr_to_split: balance_df_i["eval_ids"][0]}, nproc
-    )
-    logger.warning(
-        f"No splits found without significant difference in attr_to_balance among {max_trials} trials. "
-        f"Selecting optimal split (trial #{balance_max_df['trial_num']}) from completed trials."
-    )
-    return data_dict, balance_df
-
-
-def get_num_classes(id_class_dict):
-    return len(set(id_class_dict.values()))
-
-
-def compute_metrics(pred):
-    labels = pred.label_ids
-    preds = pred.predictions.argmax(-1)
-
-    # calculate accuracy and macro f1 using sklearn's function
-    if len(labels.shape) == 1:
-        acc = accuracy_score(labels, preds)
-        macro_f1 = f1_score(labels, preds, average="macro")
-    else:
-        flat_labels = labels.flatten().tolist()
-        flat_preds = preds.flatten().tolist()
-        logit_label_paired = [
-            item for item in list(zip(flat_preds, flat_labels)) if item[1] != -100
-        ]
-        y_pred = [item[0] for item in logit_label_paired]
-        y_true = [item[1] for item in logit_label_paired]
-
-        acc = accuracy_score(y_true, y_pred)
-        macro_f1 = f1_score(y_true, y_pred, average="macro")
-
-    return {"accuracy": acc, "macro_f1": macro_f1}
-
-
-def get_default_train_args(model, classifier, data, output_dir):
-    num_layers = pu.quant_layers(model)
-    freeze_layers = 0
-    batch_size = 12
-    if classifier == "cell":
-        epochs = 10
-        evaluation_strategy = "epoch"
-        load_best_model_at_end = True
-    else:
-        epochs = 1
-        evaluation_strategy = "no"
-        load_best_model_at_end = False
-
-    if num_layers == 6:
-        default_training_args = {
-            "learning_rate": 5e-5,
-            "lr_scheduler_type": "linear",
-            "warmup_steps": 500,
-            "per_device_train_batch_size": batch_size,
-            "per_device_eval_batch_size": batch_size,
-        }
-    else:
-        default_training_args = {
-            "per_device_train_batch_size": batch_size,
-            "per_device_eval_batch_size": batch_size,
-        }
-
-    training_args = {
-        "num_train_epochs": epochs,
-        "do_train": True,
-        "do_eval": True,
-        "evaluation_strategy": evaluation_strategy,
-        "logging_steps": np.floor(len(data) / batch_size / 8),  # 8 evals per epoch
-        "save_strategy": "epoch",
-        "group_by_length": False,
-        "length_column_name": "length",
-        "disable_tqdm": False,
-        "weight_decay": 0.001,
-        "load_best_model_at_end": load_best_model_at_end,
-    }
-    training_args.update(default_training_args)
-
-    return training_args, freeze_layers
-
-
-def load_best_model(directory, model_type, num_classes, mode="eval"):
-    file_dict = dict()
-    for subdir, dirs, files in os.walk(directory):
-        for file in files:
-            if file.endswith("result.json"):
-                with open(f"{subdir}/{file}", "rb") as fp:
-                    result_json = json.load(fp)
-                file_dict[f"{subdir}"] = result_json["eval_macro_f1"]
-    file_df = pd.DataFrame(
-        {"dir": file_dict.keys(), "eval_macro_f1": file_dict.values()}
-    )
-    model_superdir = (
-        "run-"
-        + file_df.iloc[file_df["eval_macro_f1"].idxmax()]["dir"]
-        .split("_objective_")[2]
-        .split("_")[0]
-    )
-
-    for subdir, dirs, files in os.walk(f"{directory}/{model_superdir}"):
-        for file in files:
-            if file.endswith("model.safetensors"):
-                model = pu.load_model(model_type, num_classes, f"{subdir}", mode)
-    return model
-
-
-class StratifiedKFold3(StratifiedKFold):
-    def split(self, targets, labels, test_ratio=0.5, groups=None):
-        s = super().split(targets, labels, groups)
-        for train_indxs, test_indxs in s:
-            if test_ratio == 0:
-                yield train_indxs, test_indxs, None
-            else:
-                labels_test = np.array(labels)[test_indxs]
-                valid_indxs, test_indxs = train_test_split(
-                    test_indxs,
-                    stratify=labels_test,
-                    test_size=test_ratio,
-                    random_state=0,
-                )
-                yield train_indxs, valid_indxs, test_indxs

geneformer/collator_for_classification.py

@@ -1,667 +0,0 @@
-"""
-Geneformer collator for gene and cell classification.
-Huggingface data collator modified to accommodate single-cell transcriptomics data for gene and cell classification.
-"""
-
-import warnings
-from enum import Enum
-from typing import Dict, List, Optional, Union
-
-import numpy as np
-import torch
-from transformers import (
-    BatchEncoding,
-    DataCollatorForTokenClassification,
-    SpecialTokensMixin,
-)
-from transformers.utils import is_tf_available, is_torch_available, logging, to_py_obj
-from transformers.utils.generic import _is_tensorflow, _is_torch
-
-EncodedInput = List[int]
-logger = logging.get_logger(__name__)
-VERY_LARGE_INTEGER = int(
-    1e30
-)  # This is used to set the max input length for a model with infinite size input
-LARGE_INTEGER = int(
-    1e20
-)  # This is used when we need something big but slightly smaller than VERY_LARGE_INTEGER
-
-# precollator functions
-
-
-class ExplicitEnum(Enum):
-    """
-    Enum with more explicit error message for missing values.
-    """
-
-    @classmethod
-    def _missing_(cls, value):
-        raise ValueError(
-            "%r is not a valid %s, please select one of %s"
-            % (value, cls.__name__, str(list(cls._value2member_map_.keys())))
-        )
-
-
-class TruncationStrategy(ExplicitEnum):
-    """
-    Possible values for the ``truncation`` argument in :meth:`PreTrainedTokenizerBase.__call__`. Useful for
-    tab-completion in an IDE.
-    """
-
-    ONLY_FIRST = "only_first"
-    ONLY_SECOND = "only_second"
-    LONGEST_FIRST = "longest_first"
-    DO_NOT_TRUNCATE = "do_not_truncate"
-
-
-class PaddingStrategy(ExplicitEnum):
-    """
-    Possible values for the ``padding`` argument in :meth:`PreTrainedTokenizerBase.__call__`. Useful for tab-completion
-    in an IDE.
-    """
-
-    LONGEST = "longest"
-    MAX_LENGTH = "max_length"
-    DO_NOT_PAD = "do_not_pad"
-
-
-class TensorType(ExplicitEnum):
-    """
-    Possible values for the ``return_tensors`` argument in :meth:`PreTrainedTokenizerBase.__call__`. Useful for
-    tab-completion in an IDE.
-    """
-
-    PYTORCH = "pt"
-    TENSORFLOW = "tf"
-    NUMPY = "np"
-    JAX = "jax"
-
-
-class PrecollatorForGeneAndCellClassification(SpecialTokensMixin):
-    def __init__(self, *args, **kwargs) -> None:
-        super().__init__(mask_token="<mask>", pad_token="<pad>")
-
-        self.token_dictionary = kwargs.get("token_dictionary")
-        self.padding_side = "right"
-        self.model_input_names = ["input_ids"]
-        self._mask_token_id = self.token_dictionary.get("<mask>")
-        self._pad_token_id = self.token_dictionary.get("<pad>")
-        self._all_special_ids = [
-            self.token_dictionary.get("<mask>"),
-            self.token_dictionary.get("<pad>"),
-        ]
-
-    @property
-    def all_special_ids(self):
-        return self._all_special_ids
-
-    @property
-    def mask_token_id(self):
-        return self._mask_token_id
-
-    @property
-    def pad_token_id(self):
-        return self._pad_token_id
-
-    def _get_padding_truncation_strategies(
-        self,
-        padding=True,
-        truncation=False,
-        max_length=None,
-        pad_to_multiple_of=None,
-        verbose=True,
-        **kwargs,
-    ):
-        """
-        Find the correct padding/truncation strategy with backward compatibility for old arguments (truncation_strategy
-        and pad_to_max_length) and behaviors.
-        """
-        old_truncation_strategy = kwargs.pop("truncation_strategy", "do_not_truncate")
-        old_pad_to_max_length = kwargs.pop("pad_to_max_length", False)
-
-        # Backward compatibility for previous behavior, maybe we should deprecate it:
-        # If you only set max_length, it activates truncation for max_length
-        if max_length is not None and padding is False and truncation is False:
-            if verbose:
-                if not self.deprecation_warnings.get(
-                    "Truncation-not-explicitly-activated", False
-                ):
-                    logger.warning(
-                        "Truncation was not explicitly activated but `max_length` is provided a specific value, "
-                        "please use `truncation=True` to explicitly truncate examples to max length. "
-                        "Defaulting to 'longest_first' truncation strategy. "
-                        "If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy "
-                        "more precisely by providing a specific strategy to `truncation`."
-                    )
-                self.deprecation_warnings["Truncation-not-explicitly-activated"] = True
-            truncation = "longest_first"
-
-        # Get padding strategy
-        if padding is False and old_pad_to_max_length:
-            if verbose:
-                warnings.warn(
-                    "The `pad_to_max_length` argument is deprecated and will be removed in a future version, "
-                    "use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or "
-                    "use `padding='max_length'` to pad to a max length. In this case, you can give a specific "
-                    "length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the "
-                    "maximal input size of the model (e.g. 512 for Bert).",
-                    FutureWarning,
-                )
-            if max_length is None:
-                padding_strategy = PaddingStrategy.LONGEST
-            else:
-                padding_strategy = PaddingStrategy.MAX_LENGTH
-        elif padding is not False:
-            if padding is True:
-                padding_strategy = (
-                    PaddingStrategy.LONGEST
-                )  # Default to pad to the longest sequence in the batch
-            elif not isinstance(padding, PaddingStrategy):
-                padding_strategy = PaddingStrategy(padding)
-            elif isinstance(padding, PaddingStrategy):
-                padding_strategy = padding
-        else:
-            padding_strategy = PaddingStrategy.DO_NOT_PAD
-
-        # Get truncation strategy
-        if truncation is False and old_truncation_strategy != "do_not_truncate":
-            if verbose:
-                warnings.warn(
-                    "The `truncation_strategy` argument is deprecated and will be removed in a future version, "
-                    "use `truncation=True` to truncate examples to a max length. You can give a specific "
-                    "length with `max_length` (e.g. `max_length=45`) or leave max_length to None to truncate to the "
-                    "maximal input size of the model (e.g. 512 for Bert). "
-                    " If you have pairs of inputs, you can give a specific truncation strategy selected among "
-                    "`truncation='only_first'` (will only truncate the first sentence in the pairs) "
-                    "`truncation='only_second'` (will only truncate the second sentence in the pairs) "
-                    "or `truncation='longest_first'` (will iteratively remove tokens from the longest sentence in the pairs).",
-                    FutureWarning,
-                )
-            truncation_strategy = TruncationStrategy(old_truncation_strategy)
-        elif truncation is not False:
-            if truncation is True:
-                truncation_strategy = (
-                    TruncationStrategy.LONGEST_FIRST
-                )  # Default to truncate the longest sequences in pairs of inputs
-            elif not isinstance(truncation, TruncationStrategy):
-                truncation_strategy = TruncationStrategy(truncation)
-            elif isinstance(truncation, TruncationStrategy):
-                truncation_strategy = truncation
-        else:
-            truncation_strategy = TruncationStrategy.DO_NOT_TRUNCATE
-
-        # Set max length if needed
-        if max_length is None:
-            if padding_strategy == PaddingStrategy.MAX_LENGTH:
-                if self.model_max_length > LARGE_INTEGER:
-                    if verbose:
-                        if not self.deprecation_warnings.get(
-                            "Asking-to-pad-to-max_length", False
-                        ):
-                            logger.warning(
-                                "Asking to pad to max_length but no maximum length is provided and the model has no predefined maximum length. "
-                                "Default to no padding."
-                            )
-                        self.deprecation_warnings["Asking-to-pad-to-max_length"] = True
-                    padding_strategy = PaddingStrategy.DO_NOT_PAD
-                else:
-                    max_length = self.model_max_length
-
-            if truncation_strategy != TruncationStrategy.DO_NOT_TRUNCATE:
-                if self.model_max_length > LARGE_INTEGER:
-                    if verbose:
-                        if not self.deprecation_warnings.get(
-                            "Asking-to-truncate-to-max_length", False
-                        ):
-                            logger.warning(
-                                "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. "
-                                "Default to no truncation."
-                            )
-                        self.deprecation_warnings[
-                            "Asking-to-truncate-to-max_length"
-                        ] = True
-                    truncation_strategy = TruncationStrategy.DO_NOT_TRUNCATE
-                else:
-                    max_length = self.model_max_length
-
-        # Test if we have a padding token
-        if padding_strategy != PaddingStrategy.DO_NOT_PAD and (
-            not self.pad_token or self.pad_token_id < 0
-        ):
-            raise ValueError(
-                "Asking to pad but the tokenizer does not have a padding token. "
-                "Please select a token to use as `pad_token` `(tokenizer.pad_token = tokenizer.eos_token e.g.)` "
-                "or add a new pad token via `tokenizer.add_special_tokens({'pad_token': '[PAD]'})`."
-            )
-
-        # Check that we will truncate to a multiple of pad_to_multiple_of if both are provided
-        if (
-            truncation_strategy != TruncationStrategy.DO_NOT_TRUNCATE
-            and padding_strategy != PaddingStrategy.DO_NOT_PAD
-            and pad_to_multiple_of is not None
-            and max_length is not None
-            and (max_length % pad_to_multiple_of != 0)
-        ):
-            raise ValueError(
-                f"Truncation and padding are both activated but "
-                f"truncation length ({max_length}) is not a multiple of pad_to_multiple_of ({pad_to_multiple_of})."
-            )
-
-        return padding_strategy, truncation_strategy, max_length, kwargs
-
-    def pad(
-        self,
-        encoded_inputs: Union[
-            BatchEncoding,
-            List[BatchEncoding],
-            Dict[str, EncodedInput],
-            Dict[str, List[EncodedInput]],
-            List[Dict[str, EncodedInput]],
-        ],
-        class_type,  # options: "gene" or "cell"
-        padding: Union[bool, str, PaddingStrategy] = True,
-        max_length: Optional[int] = None,
-        pad_to_multiple_of: Optional[int] = None,
-        return_attention_mask: Optional[bool] = True,
-        return_tensors: Optional[Union[str, TensorType]] = None,
-        verbose: bool = True,
-    ) -> BatchEncoding:
-        """
-        Pad a single encoded input or a batch of encoded inputs up to predefined length or to the max sequence length
-        in the batch.
-        Padding side (left/right) padding token ids are defined at the tokenizer level (with ``self.padding_side``,
-        ``self.pad_token_id`` and ``self.pad_token_type_id``)
-        .. note::
-            If the ``encoded_inputs`` passed are dictionary of numpy arrays, PyTorch tensors or TensorFlow tensors, the
-            result will use the same type unless you provide a different tensor type with ``return_tensors``. In the
-            case of PyTorch tensors, you will lose the specific device of your tensors however.
-        Args:
-            encoded_inputs (:class:`~transformers.BatchEncoding`, list of :class:`~transformers.BatchEncoding`, :obj:`Dict[str, List[int]]`, :obj:`Dict[str, List[List[int]]` or :obj:`List[Dict[str, List[int]]]`):
-                Tokenized inputs. Can represent one input (:class:`~transformers.BatchEncoding` or :obj:`Dict[str,
-                List[int]]`) or a batch of tokenized inputs (list of :class:`~transformers.BatchEncoding`, `Dict[str,
-                List[List[int]]]` or `List[Dict[str, List[int]]]`) so you can use this method during preprocessing as
-                well as in a PyTorch Dataloader collate function.
-                Instead of :obj:`List[int]` you can have tensors (numpy arrays, PyTorch tensors or TensorFlow tensors),
-                see the note above for the return type.
-            padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`):
-                 Select a strategy to pad the returned sequences (according to the model's padding side and padding
-                 index) among:
-                * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a
-                  single sequence if provided).
-                * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the
-                  maximum acceptable input length for the model if that argument is not provided.
-                * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
-                  different lengths).
-            max_length (:obj:`int`, `optional`):
-                Maximum length of the returned list and optionally padding length (see above).
-            pad_to_multiple_of (:obj:`int`, `optional`):
-                If set will pad the sequence to a multiple of the provided value.
-                This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
-                >= 7.5 (Volta).
-            return_attention_mask (:obj:`bool`, `optional`):
-                Whether to return the attention mask. If left to the default, will return the attention mask according
-                to the specific tokenizer's default, defined by the :obj:`return_outputs` attribute.
-                `What are attention masks? <../glossary.html#attention-mask>`__
-            return_tensors (:obj:`str` or :class:`~transformers.tokenization_utils_base.TensorType`, `optional`):
-                If set, will return tensors instead of list of python integers. Acceptable values are:
-                * :obj:`'tf'`: Return TensorFlow :obj:`tf.constant` objects.
-                * :obj:`'pt'`: Return PyTorch :obj:`torch.Tensor` objects.
-                * :obj:`'np'`: Return Numpy :obj:`np.ndarray` objects.
-            verbose (:obj:`bool`, `optional`, defaults to :obj:`True`):
-                Whether or not to print more information and warnings.
-        """
-        # If we have a list of dicts, let's convert it in a dict of lists
-        # We do this to allow using this method as a collate_fn function in PyTorch Dataloader
-        if isinstance(encoded_inputs, (list, tuple)) and isinstance(
-            encoded_inputs[0], (dict, BatchEncoding)
-        ):
-            encoded_inputs = {
-                key: [example[key] for example in encoded_inputs]
-                for key in encoded_inputs[0].keys()
-            }
-
-        # The model's main input name, usually `input_ids`, has be passed for padding
-        if self.model_input_names[0] not in encoded_inputs:
-            raise ValueError(
-                "You should supply an encoding or a list of encodings to this method"
-                f"that includes {self.model_input_names[0]}, but you provided {list(encoded_inputs.keys())}"
-            )
-
-        required_input = encoded_inputs[self.model_input_names[0]]
-
-        if not required_input:
-            if return_attention_mask:
-                encoded_inputs["attention_mask"] = []
-            return encoded_inputs
-
-        # If we have PyTorch/TF/NumPy tensors/arrays as inputs, we cast them as python objects
-        # and rebuild them afterwards if no return_tensors is specified
-        # Note that we lose the specific device the tensor may be on for PyTorch
-
-        first_element = required_input[0]
-        if isinstance(first_element, (list, tuple)):
-            # first_element might be an empty list/tuple in some edge cases so we grab the first non empty element.
-            index = 0
-            while len(required_input[index]) == 0:
-                index += 1
-            if index < len(required_input):
-                first_element = required_input[index][0]
-        # At this state, if `first_element` is still a list/tuple, it's an empty one so there is nothing to do.
-        if not isinstance(first_element, (int, list, tuple)):
-            if is_tf_available() and _is_tensorflow(first_element):
-                return_tensors = "tf" if return_tensors is None else return_tensors
-            elif is_torch_available() and _is_torch(first_element):
-                return_tensors = "pt" if return_tensors is None else return_tensors
-            elif isinstance(first_element, np.ndarray):
-                return_tensors = "np" if return_tensors is None else return_tensors
-            else:
-                raise ValueError(
-                    f"type of {first_element} unknown: {type(first_element)}. "
-                    f"Should be one of a python, numpy, pytorch or tensorflow object."
-                )
-
-            for key, value in encoded_inputs.items():
-                encoded_inputs[key] = to_py_obj(value)
-
-        # Convert padding_strategy in PaddingStrategy
-        padding_strategy, _, max_length, _ = self._get_padding_truncation_strategies(
-            padding=padding, max_length=max_length, verbose=verbose
-        )
-
-        required_input = encoded_inputs[self.model_input_names[0]]
-        if required_input and not isinstance(required_input[0], (list, tuple)):
-            encoded_inputs = self._pad(
-                encoded_inputs,
-                class_type=class_type,
-                max_length=max_length,
-                padding_strategy=padding_strategy,
-                pad_to_multiple_of=pad_to_multiple_of,
-                return_attention_mask=return_attention_mask,
-            )
-            return BatchEncoding(encoded_inputs, tensor_type=return_tensors)
-
-        batch_size = len(required_input)
-        assert all(
-            len(v) == batch_size for v in encoded_inputs.values()
-        ), "Some items in the output dictionary have a different batch size than others."
-
-        if padding_strategy == PaddingStrategy.LONGEST:
-            max_length = max(len(inputs) for inputs in required_input)
-            padding_strategy = PaddingStrategy.MAX_LENGTH
-
-        batch_outputs = {}
-        for i in range(batch_size):
-            inputs = dict((k, v[i]) for k, v in encoded_inputs.items())
-            outputs = self._pad(
-                inputs,
-                class_type=class_type,
-                max_length=max_length,
-                padding_strategy=padding_strategy,
-                pad_to_multiple_of=pad_to_multiple_of,
-                return_attention_mask=return_attention_mask,
-            )
-
-            for key, value in outputs.items():
-                if key not in batch_outputs:
-                    batch_outputs[key] = []
-                batch_outputs[key].append(value)
-        if class_type == "cell":
-            del batch_outputs["label"]
-        return BatchEncoding(batch_outputs, tensor_type=return_tensors)
-
-    def _pad(
-        self,
-        encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
-        class_type,  # options: "gene" or "cell"
-        max_length: Optional[int] = None,
-        padding_strategy: PaddingStrategy = PaddingStrategy.LONGEST,
-        pad_to_multiple_of: Optional[int] = None,
-        return_attention_mask: Optional[bool] = True,
-    ) -> dict:
-        """
-        Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
-        Args:
-            encoded_inputs: Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
-            max_length: maximum length of the returned list and optionally padding length (see below).
-                Will truncate by taking into account the special tokens.
-            padding_strategy: PaddingStrategy to use for padding.
-                - PaddingStrategy.LONGEST Pad to the longest sequence in the batch
-                - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)
-                - PaddingStrategy.DO_NOT_PAD: Do not pad
-                The tokenizer padding sides are defined in self.padding_side:
-                    - 'left': pads on the left of the sequences
-                    - 'right': pads on the right of the sequences
-            pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
-                This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
-                >= 7.5 (Volta).
-            return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics)
-        """
-        # Load from model defaults
-        if return_attention_mask is None:
-            return_attention_mask = "attention_mask" in self.model_input_names
-
-        required_input = encoded_inputs[self.model_input_names[0]]
-
-        if padding_strategy == PaddingStrategy.LONGEST:
-            max_length = len(required_input)
-
-        if (
-            max_length is not None
-            and pad_to_multiple_of is not None
-            and (max_length % pad_to_multiple_of != 0)
-        ):
-            max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of
-
-        needs_to_be_padded = (
-            padding_strategy != PaddingStrategy.DO_NOT_PAD
-            and len(required_input) != max_length
-        )
-
-        if needs_to_be_padded:
-            difference = max_length - len(required_input)
-            if self.padding_side == "right":
-                if return_attention_mask:
-                    encoded_inputs["attention_mask"] = [1] * len(required_input) + [
-                        0
-                    ] * difference
-                if "token_type_ids" in encoded_inputs:
-                    encoded_inputs["token_type_ids"] = (
-                        encoded_inputs["token_type_ids"]
-                        + [self.pad_token_type_id] * difference
-                    )
-                if "special_tokens_mask" in encoded_inputs:
-                    encoded_inputs["special_tokens_mask"] = (
-                        encoded_inputs["special_tokens_mask"] + [1] * difference
-                    )
-                encoded_inputs[self.model_input_names[0]] = (
-                    required_input + [self.pad_token_id] * difference
-                )
-                if class_type == "gene":
-                    encoded_inputs["labels"] = (
-                        encoded_inputs["labels"] + [-100] * difference
-                    )
-            elif self.padding_side == "left":
-                if return_attention_mask:
-                    encoded_inputs["attention_mask"] = [0] * difference + [1] * len(
-                        required_input
-                    )
-                if "token_type_ids" in encoded_inputs:
-                    encoded_inputs["token_type_ids"] = [
-                        self.pad_token_type_id
-                    ] * difference + encoded_inputs["token_type_ids"]
-                if "special_tokens_mask" in encoded_inputs:
-                    encoded_inputs["special_tokens_mask"] = [
-                        1
-                    ] * difference + encoded_inputs["special_tokens_mask"]
-                encoded_inputs[self.model_input_names[0]] = [
-                    self.pad_token_id
-                ] * difference + required_input
-                if class_type == "gene":
-                    encoded_inputs["labels"] = [-100] * difference + encoded_inputs[
-                        "labels"
-                    ]
-            else:
-                raise ValueError("Invalid padding strategy:" + str(self.padding_side))
-        elif return_attention_mask and "attention_mask" not in encoded_inputs:
-            encoded_inputs["attention_mask"] = [1] * len(required_input)
-
-        return encoded_inputs
-
-    def get_special_tokens_mask(
-        self,
-        token_ids_0: List[int],
-        token_ids_1: Optional[List[int]] = None,
-        already_has_special_tokens: bool = False,
-    ) -> List[int]:
-        """
-        Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
-        special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods.
-        Args:
-            token_ids_0 (:obj:`List[int]`):
-                List of ids of the first sequence.
-            token_ids_1 (:obj:`List[int]`, `optional`):
-                List of ids of the second sequence.
-            already_has_special_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`):
-                Whether or not the token list is already formatted with special tokens for the model.
-        Returns:
-            A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
-        """
-        assert already_has_special_tokens and token_ids_1 is None, (
-            "You cannot use ``already_has_special_tokens=False`` with this tokenizer. "
-            "Please use a slow (full python) tokenizer to activate this argument."
-            "Or set `return_special_tokens_mask=True` when calling the encoding method "
-            "to get the special tokens mask in any tokenizer. "
-        )
-
-        all_special_ids = self.all_special_ids  # cache the property
-
-        special_tokens_mask = [
-            1 if token in all_special_ids else 0 for token in token_ids_0
-        ]
-
-        return special_tokens_mask
-
-    def convert_tokens_to_ids(
-        self, tokens: Union[str, List[str]]
-    ) -> Union[int, List[int]]:
-        """
-        Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the
-        vocabulary.
-        Args:
-            tokens (:obj:`str` or :obj:`List[str]`): One or several token(s) to convert to token id(s).
-        Returns:
-            :obj:`int` or :obj:`List[int]`: The token id or list of token ids.
-        """
-        if tokens is None:
-            return None
-
-        if isinstance(tokens, str):
-            return self._convert_token_to_id_with_added_voc(tokens)
-
-        ids = []
-        for token in tokens:
-            ids.append(self._convert_token_to_id_with_added_voc(token))
-        return ids
-
-    def _convert_token_to_id_with_added_voc(self, token):
-        if token is None:
-            return None
-
-        return self.token_dictionary.get(token)
-
-    def __len__(self):
-        return len(self.token_dictionary)
-
-
-# collator functions
-
-
-class DataCollatorForGeneClassification(DataCollatorForTokenClassification):
-    """
-    Data collator that will dynamically pad the inputs received, as well as the labels.
-    Args:
-        tokenizer (:class:`~transformers.PreTrainedTokenizer` or :class:`~transformers.PreTrainedTokenizerFast`):
-            The tokenizer used for encoding the data.
-        padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`):
-            Select a strategy to pad the returned sequences (according to the model's padding side and padding index)
-            among:
-            * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
-              sequence if provided).
-            * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the
-              maximum acceptable input length for the model if that argument is not provided.
-            * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
-              different lengths).
-        max_length (:obj:`int`, `optional`):
-            Maximum length of the returned list and optionally padding length (see above).
-        pad_to_multiple_of (:obj:`int`, `optional`):
-            If set will pad the sequence to a multiple of the provided value.
-            This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >=
-            7.5 (Volta).
-        label_pad_token_id (:obj:`int`, `optional`, defaults to -100):
-            The id to use when padding the labels (-100 will be automatically ignore by PyTorch loss functions).
-    """
-
-    class_type = "gene"
-    padding: Union[bool, str, PaddingStrategy] = True
-    max_length: Optional[int] = None
-    pad_to_multiple_of: Optional[int] = None
-    label_pad_token_id: int = -100
-
-    def __init__(self, *args, **kwargs) -> None:
-        self.token_dictionary = kwargs.pop("token_dictionary")
-        super().__init__(
-            tokenizer=PrecollatorForGeneAndCellClassification(
-                token_dictionary=self.token_dictionary
-            ),
-            padding=self.padding,
-            max_length=self.max_length,
-            pad_to_multiple_of=self.pad_to_multiple_of,
-            label_pad_token_id=self.label_pad_token_id,
-            *args,
-            **kwargs,
-        )
-
-    def _prepare_batch(self, features):
-        label_name = "label" if "label" in features[0].keys() else "labels"
-        labels = (
-            [feature[label_name] for feature in features]
-            if label_name in features[0].keys()
-            else None
-        )
-        batch = self.tokenizer.pad(
-            features,
-            class_type=self.class_type,
-            padding=self.padding,
-            max_length=self.max_length,
-            pad_to_multiple_of=self.pad_to_multiple_of,
-            return_tensors="pt",
-        )
-        return batch
-
-    def __call__(self, features):
-        batch = self._prepare_batch(features)
-
-        batch = {k: torch.tensor(v, dtype=torch.int64) for k, v in batch.items()}
-        return batch
-
-
-class DataCollatorForCellClassification(DataCollatorForGeneClassification):
-    class_type = "cell"
-
-    def _prepare_batch(self, features):
-        batch = super()._prepare_batch(features)
-
-        # Special handling for labels.
-        # Ensure that tensor is created with the correct type
-        # (it should be automatically the case, but let's make sure of it.)
-        first = features[0]
-        if "label" in first and first["label"] is not None:
-            label = (
-                first["label"].item()
-                if isinstance(first["label"], torch.Tensor)
-                else first["label"]
-            )
-            dtype = torch.long if isinstance(label, int) else torch.float
-            batch["labels"] = torch.tensor([f["label"] for f in features], dtype=dtype)
-
-        return batch

geneformer/emb_extractor.py

@@ -1,863 +0,0 @@
-"""
-Geneformer embedding extractor.
-
-**Description:**
-
-| Extracts gene or cell embeddings.
-| Plots cell embeddings as heatmaps or UMAPs.
-| Generates cell state embedding dictionary for use with InSilicoPerturber.
-
-"""
-
-# imports
-import logging
-import pickle
-from collections import Counter
-from pathlib import Path
-
-import anndata
-import matplotlib.pyplot as plt
-import pandas as pd
-import scanpy as sc
-import seaborn as sns
-import torch
-from tdigest import TDigest
-from tqdm.auto import trange
-
-from . import TOKEN_DICTIONARY_FILE
-from . import perturber_utils as pu
-
-logger = logging.getLogger(__name__)
-
-
-# extract embeddings
-def get_embs(
-    model,
-    filtered_input_data,
-    emb_mode,
-    layer_to_quant,
-    pad_token_id,
-    forward_batch_size,
-    token_gene_dict,
-    special_token=False,
-    summary_stat=None,
-    silent=False,
-):
-    model_input_size = pu.get_model_input_size(model)
-    total_batch_length = len(filtered_input_data)
-
-    if summary_stat is None:
-        embs_list = []
-    elif summary_stat is not None:
-        # get # of emb dims
-        emb_dims = pu.get_model_emb_dims(model)
-        if emb_mode == "cell":
-            # initiate tdigests for # of emb dims
-            embs_tdigests = [TDigest() for _ in range(emb_dims)]
-        if emb_mode == "gene":
-            gene_set = list(
-                {
-                    element
-                    for sublist in filtered_input_data["input_ids"]
-                    for element in sublist
-                }
-            )
-            # initiate dict with genes as keys and tdigests for # of emb dims as values
-            embs_tdigests_dict = {
-                k: [TDigest() for _ in range(emb_dims)] for k in gene_set
-            }
-
-    # Check if CLS and EOS token is present in the token dictionary
-    cls_present = any("<cls>" in value for value in token_gene_dict.values())
-    eos_present = any("<eos>" in value for value in token_gene_dict.values())
-    if emb_mode == "cls":
-        assert cls_present, "<cls> token missing in token dictionary"
-        # Check to make sure that the first token of the filtered input data is cls token
-        gene_token_dict = {v: k for k, v in token_gene_dict.items()}
-        cls_token_id = gene_token_dict["<cls>"]
-        assert (
-            filtered_input_data["input_ids"][0][0] == cls_token_id
-        ), "First token is not <cls> token value"
-    elif emb_mode == "cell":
-        if cls_present:
-            logger.warning(
-                "CLS token present in token dictionary, excluding from average."
-            )
-        if eos_present:
-            logger.warning(
-                "EOS token present in token dictionary, excluding from average."
-            )
-
-    overall_max_len = 0
-
-    for i in trange(0, total_batch_length, forward_batch_size, leave=(not silent)):
-        max_range = min(i + forward_batch_size, total_batch_length)
-
-        minibatch = filtered_input_data.select([i for i in range(i, max_range)])
-
-        max_len = int(max(minibatch["length"]))
-        original_lens = torch.tensor(minibatch["length"], device="cuda")
-        minibatch.set_format(type="torch")
-
-        input_data_minibatch = minibatch["input_ids"]
-        input_data_minibatch = pu.pad_tensor_list(
-            input_data_minibatch, max_len, pad_token_id, model_input_size
-        )
-
-        with torch.no_grad():
-            outputs = model(
-                input_ids=input_data_minibatch.to("cuda"),
-                attention_mask=pu.gen_attention_mask(minibatch),
-            )
-
-        embs_i = outputs.hidden_states[layer_to_quant]
-
-        if emb_mode == "cell":
-            if cls_present:
-                non_cls_embs = embs_i[:, 1:, :]  # Get all layers except the embs
-                if eos_present:
-                    mean_embs = pu.mean_nonpadding_embs(non_cls_embs, original_lens - 2)
-                else:
-                    mean_embs = pu.mean_nonpadding_embs(non_cls_embs, original_lens - 1)
-            else:
-                mean_embs = pu.mean_nonpadding_embs(embs_i, original_lens)
-            if summary_stat is None:
-                embs_list.append(mean_embs)
-            elif summary_stat is not None:
-                # update tdigests with current batch for each emb dim
-                accumulate_tdigests(embs_tdigests, mean_embs, emb_dims)
-            del mean_embs
-        elif emb_mode == "gene":
-            if summary_stat is None:
-                embs_list.append(embs_i)
-            elif summary_stat is not None:
-                for h in trange(len(minibatch)):
-                    length_h = minibatch[h]["length"]
-                    input_ids_h = minibatch[h]["input_ids"][0:length_h]
-
-                    # double check dimensions before unsqueezing
-                    embs_i_dim = embs_i.dim()
-                    if embs_i_dim != 3:
-                        logger.error(
-                            f"Embedding tensor should have 3 dimensions, not {embs_i_dim}"
-                        )
-                        raise
-
-                    embs_h = embs_i[h, :, :].unsqueeze(dim=1)
-                    dict_h = dict(zip(input_ids_h, embs_h))
-                    for k in dict_h.keys():
-                        accumulate_tdigests(
-                            embs_tdigests_dict[int(k)], dict_h[k], emb_dims
-                        )
-                    del embs_h
-                    del dict_h
-        elif emb_mode == "cls":
-            cls_embs = embs_i[:, 0, :].clone().detach()  # CLS token layer
-            embs_list.append(cls_embs)
-            del cls_embs
-
-        overall_max_len = max(overall_max_len, max_len)
-        del outputs
-        del minibatch
-        del input_data_minibatch
-        del embs_i
-
-        torch.cuda.empty_cache()
-
-    if summary_stat is None:
-        if (emb_mode == "cell") or (emb_mode == "cls"):
-            embs_stack = torch.cat(embs_list, dim=0)
-        elif emb_mode == "gene":
-            embs_stack = pu.pad_tensor_list(
-                embs_list,
-                overall_max_len,
-                pad_token_id,
-                model_input_size,
-                1,
-                pu.pad_3d_tensor,
-            )
-
-    # calculate summary stat embs from approximated tdigests
-    elif summary_stat is not None:
-        if emb_mode == "cell":
-            if summary_stat == "mean":
-                summary_emb_list = tdigest_mean(embs_tdigests, emb_dims)
-            elif summary_stat == "median":
-                summary_emb_list = tdigest_median(embs_tdigests, emb_dims)
-            embs_stack = torch.tensor(summary_emb_list)
-        elif emb_mode == "gene":
-            if summary_stat == "mean":
-                [
-                    update_tdigest_dict_mean(embs_tdigests_dict, gene, emb_dims)
-                    for gene in embs_tdigests_dict.keys()
-                ]
-            elif summary_stat == "median":
-                [
-                    update_tdigest_dict_median(embs_tdigests_dict, gene, emb_dims)
-                    for gene in embs_tdigests_dict.keys()
-                ]
-            return embs_tdigests_dict
-
-    return embs_stack
-
-
-def accumulate_tdigests(embs_tdigests, mean_embs, emb_dims):
-    # note: tdigest batch update known to be slow so updating serially
-    [
-        embs_tdigests[j].update(mean_embs[i, j].item())
-        for i in range(mean_embs.size(0))
-        for j in range(emb_dims)
-    ]
-
-
-def update_tdigest_dict(embs_tdigests_dict, gene, gene_embs, emb_dims):
-    embs_tdigests_dict[gene] = accumulate_tdigests(
-        embs_tdigests_dict[gene], gene_embs, emb_dims
-    )
-
-
-def update_tdigest_dict_mean(embs_tdigests_dict, gene, emb_dims):
-    embs_tdigests_dict[gene] = tdigest_mean(embs_tdigests_dict[gene], emb_dims)
-
-
-def update_tdigest_dict_median(embs_tdigests_dict, gene, emb_dims):
-    embs_tdigests_dict[gene] = tdigest_median(embs_tdigests_dict[gene], emb_dims)
-
-
-def summarize_gene_embs(h, minibatch, embs_i, embs_tdigests_dict, emb_dims):
-    length_h = minibatch[h]["length"]
-    input_ids_h = minibatch[h]["input_ids"][0:length_h]
-    embs_h = embs_i[h, :, :].unsqueeze(dim=1)
-    dict_h = dict(zip(input_ids_h, embs_h))
-    [
-        update_tdigest_dict(embs_tdigests_dict, k, dict_h[k], emb_dims)
-        for k in dict_h.keys()
-    ]
-
-
-def tdigest_mean(embs_tdigests, emb_dims):
-    return [embs_tdigests[i].trimmed_mean(0, 100) for i in range(emb_dims)]
-
-
-def tdigest_median(embs_tdigests, emb_dims):
-    return [embs_tdigests[i].percentile(50) for i in range(emb_dims)]
-
-
-def label_cell_embs(embs, downsampled_data, emb_labels):
-    embs_df = pd.DataFrame(embs.cpu().numpy())
-    if emb_labels is not None:
-        for label in emb_labels:
-            emb_label = downsampled_data[label]
-            embs_df[label] = emb_label
-    return embs_df
-
-
-def label_gene_embs(embs, downsampled_data, token_gene_dict):
-    gene_set = {
-        element for sublist in downsampled_data["input_ids"] for element in sublist
-    }
-    gene_emb_dict = {k: [] for k in gene_set}
-    for i in range(embs.size()[0]):
-        length = downsampled_data[i]["length"]
-        dict_i = dict(
-            zip(
-                downsampled_data[i]["input_ids"][0:length],
-                embs[i, :, :].unsqueeze(dim=1),
-            )
-        )
-        for k in dict_i.keys():
-            gene_emb_dict[k].append(dict_i[k])
-    for k in gene_emb_dict.keys():
-        gene_emb_dict[k] = (
-            torch.squeeze(torch.mean(torch.stack(gene_emb_dict[k]), dim=0), dim=0)
-            .cpu()
-            .numpy()
-        )
-    embs_df = pd.DataFrame(gene_emb_dict).T
-    embs_df.index = [token_gene_dict[token] for token in embs_df.index]
-    return embs_df
-
-
-def plot_umap(embs_df, emb_dims, label, output_file, kwargs_dict, seed=0):
-    only_embs_df = embs_df.iloc[:, :emb_dims]
-    only_embs_df.index = pd.RangeIndex(0, only_embs_df.shape[0], name=None).astype(str)
-    only_embs_df.columns = pd.RangeIndex(0, only_embs_df.shape[1], name=None).astype(
-        str
-    )
-    vars_dict = {"embs": only_embs_df.columns}
-    obs_dict = {"cell_id": list(only_embs_df.index), f"{label}": list(embs_df[label])}
-    adata = anndata.AnnData(X=only_embs_df, obs=obs_dict, var=vars_dict)
-    sc.tl.pca(adata, svd_solver="arpack")
-    sc.pp.neighbors(adata, random_state=seed)
-    sc.tl.umap(adata, random_state=seed)
-    sns.set(rc={"figure.figsize": (10, 10)}, font_scale=2.3)
-    sns.set_style("white")
-    default_kwargs_dict = {"size": 200}
-    if kwargs_dict is not None:
-        default_kwargs_dict.update(kwargs_dict)
-
-    cats = set(embs_df[label])
-
-    with plt.rc_context():
-        ax = sc.pl.umap(adata, color=label, show=False, **default_kwargs_dict)
-        ax.legend(
-            markerscale=2,
-            frameon=False,
-            loc="center left",
-            bbox_to_anchor=(1, 0.5),
-            ncol=(1 if len(cats) <= 14 else 2 if len(cats) <= 30 else 3),
-        )
-        plt.show()
-        plt.savefig(output_file, bbox_inches="tight")
-
-
-def gen_heatmap_class_colors(labels, df):
-    pal = sns.cubehelix_palette(
-        len(Counter(labels).keys()),
-        light=0.9,
-        dark=0.1,
-        hue=1,
-        reverse=True,
-        start=1,
-        rot=-2,
-    )
-    lut = dict(zip(map(str, Counter(labels).keys()), pal))
-    colors = pd.Series(labels, index=df.index).map(lut)
-    return colors
-
-
-def gen_heatmap_class_dict(classes, label_colors_series):
-    class_color_dict_df = pd.DataFrame(
-        {"classes": classes, "color": label_colors_series}
-    )
-    class_color_dict_df = class_color_dict_df.drop_duplicates(subset=["classes"])
-    return dict(zip(class_color_dict_df["classes"], class_color_dict_df["color"]))
-
-
-def make_colorbar(embs_df, label):
-    labels = list(embs_df[label])
-
-    cell_type_colors = gen_heatmap_class_colors(labels, embs_df)
-    label_colors = pd.DataFrame(cell_type_colors, columns=[label])
-
-    # create dictionary for colors and classes
-    label_color_dict = gen_heatmap_class_dict(labels, label_colors[label])
-    return label_colors, label_color_dict
-
-
-def plot_heatmap(embs_df, emb_dims, label, output_file, kwargs_dict):
-    sns.set_style("white")
-    sns.set(font_scale=2)
-    plt.figure(figsize=(15, 15), dpi=150)
-    label_colors, label_color_dict = make_colorbar(embs_df, label)
-
-    default_kwargs_dict = {
-        "row_cluster": True,
-        "col_cluster": True,
-        "row_colors": label_colors,
-        "standard_scale": 1,
-        "linewidths": 0,
-        "xticklabels": False,
-        "yticklabels": False,
-        "figsize": (15, 15),
-        "center": 0,
-        "cmap": "magma",
-    }
-
-    if kwargs_dict is not None:
-        default_kwargs_dict.update(kwargs_dict)
-    g = sns.clustermap(
-        embs_df.iloc[:, 0:emb_dims].apply(pd.to_numeric), **default_kwargs_dict
-    )
-
-    plt.setp(g.ax_row_colors.get_xmajorticklabels(), rotation=45, ha="right")
-
-    for label_color in list(label_color_dict.keys()):
-        g.ax_col_dendrogram.bar(
-            0, 0, color=label_color_dict[label_color], label=label_color, linewidth=0
-        )
-
-        g.ax_col_dendrogram.legend(
-            title=f"{label}",
-            loc="lower center",
-            ncol=4,
-            bbox_to_anchor=(0.5, 1),
-            facecolor="white",
-        )
-    plt.show()
-    logger.info(f"Output file: {output_file}")
-    plt.savefig(output_file, bbox_inches="tight")
-
-
-class EmbExtractor:
-    valid_option_dict = {
-        "model_type": {"Pretrained", "GeneClassifier", "CellClassifier"},
-        "num_classes": {int},
-        "emb_mode": {"cls", "cell", "gene"},
-        "cell_emb_style": {"mean_pool"},
-        "gene_emb_style": {"mean_pool"},
-        "filter_data": {None, dict},
-        "max_ncells": {None, int},
-        "emb_layer": {-1, 0},
-        "emb_label": {None, list},
-        "labels_to_plot": {None, list},
-        "forward_batch_size": {int},
-        "token_dictionary_file": {None, str},
-        "nproc": {int},
-        "summary_stat": {None, "mean", "median", "exact_mean", "exact_median"},
-    }
-
-    def __init__(
-        self,
-        model_type="Pretrained",
-        num_classes=0,
-        emb_mode="cls",
-        cell_emb_style="mean_pool",
-        gene_emb_style="mean_pool",
-        filter_data=None,
-        max_ncells=1000,
-        emb_layer=-1,
-        emb_label=None,
-        labels_to_plot=None,
-        forward_batch_size=100,
-        nproc=4,
-        summary_stat=None,
-        token_dictionary_file=None,
-    ):
-        """
-        Initialize embedding extractor.
-
-        **Parameters:**
-
-        model_type : {"Pretrained", "GeneClassifier", "CellClassifier"}
-            | Whether model is the pretrained Geneformer or a fine-tuned gene or cell classifier.
-        num_classes : int
-            | If model is a gene or cell classifier, specify number of classes it was trained to classify.
-            | For the pretrained Geneformer model, number of classes is 0 as it is not a classifier.
-        emb_mode : {"cls", "cell", "gene"}
-            | Whether to output CLS, cell, or gene embeddings.
-            | CLS embeddings are cell embeddings derived from the CLS token in the front of the rank value encoding.
-        cell_emb_style : {"mean_pool"}
-            | Method for summarizing cell embeddings if not using CLS token.
-            | Currently only option is mean pooling of gene embeddings for given cell.
-        gene_emb_style : "mean_pool"
-            | Method for summarizing gene embeddings.
-            | Currently only option is mean pooling of contextual gene embeddings for given gene.
-        filter_data : None, dict
-            | Default is to extract embeddings from all input data.
-            | Otherwise, dictionary specifying .dataset column name and list of values to filter by.
-        max_ncells : None, int
-            | Maximum number of cells to extract embeddings from.
-            | Default is 1000 cells randomly sampled from input data.
-            | If None, will extract embeddings from all cells.
-        emb_layer : {-1, 0}
-            | Embedding layer to extract.
-            | The last layer is most specifically weighted to optimize the given learning objective.
-            | Generally, it is best to extract the 2nd to last layer to get a more general representation.
-            | -1: 2nd to last layer
-            | 0: last layer
-        emb_label : None, list
-            | List of column name(s) in .dataset to add as labels to embedding output.
-        labels_to_plot : None, list
-            | Cell labels to plot.
-            | Shown as color bar in heatmap.
-            | Shown as cell color in umap.
-            | Plotting umap requires labels to plot.
-        forward_batch_size : int
-            | Batch size for forward pass.
-        nproc : int
-            | Number of CPU processes to use.
-        summary_stat : {None, "mean", "median", "exact_mean", "exact_median"}
-            | If exact_mean or exact_median, outputs only exact mean or median embedding of input data.
-            | If mean or median, outputs only approximated mean or median embedding of input data.
-            | Non-exact recommended if encountering memory constraints while generating goal embedding positions.
-            | Non-exact is slower but more memory-efficient.
-        token_dictionary_file : Path
-            | Default is the Geneformer token dictionary
-            | Path to pickle file containing token dictionary (Ensembl ID:token).
-
-        **Examples:**
-
-        .. code-block :: python
-
-            >>> from geneformer import EmbExtractor
-            >>> embex = EmbExtractor(model_type="CellClassifier",
-            ...         num_classes=3,
-            ...         emb_mode="cell",
-            ...         filter_data={"cell_type":["cardiomyocyte"]},
-            ...         max_ncells=1000,
-            ...         emb_layer=-1,
-            ...         emb_label=["disease", "cell_type"],
-            ...         labels_to_plot=["disease", "cell_type"])
-
-        """
-
-        self.model_type = model_type
-        self.num_classes = num_classes
-        self.emb_mode = emb_mode
-        self.cell_emb_style = cell_emb_style
-        self.gene_emb_style = gene_emb_style
-        self.filter_data = filter_data
-        self.max_ncells = max_ncells
-        self.emb_layer = emb_layer
-        self.emb_label = emb_label
-        self.labels_to_plot = labels_to_plot
-        self.token_dictionary_file = token_dictionary_file
-        self.forward_batch_size = forward_batch_size
-        self.nproc = nproc
-        if (summary_stat is not None) and ("exact" in summary_stat):
-            self.summary_stat = None
-            self.exact_summary_stat = summary_stat
-        else:
-            self.summary_stat = summary_stat
-            self.exact_summary_stat = None
-
-        self.validate_options()
-
-        # load token dictionary (Ensembl IDs:token)
-        if self.token_dictionary_file is None:
-            token_dictionary_file = TOKEN_DICTIONARY_FILE
-        with open(token_dictionary_file, "rb") as f:
-            self.gene_token_dict = pickle.load(f)
-
-        self.token_gene_dict = {v: k for k, v in self.gene_token_dict.items()}
-        self.pad_token_id = self.gene_token_dict.get("<pad>")
-
-    def validate_options(self):
-        # confirm arguments are within valid options and compatible with each other
-        for attr_name, valid_options in self.valid_option_dict.items():
-            attr_value = self.__dict__[attr_name]
-            if not isinstance(attr_value, (list, dict)):
-                if attr_value in valid_options:
-                    continue
-            valid_type = False
-            for option in valid_options:
-                if (option in [int, list, dict, bool, str]) and isinstance(
-                    attr_value, option
-                ):
-                    valid_type = True
-                    break
-            if valid_type:
-                continue
-            logger.error(
-                f"Invalid option for {attr_name}. "
-                f"Valid options for {attr_name}: {valid_options}"
-            )
-            raise
-
-        if self.filter_data is not None:
-            for key, value in self.filter_data.items():
-                if not isinstance(value, list):
-                    self.filter_data[key] = [value]
-                    logger.warning(
-                        "Values in filter_data dict must be lists. "
-                        f"Changing {key} value to list ([{value}])."
-                    )
-
-    def extract_embs(
-        self,
-        model_directory,
-        input_data_file,
-        output_directory,
-        output_prefix,
-        output_torch_embs=False,
-        cell_state=None,
-    ):
-        """
-        Extract embeddings from input data and save as results in output_directory.
-
-        **Parameters:**
-
-        model_directory : Path
-            | Path to directory containing model
-        input_data_file : Path
-            | Path to directory containing .dataset inputs
-        output_directory : Path
-            | Path to directory where embedding data will be saved as csv
-        output_prefix : str
-            | Prefix for output file
-        output_torch_embs : bool
-            | Whether or not to also output the embeddings as a tensor.
-            | Note, if true, will output embeddings as both dataframe and tensor.
-        cell_state : dict
-            | Cell state key and value for state embedding extraction.
-
-        **Examples:**
-
-        .. code-block :: python
-
-            >>> embs = embex.extract_embs("path/to/model",
-            ...                           "path/to/input_data",
-            ...                           "path/to/output_directory",
-            ...                           "output_prefix")
-
-        """
-
-        filtered_input_data = pu.load_and_filter(
-            self.filter_data, self.nproc, input_data_file
-        )
-
-        # Check to make sure that all the labels exist in the tokenized data:
-        if self.emb_label is not None:
-            for label in self.emb_label:
-                assert label in filtered_input_data.features.keys(), f"Attribute `{label}` not present in dataset features"
-
-        if cell_state is not None:
-            filtered_input_data = pu.filter_by_dict(
-                filtered_input_data, cell_state, self.nproc
-            )
-        downsampled_data = pu.downsample_and_sort(filtered_input_data, self.max_ncells)
-        model = pu.load_model(
-            self.model_type, self.num_classes, model_directory, mode="eval"
-        )
-        layer_to_quant = pu.quant_layers(model) + self.emb_layer
-        embs = get_embs(
-            model=model,
-            filtered_input_data=downsampled_data,
-            emb_mode=self.emb_mode,
-            layer_to_quant=layer_to_quant,
-            pad_token_id=self.pad_token_id,
-            forward_batch_size=self.forward_batch_size,
-            token_gene_dict=self.token_gene_dict,
-            summary_stat=self.summary_stat,
-        )
-
-        if self.emb_mode == "cell":
-            if self.summary_stat is None:
-                embs_df = label_cell_embs(embs, downsampled_data, self.emb_label)
-            elif self.summary_stat is not None:
-                embs_df = pd.DataFrame(embs.cpu().numpy()).T
-        elif self.emb_mode == "gene":
-            if self.summary_stat is None:
-                embs_df = label_gene_embs(embs, downsampled_data, self.token_gene_dict)
-            elif self.summary_stat is not None:
-                embs_df = pd.DataFrame(embs).T
-                embs_df.index = [self.token_gene_dict[token] for token in embs_df.index]
-        elif self.emb_mode == "cls":
-            embs_df = label_cell_embs(embs, downsampled_data, self.emb_label)
-
-        # save embeddings to output_path
-        if cell_state is None:
-            output_path = (Path(output_directory) / output_prefix).with_suffix(".csv")
-            embs_df.to_csv(output_path)
-
-        if self.exact_summary_stat == "exact_mean":
-            embs = embs.mean(dim=0)
-            emb_dims = pu.get_model_emb_dims(model)
-            embs_df = pd.DataFrame(
-                embs_df[0 : emb_dims - 1].mean(axis="rows"),
-                columns=[self.exact_summary_stat],
-            ).T
-        elif self.exact_summary_stat == "exact_median":
-            embs = torch.median(embs, dim=0)[0]
-            emb_dims = pu.get_model_emb_dims(model)
-            embs_df = pd.DataFrame(
-                embs_df[0 : emb_dims - 1].median(axis="rows"),
-                columns=[self.exact_summary_stat],
-            ).T
-
-        if cell_state is not None:
-            return embs
-        else:
-            if output_torch_embs:
-                return embs_df, embs
-            else:
-                return embs_df
-
-    def get_state_embs(
-        self,
-        cell_states_to_model,
-        model_directory,
-        input_data_file,
-        output_directory,
-        output_prefix,
-        output_torch_embs=True,
-    ):
-        """
-        Extract exact mean or exact median cell state embedding positions from input data and save as results in output_directory.
-
-        **Parameters:**
-
-        cell_states_to_model : None, dict
-            | Cell states to model if testing perturbations that achieve goal state change.
-            | Four-item dictionary with keys: state_key, start_state, goal_state, and alt_states
-            | state_key: key specifying name of column in .dataset that defines the start/goal states
-            | start_state: value in the state_key column that specifies the start state
-            | goal_state: value in the state_key column taht specifies the goal end state
-            | alt_states: list of values in the state_key column that specify the alternate end states
-            | For example:
-            |      {"state_key": "disease",
-            |      "start_state": "dcm",
-            |      "goal_state": "nf",
-            |      "alt_states": ["hcm", "other1", "other2"]}
-        model_directory : Path
-            | Path to directory containing model
-        input_data_file : Path
-            | Path to directory containing .dataset inputs
-        output_directory : Path
-            | Path to directory where embedding data will be saved as csv
-        output_prefix : str
-            | Prefix for output file
-        output_torch_embs : bool
-            | Whether or not to also output the embeddings as a tensor.
-            | Note, if true, will output embeddings as both dataframe and tensor.
-
-        **Outputs**
-
-        | Outputs state_embs_dict for use with in silico perturber.
-        | Format is dictionary of embedding positions of each cell state to model shifts from/towards.
-        | Keys specify each possible cell state to model.
-        | Values are target embedding positions as torch.tensor.
-        | For example:
-        |      {"nf": emb_nf,
-        |      "hcm": emb_hcm,
-        |      "dcm": emb_dcm,
-        |      "other1": emb_other1,
-        |      "other2": emb_other2}
-        """
-
-        pu.validate_cell_states_to_model(cell_states_to_model)
-        valid_summary_stats = ["exact_mean", "exact_median"]
-        if self.exact_summary_stat not in valid_summary_stats:
-            logger.error(
-                "For extracting state embs, summary_stat in EmbExtractor "
-                f"must be set to option in {valid_summary_stats}"
-            )
-            raise
-
-        if self.emb_label is not None:
-            logger.error(
-                "For extracting state embs, emb_label should be None since labels are based on state embs dict keys."
-            )
-            raise
-        
-        state_embs_dict = dict()
-        state_key = cell_states_to_model["state_key"]
-        for k, v in cell_states_to_model.items():
-            if k == "state_key":
-                continue
-            elif (k == "start_state") or (k == "goal_state"):
-                state_embs_dict[v] = self.extract_embs(
-                    model_directory,
-                    input_data_file,
-                    output_directory,
-                    output_prefix,
-                    output_torch_embs,
-                    cell_state={state_key: v},
-                )
-            else:  # k == "alt_states"
-                for alt_state in v:
-                    state_embs_dict[alt_state] = self.extract_embs(
-                        model_directory,
-                        input_data_file,
-                        output_directory,
-                        output_prefix,
-                        output_torch_embs,
-                        cell_state={state_key: alt_state},
-                    )
-
-        output_path = (Path(output_directory) / output_prefix).with_suffix(".pkl")
-        with open(output_path, "wb") as fp:
-            pickle.dump(state_embs_dict, fp)
-
-        return state_embs_dict
-
-    def plot_embs(
-        self,
-        embs,
-        plot_style,
-        output_directory,
-        output_prefix,
-        max_ncells_to_plot=1000,
-        kwargs_dict=None,
-    ):
-        """
-        Plot embeddings, coloring by provided labels.
-
-        **Parameters:**
-
-        embs : pandas.core.frame.DataFrame
-            | Pandas dataframe containing embeddings output from extract_embs
-        plot_style : str
-            | Style of plot: "heatmap" or "umap"
-        output_directory : Path
-            | Path to directory where plots will be saved as pdf
-        output_prefix : str
-            | Prefix for output file
-        max_ncells_to_plot : None, int
-            | Maximum number of cells to plot.
-            | Default is 1000 cells randomly sampled from embeddings.
-            | If None, will plot embeddings from all cells.
-        kwargs_dict : dict
-            | Dictionary of kwargs to pass to plotting function.
-
-        **Examples:**
-
-        .. code-block :: python
-
-            >>> embex.plot_embs(embs=embs,
-            ...                 plot_style="heatmap",
-            ...                 output_directory="path/to/output_directory",
-            ...                 output_prefix="output_prefix")
-
-        """
-
-        if plot_style not in ["heatmap", "umap"]:
-            logger.error(
-                "Invalid option for 'plot_style'. " "Valid options: {'heatmap','umap'}"
-            )
-            raise
-
-        if (plot_style == "umap") and (self.labels_to_plot is None):
-            logger.error("Plotting UMAP requires 'labels_to_plot'. ")
-            raise
-
-        if max_ncells_to_plot is not None:
-            if max_ncells_to_plot > self.max_ncells:
-                max_ncells_to_plot = self.max_ncells
-                logger.warning(
-                    "max_ncells_to_plot must be <= max_ncells. "
-                    f"Changing max_ncells_to_plot to {self.max_ncells}."
-                )
-            elif max_ncells_to_plot < self.max_ncells:
-                embs = embs.sample(max_ncells_to_plot, axis=0)
-
-        if self.emb_label is None:
-            label_len = 0
-        else:
-            label_len = len(self.emb_label)
-
-        emb_dims = embs.shape[1] - label_len
-
-        if self.emb_label is None:
-            emb_labels = None
-        else:
-            emb_labels = embs.columns[emb_dims:]
-
-        if plot_style == "umap":
-            for label in self.labels_to_plot:
-                if label not in emb_labels:
-                    logger.warning(
-                        f"Label {label} from labels_to_plot "
-                        f"not present in provided embeddings dataframe."
-                    )
-                    continue
-                output_prefix_label = output_prefix + f"_umap_{label}"
-                output_file = (
-                    Path(output_directory) / output_prefix_label
-                ).with_suffix(".pdf")
-                plot_umap(embs, emb_dims, label, output_file, kwargs_dict)
-
-        if plot_style == "heatmap":
-            for label in self.labels_to_plot:
-                if label not in emb_labels:
-                    logger.warning(
-                        f"Label {label} from labels_to_plot "
-                        f"not present in provided embeddings dataframe."
-                    )
-                    continue
-                output_prefix_label = output_prefix + f"_heatmap_{label}"
-                output_file = (
-                    Path(output_directory) / output_prefix_label
-                ).with_suffix(".pdf")
-                plot_heatmap(embs, emb_dims, label, output_file, kwargs_dict)

geneformer/evaluation_utils.py

@@ -1,287 +0,0 @@
-import logging
-import math
-import pickle
-from pathlib import Path
-
-import matplotlib.pyplot as plt
-import numpy as np
-import pandas as pd
-import seaborn as sns
-import torch
-from datasets.utils.logging import disable_progress_bar, enable_progress_bar
-from sklearn import preprocessing
-from sklearn.metrics import (
-    ConfusionMatrixDisplay,
-    accuracy_score,
-    auc,
-    confusion_matrix,
-    f1_score,
-    roc_curve,
-)
-from tqdm.auto import trange
-
-from . import TOKEN_DICTIONARY_FILE
-from .emb_extractor import make_colorbar
-
-logger = logging.getLogger(__name__)
-
-
-def preprocess_classifier_batch(cell_batch, max_len, label_name):
-    if max_len is None:
-        max_len = max([len(i) for i in cell_batch["input_ids"]])
-
-    # load token dictionary (Ensembl IDs:token)
-    with open(TOKEN_DICTIONARY_FILE, "rb") as f:
-        gene_token_dict = pickle.load(f)
-
-    def pad_label_example(example):
-        example[label_name] = np.pad(
-            example[label_name],
-            (0, max_len - len(example["input_ids"])),
-            mode="constant",
-            constant_values=-100,
-        )
-        example["input_ids"] = np.pad(
-            example["input_ids"],
-            (0, max_len - len(example["input_ids"])),
-            mode="constant",
-            constant_values=gene_token_dict.get("<pad>"),
-        )
-        example["attention_mask"] = (
-            example["input_ids"] != gene_token_dict.get("<pad>")
-        ).astype(int)
-        return example
-
-    padded_batch = cell_batch.map(pad_label_example)
-    return padded_batch
-
-
-# Function to find the largest number smaller
-# than or equal to N that is divisible by k
-def find_largest_div(N, K):
-    rem = N % K
-    if rem == 0:
-        return N
-    else:
-        return N - rem
-
-
-def vote(logit_list):
-    m = max(logit_list)
-    logit_list.index(m)
-    indices = [i for i, x in enumerate(logit_list) if x == m]
-    if len(indices) > 1:
-        return "tie"
-    else:
-        return indices[0]
-
-
-def py_softmax(vector):
-    e = np.exp(vector)
-    return e / e.sum()
-
-
-def classifier_predict(model, classifier_type, evalset, forward_batch_size):
-    if classifier_type == "gene":
-        label_name = "labels"
-    elif classifier_type == "cell":
-        label_name = "label"
-
-    predict_logits = []
-    predict_labels = []
-    model.eval()
-
-    # ensure there is at least 2 examples in each batch to avoid incorrect tensor dims
-    evalset_len = len(evalset)
-    max_divisible = find_largest_div(evalset_len, forward_batch_size)
-    if len(evalset) - max_divisible == 1:
-        evalset_len = max_divisible
-
-    max_evalset_len = max(evalset.select([i for i in range(evalset_len)])["length"])
-
-    disable_progress_bar()  # disable progress bar for preprocess_classifier_batch mapping
-    for i in trange(0, evalset_len, forward_batch_size):
-        max_range = min(i + forward_batch_size, evalset_len)
-        batch_evalset = evalset.select([i for i in range(i, max_range)])
-        padded_batch = preprocess_classifier_batch(
-            batch_evalset, max_evalset_len, label_name
-        )
-        padded_batch.set_format(type="torch")
-
-        input_data_batch = padded_batch["input_ids"]
-        attn_msk_batch = padded_batch["attention_mask"]
-        label_batch = padded_batch[label_name]
-        with torch.no_grad():
-            outputs = model(
-                input_ids=input_data_batch.to("cuda"),
-                attention_mask=attn_msk_batch.to("cuda"),
-                labels=label_batch.to("cuda"),
-            )
-            predict_logits += [torch.squeeze(outputs.logits.to("cpu"))]
-            predict_labels += [torch.squeeze(label_batch.to("cpu"))]
-
-    enable_progress_bar()
-    logits_by_cell = torch.cat(predict_logits)
-    last_dim = len(logits_by_cell.shape) - 1
-    all_logits = logits_by_cell.reshape(-1, logits_by_cell.shape[last_dim])
-    labels_by_cell = torch.cat(predict_labels)
-    all_labels = torch.flatten(labels_by_cell)
-    logit_label_paired = [
-        item
-        for item in list(zip(all_logits.tolist(), all_labels.tolist()))
-        if item[1] != -100
-    ]
-    y_pred = [vote(item[0]) for item in logit_label_paired]
-    y_true = [item[1] for item in logit_label_paired]
-    logits_list = [item[0] for item in logit_label_paired]
-    return y_pred, y_true, logits_list
-
-
-def get_metrics(y_pred, y_true, logits_list, num_classes, labels):
-    conf_mat = confusion_matrix(y_true, y_pred, labels=list(labels))
-    macro_f1 = f1_score(y_true, y_pred, average="macro")
-    acc = accuracy_score(y_true, y_pred)
-    roc_metrics = None  # roc metrics not reported for multiclass
-    if num_classes == 2:
-        y_score = [py_softmax(item)[1] for item in logits_list]
-        fpr, tpr, _ = roc_curve(y_true, y_score)
-        mean_fpr = np.linspace(0, 1, 100)
-        interp_tpr = np.interp(mean_fpr, fpr, tpr)
-        interp_tpr[0] = 0.0
-        tpr_wt = len(tpr)
-        roc_auc = auc(fpr, tpr)
-        roc_metrics = {
-            "fpr": fpr,
-            "tpr": tpr,
-            "interp_tpr": interp_tpr,
-            "auc": roc_auc,
-            "tpr_wt": tpr_wt,
-        }
-    return conf_mat, macro_f1, acc, roc_metrics
-
-
-# get cross-validated mean and sd metrics
-def get_cross_valid_roc_metrics(all_tpr, all_roc_auc, all_tpr_wt):
-    wts = [count / sum(all_tpr_wt) for count in all_tpr_wt]
-    all_weighted_tpr = [a * b for a, b in zip(all_tpr, wts)]
-    mean_tpr = np.sum(all_weighted_tpr, axis=0)
-    mean_tpr[-1] = 1.0
-    all_weighted_roc_auc = [a * b for a, b in zip(all_roc_auc, wts)]
-    roc_auc = np.sum(all_weighted_roc_auc)
-    roc_auc_sd = math.sqrt(np.average((all_roc_auc - roc_auc) ** 2, weights=wts))
-    return mean_tpr, roc_auc, roc_auc_sd
-
-
-# plot ROC curve
-def plot_ROC(roc_metric_dict, model_style_dict, title, output_dir, output_prefix):
-    fig = plt.figure()
-    fig.set_size_inches(10, 8)
-    sns.set(font_scale=2)
-    sns.set_style("white")
-    lw = 3
-    for model_name in roc_metric_dict.keys():
-        mean_fpr = roc_metric_dict[model_name]["mean_fpr"]
-        mean_tpr = roc_metric_dict[model_name]["mean_tpr"]
-        roc_auc = roc_metric_dict[model_name]["roc_auc"]
-        roc_auc_sd = roc_metric_dict[model_name]["roc_auc_sd"]
-        color = model_style_dict[model_name]["color"]
-        linestyle = model_style_dict[model_name]["linestyle"]
-        if len(roc_metric_dict[model_name]["all_roc_auc"]) > 1:
-            label = f"{model_name} (AUC {roc_auc:0.2f} $\pm$ {roc_auc_sd:0.2f})"
-        else:
-            label = f"{model_name} (AUC {roc_auc:0.2f})"
-        plt.plot(
-            mean_fpr, mean_tpr, color=color, linestyle=linestyle, lw=lw, label=label
-        )
-
-    plt.plot([0, 1], [0, 1], color="black", lw=lw, linestyle="--")
-    plt.xlim([0.0, 1.0])
-    plt.ylim([0.0, 1.05])
-    plt.xlabel("False Positive Rate")
-    plt.ylabel("True Positive Rate")
-    plt.title(title)
-    plt.legend(loc="lower right")
-
-    output_file = (Path(output_dir) / f"{output_prefix}_roc").with_suffix(".pdf")
-    plt.savefig(output_file, bbox_inches="tight")
-    plt.show()
-
-
-# plot confusion matrix
-def plot_confusion_matrix(
-    conf_mat_df, title, output_dir, output_prefix, custom_class_order
-):
-    fig = plt.figure()
-    fig.set_size_inches(10, 10)
-    sns.set(font_scale=1)
-    sns.set_style("whitegrid", {"axes.grid": False})
-    if custom_class_order is not None:
-        conf_mat_df = conf_mat_df.reindex(
-            index=custom_class_order, columns=custom_class_order
-        )
-    display_labels = generate_display_labels(conf_mat_df)
-    conf_mat = preprocessing.normalize(conf_mat_df.to_numpy(), norm="l1")
-    display = ConfusionMatrixDisplay(
-        confusion_matrix=conf_mat, display_labels=display_labels
-    )
-    display.plot(cmap="Blues", values_format=".2g")
-    plt.title(title)
-    plt.show()
-
-    output_file = (Path(output_dir) / f"{output_prefix}_conf_mat").with_suffix(".pdf")
-    display.figure_.savefig(output_file, bbox_inches="tight")
-
-
-def generate_display_labels(conf_mat_df):
-    display_labels = []
-    i = 0
-    for label in conf_mat_df.index:
-        display_labels += [f"{label}\nn={conf_mat_df.iloc[i,:].sum():.0f}"]
-        i = i + 1
-    return display_labels
-
-
-def plot_predictions(predictions_df, title, output_dir, output_prefix, kwargs_dict):
-    sns.set(font_scale=2)
-    plt.figure(figsize=(10, 10), dpi=150)
-    label_colors, label_color_dict = make_colorbar(predictions_df, "true")
-    predictions_df = predictions_df.drop(columns=["true"])
-    predict_colors_list = [label_color_dict[label] for label in predictions_df.columns]
-    predict_label_list = [label for label in predictions_df.columns]
-    predict_colors = pd.DataFrame(
-        pd.Series(predict_colors_list, index=predict_label_list), columns=["predicted"]
-    )
-
-    default_kwargs_dict = {
-        "row_cluster": False,
-        "col_cluster": False,
-        "row_colors": label_colors,
-        "col_colors": predict_colors,
-        "linewidths": 0,
-        "xticklabels": False,
-        "yticklabels": False,
-        "center": 0,
-        "cmap": "vlag",
-    }
-
-    if kwargs_dict is not None:
-        default_kwargs_dict.update(kwargs_dict)
-    g = sns.clustermap(predictions_df, **default_kwargs_dict)
-
-    plt.setp(g.ax_row_colors.get_xmajorticklabels(), rotation=45, ha="right")
-
-    for label_color in list(label_color_dict.keys()):
-        g.ax_col_dendrogram.bar(
-            0, 0, color=label_color_dict[label_color], label=label_color, linewidth=0
-        )
-
-        g.ax_col_dendrogram.legend(
-            title=f"{title}",
-            loc="lower center",
-            ncol=4,
-            bbox_to_anchor=(0.5, 1),
-            facecolor="white",
-        )
-
-    output_file = (Path(output_dir) / f"{output_prefix}_pred").with_suffix(".pdf")
-    plt.savefig(output_file, bbox_inches="tight")

geneformer/in_silico_perturber.py

@@ -1,1579 +0,0 @@
-"""
-Geneformer in silico perturber.
-
-**Usage:**
-
-.. code-block :: python
-
-    >>> from geneformer import InSilicoPerturber
-    >>> isp = InSilicoPerturber(perturb_type="delete",
-    ...                         perturb_rank_shift=None,
-    ...                         genes_to_perturb="all",
-    ...                         model_type="CellClassifier",
-    ...                         num_classes=0,
-    ...                         emb_mode="cell",
-    ...                         filter_data={"cell_type":["cardiomyocyte"]},
-    ...                         cell_states_to_model={"state_key": "disease", "start_state": "dcm", "goal_state": "nf", "alt_states": ["hcm", "other1", "other2"]},
-    ...                         state_embs_dict ={"nf": emb_nf, "hcm": emb_hcm, "dcm": emb_dcm, "other1": emb_other1, "other2": emb_other2},
-    ...                         max_ncells=None,
-    ...                         emb_layer=0,
-    ...                         forward_batch_size=100,
-    ...                         nproc=16)
-    >>> isp.perturb_data("path/to/model",
-    ...                  "path/to/input_data",
-    ...                  "path/to/output_directory",
-    ...                  "output_prefix")
-
-**Description:**
-
-| Performs in silico perturbation (e.g. deletion or overexpression) of defined set of genes or all genes in sample of cells.
-| Outputs impact of perturbation on cell or gene embeddings.
-| Output files are analyzed with ``in_silico_perturber_stats``.
-
-"""
-
-import logging
-
-# imports
-import os
-import pickle
-from collections import defaultdict
-
-import torch
-from datasets import Dataset
-from multiprocess import set_start_method
-from tqdm.auto import trange
-
-from . import TOKEN_DICTIONARY_FILE
-from . import perturber_utils as pu
-from .emb_extractor import get_embs
-
-import datasets
-datasets.logging.disable_progress_bar()
-
-
-logger = logging.getLogger(__name__)
-
-
-class InSilicoPerturber:
-    valid_option_dict = {
-        "perturb_type": {"delete", "overexpress", "inhibit", "activate"},
-        "perturb_rank_shift": {None, 1, 2, 3},
-        "genes_to_perturb": {"all", list},
-        "combos": {0, 1},
-        "anchor_gene": {None, str},
-        "model_type": {"Pretrained", "GeneClassifier", "CellClassifier", "MTLCellClassifier", "MTLCellClassifier-Quantized"},
-        "num_classes": {int},
-        "emb_mode": {"cls", "cell", "cls_and_gene", "cell_and_gene"},
-        "cell_emb_style": {"mean_pool"},
-        "filter_data": {None, dict},
-        "cell_states_to_model": {None, dict},
-        "state_embs_dict": {None, dict},
-        "max_ncells": {None, int},
-        "cell_inds_to_perturb": {"all", dict},
-        "emb_layer": {-1, 0},
-        "token_dictionary_file": {None, str},
-        "forward_batch_size": {int},
-        "nproc": {int},
-    }
-
-    def __init__(
-        self,
-        perturb_type="delete",
-        perturb_rank_shift=None,
-        genes_to_perturb="all",
-        combos=0,
-        anchor_gene=None,
-        model_type="Pretrained",
-        num_classes=0,
-        emb_mode="cls",
-        cell_emb_style="mean_pool",
-        filter_data=None,
-        cell_states_to_model=None,
-        state_embs_dict=None,
-        max_ncells=None,
-        cell_inds_to_perturb="all",
-        emb_layer=-1,
-        forward_batch_size=100,
-        nproc=4,
-        token_dictionary_file=None,
-        clear_mem_ncells=1000,
-    ):
-        """
-        Initialize in silico perturber.
-
-        **Parameters:**
-
-        perturb_type : {"delete", "overexpress", "inhibit", "activate"}
-            | Type of perturbation.
-            | "delete": delete gene from rank value encoding
-            | "overexpress": move gene to front of rank value encoding
-            | *(TBA)* "inhibit": move gene to lower quartile of rank value encoding
-            | *(TBA)* "activate": move gene to higher quartile of rank value encoding
-        *(TBA)* perturb_rank_shift : None, {1,2,3}
-            | Number of quartiles by which to shift rank of gene.
-            | For example, if perturb_type="activate" and perturb_rank_shift=1:
-            |     genes in 4th quartile will move to middle of 3rd quartile.
-            |     genes in 3rd quartile will move to middle of 2nd quartile.
-            |     genes in 2nd quartile will move to middle of 1st quartile.
-            |     genes in 1st quartile will move to front of rank value encoding.
-            | For example, if perturb_type="inhibit" and perturb_rank_shift=2:
-            |     genes in 1st quartile will move to middle of 3rd quartile.
-            |     genes in 2nd quartile will move to middle of 4th quartile.
-            |     genes in 3rd or 4th quartile will move to bottom of rank value encoding.
-        genes_to_perturb : "all", list
-            | Default is perturbing each gene detected in each cell in the dataset.
-            | Otherwise, may provide a list of ENSEMBL IDs of genes to perturb.
-            | If gene list is provided, then perturber will only test perturbing them all together
-            | (rather than testing each possible combination of the provided genes).
-        combos : {0,1}
-            | Whether to perturb genes individually (0) or in pairs (1).
-        anchor_gene : None, str
-            | ENSEMBL ID of gene to use as anchor in combination perturbations.
-            | For example, if combos=1 and anchor_gene="ENSG00000148400":
-            |     anchor gene will be perturbed in combination with each other gene.
-        model_type : {"Pretrained", "GeneClassifier", "CellClassifier", "MTLCellClassifier", "MTLCellClassifier-Quantized"}
-            | Whether model is the pretrained Geneformer or a fine-tuned gene, cell, or multitask cell classifier (+/- 8bit quantization).
-        num_classes : int
-            | If model is a gene or cell classifier, specify number of classes it was trained to classify.
-            | For the pretrained Geneformer model, number of classes is 0 as it is not a classifier.
-        emb_mode : {"cls", "cell", "cls_and_gene","cell_and_gene"}
-            | Whether to output impact of perturbation on CLS token, cell, and/or gene embeddings.
-            | Gene embedding shifts only available as compared to original cell, not comparing to goal state.
-        cell_emb_style : "mean_pool"
-            | Method for summarizing cell embeddings if not using CLS token.
-            | Currently only option is mean pooling of gene embeddings for given cell.
-        filter_data : None, dict
-            | Default is to use all input data for in silico perturbation study.
-            | Otherwise, dictionary specifying .dataset column name and list of values to filter by.
-        cell_states_to_model : None, dict
-            | Cell states to model if testing perturbations that achieve goal state change.
-            | Four-item dictionary with keys: state_key, start_state, goal_state, and alt_states
-            | state_key: key specifying name of column in .dataset that defines the start/goal states
-            | start_state: value in the state_key column that specifies the start state
-            | goal_state: value in the state_key column taht specifies the goal end state
-            | alt_states: list of values in the state_key column that specify the alternate end states
-            | For example: {"state_key": "disease",
-            |               "start_state": "dcm",
-            |               "goal_state": "nf",
-            |               "alt_states": ["hcm", "other1", "other2"]}
-        state_embs_dict : None, dict
-            | Embedding positions of each cell state to model shifts from/towards (e.g. mean or median).
-            | Dictionary with keys specifying each possible cell state to model.
-            | Values are target embedding positions as torch.tensor.
-            | For example: {"nf": emb_nf,
-            |               "hcm": emb_hcm,
-            |               "dcm": emb_dcm,
-            |               "other1": emb_other1,
-            |               "other2": emb_other2}
-        max_ncells : None, int
-            | Maximum number of cells to test.
-            | If None, will test all cells.
-        cell_inds_to_perturb : "all", list
-            | Default is perturbing each cell in the dataset.
-            | Otherwise, may provide a dict of indices of cells to perturb with keys start_ind and end_ind.
-            | start_ind: the first index to perturb.
-            | end_ind: the last index to perturb (exclusive).
-            | Indices will be selected *after* the filter_data criteria and sorting.
-            | Useful for splitting extremely large datasets across separate GPUs.
-        emb_layer : {-1, 0}
-            | Embedding layer to use for quantification.
-            | 0: last layer (recommended for questions closely tied to model's training objective)
-            | -1: 2nd to last layer (recommended for questions requiring more general representations)
-        forward_batch_size : int
-            | Batch size for forward pass.
-        nproc : int
-            | Number of CPU processes to use.
-        token_dictionary_file : Path
-            | Path to pickle file containing token dictionary (Ensembl ID:token).
-        clear_mem_ncells : int
-            | Clear memory every n cells.
-        """
-        try:
-            set_start_method("spawn")
-        except RuntimeError:
-            pass
-
-        self.perturb_type = perturb_type
-        self.perturb_rank_shift = perturb_rank_shift
-        self.genes_to_perturb = genes_to_perturb
-        self.combos = combos
-        self.anchor_gene = anchor_gene
-        if self.genes_to_perturb == "all":
-            self.perturb_group = False
-        else:
-            self.perturb_group = True
-            if (self.anchor_gene is not None) or (self.combos != 0):
-                self.anchor_gene = None
-                self.combos = 0
-                logger.warning(
-                    "anchor_gene set to None and combos set to 0. "
-                    "If providing list of genes to perturb, "
-                    "list of genes_to_perturb will be perturbed together, "
-                    "without anchor gene or combinations."
-                )
-        self.model_type = model_type
-        self.num_classes = num_classes
-        self.emb_mode = emb_mode
-        self.cell_emb_style = cell_emb_style
-        self.filter_data = filter_data
-        self.cell_states_to_model = cell_states_to_model
-        self.state_embs_dict = state_embs_dict
-        self.max_ncells = max_ncells
-        self.cell_inds_to_perturb = cell_inds_to_perturb
-        self.emb_layer = emb_layer
-        self.forward_batch_size = forward_batch_size
-        self.nproc = nproc
-        self.token_dictionary_file = token_dictionary_file
-        self.clear_mem_ncells = clear_mem_ncells
-
-        self.validate_options()
-
-        # load token dictionary (Ensembl IDs:token)
-        if self.token_dictionary_file is None:
-            token_dictionary_file = TOKEN_DICTIONARY_FILE
-        with open(token_dictionary_file, "rb") as f:
-            self.gene_token_dict = pickle.load(f)
-        self.token_gene_dict = {v: k for k, v in self.gene_token_dict.items()}
-
-        self.pad_token_id = self.gene_token_dict.get("<pad>")
-        self.cls_token_id = self.gene_token_dict.get("<cls>")
-        self.eos_token_id = self.gene_token_dict.get("<eos>")
-
-        # Identify if special token is present in the token dictionary
-        if (self.cls_token_id is not None) and (self.eos_token_id is not None):
-            self.special_token = True
-        else:
-            if "cls" in self.emb_mode:
-                logger.error(
-                    f"emb_mode set to {self.emb_mode} but <cls> or <eos> token not in token dictionary."
-                )
-                raise
-            self.special_token = False
-
-        if self.anchor_gene is None:
-            self.anchor_token = None
-        else:
-            try:
-                self.anchor_token = [self.gene_token_dict[self.anchor_gene]]
-            except KeyError:
-                logger.error(f"Anchor gene {self.anchor_gene} not in token dictionary.")
-                raise
-
-        if self.genes_to_perturb == "all":
-            self.tokens_to_perturb = "all"
-        else:
-            missing_genes = [
-                gene
-                for gene in self.genes_to_perturb
-                if gene not in self.gene_token_dict.keys()
-            ]
-            if len(missing_genes) == len(self.genes_to_perturb):
-                logger.error(
-                    "None of the provided genes to perturb are in token dictionary."
-                )
-                raise
-            elif len(missing_genes) > 0:
-                logger.warning(
-                    f"Genes to perturb {missing_genes} are not in token dictionary."
-                )
-            self.tokens_to_perturb = [
-                self.gene_token_dict.get(gene) for gene in self.genes_to_perturb
-            ]
-
-    def validate_options(self):
-        # first disallow options under development
-        if self.perturb_type in ["inhibit", "activate"]:
-            logger.error(
-                "In silico inhibition and activation currently under development. "
-                "Current valid options for 'perturb_type': 'delete' or 'overexpress'"
-            )
-            raise
-        if (self.combos > 0) and (self.anchor_gene is None):
-            logger.error(
-                "Combination perturbation without anchor gene is currently under development. "
-                "Currently, must provide anchor gene for combination perturbation."
-            )
-            raise
-
-        # confirm arguments are within valid options and compatible with each other
-        for attr_name, valid_options in self.valid_option_dict.items():
-            attr_value = self.__dict__[attr_name]
-            if type(attr_value) not in {list, dict}:
-                if attr_value in valid_options:
-                    continue
-                if attr_name in ["anchor_gene"]:
-                    if type(attr_name) in {str}:
-                        continue
-            valid_type = False
-            for option in valid_options:
-                if (option in [bool, int, list, dict, str]) and isinstance(
-                    attr_value, option
-                ):
-                    valid_type = True
-                    break
-            if valid_type:
-                continue
-            logger.error(
-                f"Invalid option for {attr_name}. "
-                f"Valid options for {attr_name}: {valid_options}"
-            )
-            raise
-
-        if self.perturb_type in ["delete", "overexpress"]:
-            if self.perturb_rank_shift is not None:
-                if self.perturb_type == "delete":
-                    logger.warning(
-                        "perturb_rank_shift set to None. "
-                        "If perturb type is delete then gene is deleted entirely "
-                        "rather than shifted by quartile"
-                    )
-                elif self.perturb_type == "overexpress":
-                    logger.warning(
-                        "perturb_rank_shift set to None. "
-                        "If perturb type is overexpress then gene is moved to front "
-                        "of rank value encoding rather than shifted by quartile"
-                    )
-            self.perturb_rank_shift = None
-
-        if (self.anchor_gene is not None) and (self.emb_mode == "cell_and_gene"):
-            self.emb_mode = "cell"
-            logger.warning(
-                "emb_mode set to 'cell'. "
-                "Currently, analysis with anchor gene "
-                "only outputs effect on cell embeddings."
-            )
-
-        if self.cell_states_to_model is not None:
-            pu.validate_cell_states_to_model(self.cell_states_to_model)
-
-            if self.anchor_gene is not None:
-                self.anchor_gene = None
-                logger.warning(
-                    "anchor_gene set to None. "
-                    "Currently, anchor gene not available "
-                    "when modeling multiple cell states."
-                )
-
-            if self.state_embs_dict is None:
-                logger.error(
-                    "state_embs_dict must be provided for mode with cell_states_to_model. "
-                    "Format is dictionary with keys specifying each possible cell state to model. "
-                    "Values are target embedding positions as torch.tensor."
-                )
-                raise
-
-            for state_emb in self.state_embs_dict.values():
-                if not torch.is_tensor(state_emb):
-                    logger.error(
-                        "state_embs_dict must be dictionary with values being torch.tensor."
-                    )
-                    raise
-
-            keys_absent = []
-            for k, v in self.cell_states_to_model.items():
-                if (k == "start_state") or (k == "goal_state"):
-                    if v not in self.state_embs_dict.keys():
-                        keys_absent.append(v)
-                if k == "alt_states":
-                    for state in v:
-                        if state not in self.state_embs_dict.keys():
-                            keys_absent.append(state)
-            if len(keys_absent) > 0:
-                logger.error(
-                    "Each start_state, goal_state, and alt_states in cell_states_to_model "
-                    "must be a key in state_embs_dict with the value being "
-                    "the state's embedding position as torch.tensor. "
-                    f"Missing keys: {keys_absent}"
-                )
-                raise
-
-        if self.perturb_type in ["inhibit", "activate"]:
-            if self.perturb_rank_shift is None:
-                logger.error(
-                    "If perturb_type is inhibit or activate then "
-                    "quartile to shift by must be specified."
-                )
-                raise
-
-        if self.filter_data is not None:
-            for key, value in self.filter_data.items():
-                if not isinstance(value, list):
-                    self.filter_data[key] = [value]
-                    logger.warning(
-                        "Values in filter_data dict must be lists. "
-                        f"Changing {key} value to list ([{value}])."
-                    )
-
-        if self.cell_inds_to_perturb != "all":
-            if set(self.cell_inds_to_perturb.keys()) != {"start", "end"}:
-                logger.error(
-                    "If cell_inds_to_perturb is a dictionary, keys must be 'start' and 'end'."
-                )
-                raise
-            if (
-                self.cell_inds_to_perturb["start"] < 0
-                or self.cell_inds_to_perturb["end"] < 0
-            ):
-                logger.error("cell_inds_to_perturb must be positive.")
-                raise
-
-    def perturb_data(
-        self, model_directory, input_data_file, output_directory, output_prefix
-    ):
-        """
-        Perturb genes in input data and save as results in output_directory.
-
-        **Parameters:**
-
-        model_directory : Path
-            | Path to directory containing model
-        input_data_file : Path
-            | Path to directory containing .dataset inputs
-        output_directory : Path
-            | Path to directory where perturbation data will be saved as batched pickle files
-        output_prefix : str
-            | Prefix for output files
-        """
-
-        ### format output path ###
-        output_path_prefix = os.path.join(
-            output_directory, f"in_silico_{self.perturb_type}_{output_prefix}"
-        )
-
-        ### load model and define parameters ###
-        model = pu.load_model(
-            self.model_type, self.num_classes, model_directory, mode="eval"
-        )
-        self.max_len = pu.get_model_input_size(model)
-        layer_to_quant = pu.quant_layers(model) + self.emb_layer
-
-        ### filter input data ###
-        # general filtering of input data based on filter_data argument
-        filtered_input_data = pu.load_and_filter(
-            self.filter_data, self.nproc, input_data_file
-        )
-
-        # Ensure emb_mode is cls if first token of the filtered input data is cls token
-        if self.special_token:
-            if (filtered_input_data["input_ids"][0][0] == self.cls_token_id) and (
-                "cls" not in self.emb_mode
-            ):
-                logger.error(
-                    "Emb mode 'cls' or 'cls_and_gene' required when first token is <cls>."
-                )
-                raise
-            if "cls" in self.emb_mode:
-                if (filtered_input_data["input_ids"][0][0] != self.cls_token_id) or (
-                    filtered_input_data["input_ids"][0][-1] != self.eos_token_id
-                ):
-                    logger.error(
-                        "Emb mode 'cls' and 'cls_and_gene' require that first token is <cls> and last token is <eos>."
-                    )
-                    raise
-
-        filtered_input_data = self.apply_additional_filters(filtered_input_data)
-
-        if self.perturb_group is True:
-            if (self.special_token) and ("cls" in self.emb_mode):
-                self.isp_perturb_set_special(
-                    model, filtered_input_data, layer_to_quant, output_path_prefix
-                )
-            else:
-                self.isp_perturb_set(
-                    model, filtered_input_data, layer_to_quant, output_path_prefix
-                )
-        else:
-            if (self.special_token) and ("cls" in self.emb_mode):
-                self.isp_perturb_all_special(
-                    model, filtered_input_data, layer_to_quant, output_path_prefix
-                )
-            else:
-                self.isp_perturb_all(
-                    model, filtered_input_data, layer_to_quant, output_path_prefix
-                )
-
-    def apply_additional_filters(self, filtered_input_data):
-        # additional filtering of input data dependent on isp mode
-        if self.cell_states_to_model is not None:
-            # filter for cells with start_state and log result
-            filtered_input_data = pu.filter_data_by_start_state(
-                filtered_input_data, self.cell_states_to_model, self.nproc
-            )
-
-        if (self.tokens_to_perturb != "all") and (self.perturb_type != "overexpress"):
-            # filter for cells with tokens_to_perturb and log result
-            filtered_input_data = pu.filter_data_by_tokens_and_log(
-                filtered_input_data,
-                self.tokens_to_perturb,
-                self.nproc,
-                "genes_to_perturb",
-            )
-
-        if self.anchor_token is not None:
-            # filter for cells with anchor gene and log result
-            filtered_input_data = pu.filter_data_by_tokens_and_log(
-                filtered_input_data, self.anchor_token, self.nproc, "anchor_gene"
-            )
-
-        # downsample and sort largest to smallest to encounter memory constraints earlier
-        filtered_input_data = pu.downsample_and_sort(
-            filtered_input_data, self.max_ncells
-        )
-
-        # slice dataset if cells_inds_to_perturb is not "all"
-        if self.cell_inds_to_perturb != "all":
-            filtered_input_data = pu.slice_by_inds_to_perturb(
-                filtered_input_data, self.cell_inds_to_perturb
-            )
-
-        return filtered_input_data
-
-    def isp_perturb_set(
-        self,
-        model,
-        filtered_input_data: Dataset,
-        layer_to_quant: int,
-        output_path_prefix: str,
-    ):
-        def make_group_perturbation_batch(example):
-            example_input_ids = example["input_ids"]
-            example["tokens_to_perturb"] = self.tokens_to_perturb
-            indices_to_perturb = [
-                example_input_ids.index(token) if token in example_input_ids else None
-                for token in self.tokens_to_perturb
-            ]
-            indices_to_perturb = [
-                item for item in indices_to_perturb if item is not None
-            ]
-            if len(indices_to_perturb) > 0:
-                example["perturb_index"] = indices_to_perturb
-            else:
-                # -100 indicates tokens to overexpress are not present in rank value encoding
-                example["perturb_index"] = [-100]
-            if self.perturb_type == "delete":
-                example = pu.delete_indices(example)
-            elif self.perturb_type == "overexpress":
-                example = pu.overexpress_tokens(
-                    example, self.max_len, self.special_token
-                )
-                example["n_overflow"] = pu.calc_n_overflow(
-                    self.max_len,
-                    example["length"],
-                    self.tokens_to_perturb,
-                    indices_to_perturb,
-                )
-            return example
-
-        total_batch_length = len(filtered_input_data)
-        if self.cell_states_to_model is None:
-            cos_sims_dict = defaultdict(list)
-        else:
-            cos_sims_dict = {
-                state: defaultdict(list)
-                for state in pu.get_possible_states(self.cell_states_to_model)
-            }
-
-        perturbed_data = filtered_input_data.map(
-            make_group_perturbation_batch, num_proc=self.nproc
-        )
-
-        if self.perturb_type == "overexpress":
-            filtered_input_data = filtered_input_data.add_column(
-                "n_overflow", perturbed_data["n_overflow"]
-            )
-            # remove overflow genes from original data so that embeddings are comparable
-            # i.e. if original cell has genes 0:2047 and you want to overexpress new gene 2048,
-            # then the perturbed cell will be 2048+0:2046 so we compare it to an original cell 0:2046.
-            # (otherwise we will be modeling the effect of both deleting 2047 and adding 2048,
-            # rather than only adding 2048)
-            filtered_input_data = filtered_input_data.map(
-                pu.truncate_by_n_overflow, num_proc=self.nproc
-            )
-
-        if self.emb_mode == "cell_and_gene":
-            stored_gene_embs_dict = defaultdict(list)
-
-        # iterate through batches
-        for i in trange(0, total_batch_length, self.forward_batch_size):
-            max_range = min(i + self.forward_batch_size, total_batch_length)
-            inds_select = [i for i in range(i, max_range)]
-
-            minibatch = filtered_input_data.select(inds_select)
-            perturbation_batch = perturbed_data.select(inds_select)
-
-            if self.cell_emb_style == "mean_pool":
-                full_original_emb = get_embs(
-                    model,
-                    minibatch,
-                    "gene",
-                    layer_to_quant,
-                    self.pad_token_id,
-                    self.forward_batch_size,
-                    token_gene_dict=self.token_gene_dict,
-                    summary_stat=None,
-                    silent=True,
-                )
-                indices_to_perturb = perturbation_batch["perturb_index"]
-                # remove indices that were perturbed
-                original_emb = pu.remove_perturbed_indices_set(
-                    full_original_emb,
-                    self.perturb_type,
-                    indices_to_perturb,
-                    self.tokens_to_perturb,
-                    minibatch["length"],
-                )
-                full_perturbation_emb = get_embs(
-                    model,
-                    perturbation_batch,
-                    "gene",
-                    layer_to_quant,
-                    self.pad_token_id,
-                    self.forward_batch_size,
-                    token_gene_dict=self.token_gene_dict,
-                    summary_stat=None,
-                    silent=True,
-                )
-
-                # remove overexpressed genes
-                if self.perturb_type == "overexpress":
-                    perturbation_emb = full_perturbation_emb[
-                        :, len(self.tokens_to_perturb) :, :
-                    ]
-
-                elif self.perturb_type == "delete":
-                    perturbation_emb = full_perturbation_emb[
-                        :, : max(perturbation_batch["length"]), :
-                    ]
-
-                n_perturbation_genes = perturbation_emb.size()[1]
-
-                # if no goal states, the cosine similarties are the mean of gene cosine similarities
-                if (
-                    self.cell_states_to_model is None
-                    or self.emb_mode == "cell_and_gene"
-                ):
-                    gene_cos_sims = pu.quant_cos_sims(
-                        perturbation_emb,
-                        original_emb,
-                        self.cell_states_to_model,
-                        self.state_embs_dict,
-                        emb_mode="gene",
-                    )
-
-                # if there are goal states, the cosine similarities are the cell cosine similarities
-                if self.cell_states_to_model is not None:
-                    original_cell_emb = pu.mean_nonpadding_embs(
-                        full_original_emb,
-                        torch.tensor(minibatch["length"], device="cuda"),
-                        dim=1,
-                    )
-                    perturbation_cell_emb = pu.mean_nonpadding_embs(
-                        full_perturbation_emb,
-                        torch.tensor(perturbation_batch["length"], device="cuda"),
-                        dim=1,
-                    )
-                    cell_cos_sims = pu.quant_cos_sims(
-                        perturbation_cell_emb,
-                        original_cell_emb,
-                        self.cell_states_to_model,
-                        self.state_embs_dict,
-                        emb_mode="cell",
-                    )
-
-                # get cosine similarities in gene embeddings
-                # if getting gene embeddings, need gene names
-                if self.emb_mode == "cell_and_gene":
-                    gene_list = minibatch["input_ids"]
-                    # need to truncate gene_list
-                    gene_list = [
-                        [g for g in genes if g not in self.tokens_to_perturb][
-                            :n_perturbation_genes
-                        ]
-                        for genes in gene_list
-                    ]
-
-                    for cell_i, genes in enumerate(gene_list):
-                        for gene_j, affected_gene in enumerate(genes):
-                            if len(self.genes_to_perturb) > 1:
-                                tokens_to_perturb = tuple(self.tokens_to_perturb)
-                            else:
-                                tokens_to_perturb = self.tokens_to_perturb[0]
-
-                            # fill in the gene cosine similarities
-                            try:
-                                stored_gene_embs_dict[
-                                    (tokens_to_perturb, affected_gene)
-                                ].append(gene_cos_sims[cell_i, gene_j].item())
-                            except KeyError:
-                                stored_gene_embs_dict[
-                                    (tokens_to_perturb, affected_gene)
-                                ] = gene_cos_sims[cell_i, gene_j].item()
-                else:
-                    gene_list = None
-
-            if self.cell_states_to_model is None:
-                # calculate the mean of the gene cosine similarities for cell shift
-                # tensor of nonpadding lengths for each cell
-                if self.perturb_type == "overexpress":
-                    # subtract number of genes that were overexpressed
-                    # since they are removed before getting cos sims
-                    n_overexpressed = len(self.tokens_to_perturb)
-                    nonpadding_lens = [
-                        x - n_overexpressed for x in perturbation_batch["length"]
-                    ]
-                else:
-                    nonpadding_lens = perturbation_batch["length"]
-                cos_sims_data = pu.mean_nonpadding_embs(
-                    gene_cos_sims, torch.tensor(nonpadding_lens, device="cuda")
-                )
-                cos_sims_dict = self.update_perturbation_dictionary(
-                    cos_sims_dict,
-                    cos_sims_data,
-                    gene_list,
-                )
-            else:
-                cos_sims_data = cell_cos_sims
-                for state in cos_sims_dict.keys():
-                    cos_sims_dict[state] = self.update_perturbation_dictionary(
-                        cos_sims_dict[state],
-                        cos_sims_data[state],
-                        gene_list,
-                    )
-            del minibatch
-            del perturbation_batch
-            del original_emb
-            del perturbation_emb
-            del cos_sims_data
-
-            torch.cuda.empty_cache()
-
-        pu.write_perturbation_dictionary(
-            cos_sims_dict,
-            f"{output_path_prefix}_cell_embs_dict_{self.tokens_to_perturb}",
-        )
-
-        if self.emb_mode == "cell_and_gene":
-            pu.write_perturbation_dictionary(
-                stored_gene_embs_dict,
-                f"{output_path_prefix}_gene_embs_dict_{self.tokens_to_perturb}",
-            )
-
-    def isp_perturb_set_special(
-        self,
-        model,
-        filtered_input_data: Dataset,
-        layer_to_quant: int,
-        output_path_prefix: str,
-    ):
-        def make_group_perturbation_batch(example):
-            example_input_ids = example["input_ids"]
-            example["tokens_to_perturb"] = self.tokens_to_perturb
-            indices_to_perturb = [
-                example_input_ids.index(token) if token in example_input_ids else None
-                for token in self.tokens_to_perturb
-            ]
-            indices_to_perturb = [
-                item for item in indices_to_perturb if item is not None
-            ]
-            if len(indices_to_perturb) > 0:
-                example["perturb_index"] = indices_to_perturb
-            else:
-                # -100 indicates tokens to overexpress are not present in rank value encoding
-                example["perturb_index"] = [-100]
-            if self.perturb_type == "delete":
-                example = pu.delete_indices(example)
-            elif self.perturb_type == "overexpress":
-                example = pu.overexpress_tokens(
-                    example, self.max_len, self.special_token
-                )
-                example["n_overflow"] = pu.calc_n_overflow(
-                    self.max_len,
-                    example["length"],
-                    self.tokens_to_perturb,
-                    indices_to_perturb,
-                )
-            return example
-
-        total_batch_length = len(filtered_input_data)
-
-    
-        if self.cell_states_to_model is None:
-            cos_sims_dict = defaultdict(list)
-        else:
-            cos_sims_dict = {
-                state: defaultdict(list)
-                for state in pu.get_possible_states(self.cell_states_to_model)
-            }
-
-        perturbed_data = filtered_input_data.map(
-            make_group_perturbation_batch, num_proc=self.nproc
-        )
-
-        if self.perturb_type == "overexpress":
-            filtered_input_data = filtered_input_data.add_column(
-                "n_overflow", perturbed_data["n_overflow"]
-            )
-            filtered_input_data = filtered_input_data.map(
-                pu.truncate_by_n_overflow_special, num_proc=self.nproc
-            )
-
-        if self.emb_mode == "cls_and_gene":
-            stored_gene_embs_dict = defaultdict(list)
-
-        # iterate through batches
-        for i in trange(0, total_batch_length, self.forward_batch_size):
-            max_range = min(i + self.forward_batch_size, total_batch_length)
-            inds_select = [i for i in range(i, max_range)]
-
-            minibatch = filtered_input_data.select(inds_select)
-            perturbation_batch = perturbed_data.select(inds_select)
-
-            ##### CLS Embedding Mode #####
-            if self.emb_mode == "cls":
-                indices_to_perturb = perturbation_batch["perturb_index"]
-
-                original_cls_emb = get_embs(
-                    model,
-                    minibatch,
-                    "cls",
-                    layer_to_quant,
-                    self.pad_token_id,
-                    self.forward_batch_size,
-                    token_gene_dict=self.token_gene_dict,
-                    summary_stat=None,
-                    silent=True,
-                )
-
-                perturbation_cls_emb = get_embs(
-                    model,
-                    perturbation_batch,
-                    "cls",
-                    layer_to_quant,
-                    self.pad_token_id,
-                    self.forward_batch_size,
-                    token_gene_dict=self.token_gene_dict,
-                    summary_stat=None,
-                    silent=True,
-                )
-
-                # Calculate the cosine similarities
-                cls_cos_sims = pu.quant_cos_sims(
-                    perturbation_cls_emb,
-                    original_cls_emb,
-                    self.cell_states_to_model,
-                    self.state_embs_dict,
-                    emb_mode="cell",
-                )
-
-                # Update perturbation dictionary
-                if self.cell_states_to_model is None:
-                    cos_sims_dict = self.update_perturbation_dictionary(
-                        cos_sims_dict,
-                        cls_cos_sims,
-                        gene_list=None,
-                    )
-                else:
-                    for state in cos_sims_dict.keys():
-                        cos_sims_dict[state] = self.update_perturbation_dictionary(
-                            cos_sims_dict[state],
-                            cls_cos_sims[state],
-                            gene_list=None,
-                        )
-
-            ##### CLS and Gene Embedding Mode #####
-            elif self.emb_mode == "cls_and_gene":              
-                full_original_emb = get_embs(
-                    model,
-                    minibatch,
-                    "gene",
-                    layer_to_quant,
-                    self.pad_token_id,
-                    self.forward_batch_size,
-                    self.token_gene_dict,
-                    summary_stat=None,
-                    silent=True,
-                )
-                indices_to_perturb = perturbation_batch["perturb_index"]
-
-                # remove indices that were perturbed
-                original_emb = pu.remove_perturbed_indices_set(
-                    full_original_emb,
-                    self.perturb_type,
-                    indices_to_perturb,
-                    self.tokens_to_perturb,
-                    minibatch["length"],
-                )
-
-                full_perturbation_emb = get_embs(
-                    model,
-                    perturbation_batch,
-                    "gene",
-                    layer_to_quant,
-                    self.pad_token_id,
-                    self.forward_batch_size,
-                    self.token_gene_dict,
-                    summary_stat=None,
-                    silent=True,
-                )
-    
-                # remove special tokens and padding
-                original_emb = original_emb[:, 1:-1, :]
-                if self.perturb_type == "overexpress":
-                    perturbation_emb = full_perturbation_emb[
-                        :, 1 + len(self.tokens_to_perturb) : -1, :
-                    ]
-                elif self.perturb_type == "delete":
-                    perturbation_emb = full_perturbation_emb[
-                        :, 1 : max(perturbation_batch["length"]) - 1, :
-                    ]
-    
-                n_perturbation_genes = perturbation_emb.size()[1]
-
-                # truncate the original embedding as necessary
-                if self.perturb_type == "overexpress":
-                    def calc_perturbation_length(ids):
-                        if ids == [-100]:
-                            return 0
-                        else:
-                            return len(ids)
-
-                    max_tensor_size = max([length - calc_perturbation_length(ids) - 2 for length, ids in zip(minibatch["length"], indices_to_perturb)])
-
-                    max_n_overflow = max(minibatch["n_overflow"])
-                    if max_n_overflow > 0 and perturbation_emb.size()[1] < original_emb.size()[1]:
-                        original_emb = original_emb[:, 0 : perturbation_emb.size()[1], :]
-                    elif perturbation_emb.size()[1] < original_emb.size()[1]:
-                        original_emb = original_emb[:, 0:max_tensor_size, :]
-    
-                gene_cos_sims = pu.quant_cos_sims(
-                    perturbation_emb,
-                    original_emb,
-                    self.cell_states_to_model,
-                    self.state_embs_dict,
-                    emb_mode="gene",
-                )
-
-                # get cls emb
-                original_cls_emb = full_original_emb[:, 0, :]
-                perturbation_cls_emb = full_perturbation_emb[:, 0, :]
-
-                cls_cos_sims = pu.quant_cos_sims(
-                    perturbation_cls_emb,
-                    original_cls_emb,
-                    self.cell_states_to_model,
-                    self.state_embs_dict,
-                    emb_mode="cell",
-                )
-
-                # get cosine similarities in gene embeddings
-                # since getting gene embeddings, need gene names
-
-                gene_list = minibatch["input_ids"]
-                # need to truncate gene_list
-                genes_to_exclude = self.tokens_to_perturb + [
-                    self.cls_token_id,
-                    self.eos_token_id,
-                ]
-                gene_list = [
-                    [g for g in genes if g not in genes_to_exclude][
-                        :n_perturbation_genes
-                    ]
-                    for genes in gene_list
-                ]
-
-                for cell_i, genes in enumerate(gene_list):
-                    for gene_j, affected_gene in enumerate(genes):
-                        if len(self.genes_to_perturb) > 1:
-                            tokens_to_perturb = tuple(self.tokens_to_perturb)
-                        else:
-                            tokens_to_perturb = self.tokens_to_perturb[0]
-
-                        # fill in the gene cosine similarities
-                        try:
-                            stored_gene_embs_dict[
-                                (tokens_to_perturb, affected_gene)
-                            ].append(gene_cos_sims[cell_i, gene_j].item())
-                        except KeyError:
-                            stored_gene_embs_dict[
-                                (tokens_to_perturb, affected_gene)
-                            ] = gene_cos_sims[cell_i, gene_j].item()
-
-                if self.cell_states_to_model is None:
-                    cos_sims_dict = self.update_perturbation_dictionary(
-                        cos_sims_dict,
-                        cls_cos_sims,
-                        gene_list=None,
-                    )
-                else:
-                    for state in cos_sims_dict.keys():
-                        cos_sims_dict[state] = self.update_perturbation_dictionary(
-                            cos_sims_dict[state],
-                            cls_cos_sims[state],
-                            gene_list=None,
-                        )
-                del full_original_emb
-                del original_emb
-                del full_perturbation_emb
-                del perturbation_emb
-                del gene_cos_sims
-
-            del original_cls_emb
-            del perturbation_cls_emb
-            del cls_cos_sims
-            del minibatch
-            del perturbation_batch
-
-            torch.cuda.empty_cache()
-
-        pu.write_perturbation_dictionary(
-            cos_sims_dict,
-            f"{output_path_prefix}_cell_embs_dict_{self.tokens_to_perturb}",
-        )
-
-        if self.emb_mode == "cls_and_gene":
-            pu.write_perturbation_dictionary(
-                stored_gene_embs_dict,
-                f"{output_path_prefix}_gene_embs_dict_{self.tokens_to_perturb}",
-            )
-
-    def isp_perturb_all(
-        self,
-        model,
-        filtered_input_data: Dataset,
-        layer_to_quant: int,
-        output_path_prefix: str,
-    ):
-        pickle_batch = -1
-        if self.cell_states_to_model is None:
-            cos_sims_dict = defaultdict(list)
-        else:
-            cos_sims_dict = {
-                state: defaultdict(list)
-                for state in pu.get_possible_states(self.cell_states_to_model)
-            }
-
-        if self.emb_mode == "cell_and_gene":
-            stored_gene_embs_dict = defaultdict(list)
-
-        num_inds_perturbed = 1 + self.combos
-        for h in trange(len(filtered_input_data)):
-            example_cell = filtered_input_data.select([h])
-            full_original_emb = get_embs(
-                model,
-                example_cell,
-                "gene",
-                layer_to_quant,
-                self.pad_token_id,
-                self.forward_batch_size,
-                self.token_gene_dict,
-                summary_stat=None,
-                silent=True,
-            )
-
-            if self.cell_states_to_model is not None:
-                original_cell_emb = pu.compute_nonpadded_cell_embedding(
-                    full_original_emb, "mean_pool"
-                )
-
-            # gene_list is used to assign cos sims back to genes
-            gene_list = example_cell["input_ids"][0][:]
-            # need to remove the anchor gene
-            if self.anchor_token is not None:
-                for token in self.anchor_token:
-                    gene_list.remove(token)
-            # index 0 is not overexpressed so remove
-            if self.perturb_type == "overexpress":
-                gene_list = gene_list[num_inds_perturbed:]
-            # remove perturbed index for gene list dict
-            perturbed_gene_dict = {
-                gene: gene_list[:i] + gene_list[i + 1 :]
-                for i, gene in enumerate(gene_list)
-            }
-
-            perturbation_batch, indices_to_perturb = pu.make_perturbation_batch(
-                example_cell,
-                self.perturb_type,
-                self.tokens_to_perturb,
-                self.anchor_token,
-                self.combos,
-                self.nproc,
-            )
-
-            ispall_total_batch_length = len(perturbation_batch)
-            for i in trange(
-                0, ispall_total_batch_length, self.forward_batch_size, leave=False
-            ):
-                ispall_max_range = min(
-                    i + self.forward_batch_size, ispall_total_batch_length
-                )
-                perturbation_minibatch = perturbation_batch.select(
-                    [i for i in range(i, ispall_max_range)]
-                )
-                indices_to_perturb_mini = indices_to_perturb[i:ispall_max_range]
-                gene_list_mini = gene_list[
-                    i:ispall_max_range
-                ]  # only perturbed genes from this minibatch
-
-                full_perturbation_emb = get_embs(
-                    model,
-                    perturbation_minibatch,
-                    "gene",
-                    layer_to_quant,
-                    self.pad_token_id,
-                    self.forward_batch_size,
-                    self.token_gene_dict,
-                    summary_stat=None,
-                    silent=True,
-                )
-
-                del perturbation_minibatch
-
-                # need to remove overexpressed gene to quantify cosine shifts
-                if self.perturb_type == "overexpress":
-                    perturbation_emb = full_perturbation_emb[:, num_inds_perturbed:, :]
-
-                elif self.perturb_type == "delete":
-                    perturbation_emb = full_perturbation_emb
-
-                if (
-                    self.cell_states_to_model is None
-                    or self.emb_mode == "cell_and_gene"
-                ):
-                    original_emb_minibatch = pu.make_comparison_batch(
-                        full_original_emb, indices_to_perturb_mini, perturb_group=False
-                    )
-                    gene_cos_sims = pu.quant_cos_sims(
-                        perturbation_emb,
-                        original_emb_minibatch,
-                        self.cell_states_to_model,
-                        self.state_embs_dict,
-                        emb_mode="gene",
-                    )
-                    del original_emb_minibatch
-
-                if self.cell_states_to_model is not None:
-                    perturbation_cell_emb = pu.compute_nonpadded_cell_embedding(
-                        full_perturbation_emb, "mean_pool"
-                    )
-
-                    cell_cos_sims = pu.quant_cos_sims(
-                        perturbation_cell_emb,
-                        original_cell_emb,
-                        self.cell_states_to_model,
-                        self.state_embs_dict,
-                        emb_mode="cell",
-                    )
-                    del perturbation_cell_emb
-
-                if self.emb_mode == "cell_and_gene":
-                    for perturbation_i, perturbed_gene in enumerate(gene_list_mini):
-                        for gene_j, affected_gene in enumerate(
-                            perturbed_gene_dict[perturbed_gene]
-                        ):
-                            try:
-                                stored_gene_embs_dict[
-                                    (perturbed_gene, affected_gene)
-                                ].append(gene_cos_sims[perturbation_i, gene_j].item())
-                            except KeyError:
-                                stored_gene_embs_dict[
-                                    (perturbed_gene, affected_gene)
-                                ] = gene_cos_sims[perturbation_i, gene_j].item()
-
-                del full_perturbation_emb
-
-                if self.cell_states_to_model is None:
-                    cos_sims_data = torch.mean(gene_cos_sims, dim=1)
-                    cos_sims_dict = self.update_perturbation_dictionary(
-                        cos_sims_dict,
-                        cos_sims_data,
-                        gene_list_mini,
-                    )
-                else:
-                    cos_sims_data = cell_cos_sims
-                    for state in cos_sims_dict.keys():
-                        cos_sims_dict[state] = self.update_perturbation_dictionary(
-                            cos_sims_dict[state],
-                            cos_sims_data[state],
-                            gene_list_mini,
-                        )
-
-                # save dict to disk every self.clear_mem_ncells/10 (default 100) simulated cells
-                if i % self.clear_mem_ncells / 10 == 0:
-                    pu.write_perturbation_dictionary(
-                        cos_sims_dict,
-                        f"{output_path_prefix}_dict_cell_embs_{h}batch{pickle_batch}",
-                    )
-                    if self.emb_mode == "cell_and_gene":
-                        pu.write_perturbation_dictionary(
-                            stored_gene_embs_dict,
-                            f"{output_path_prefix}_dict_gene_embs_{h}batch{pickle_batch}",
-                        )
-
-                # reset and clear memory every self.clear_mem_ncells (default 1000) simulated cells or at the end of the example cell
-                if i % self.clear_mem_ncells == 0:
-                    pickle_batch += 1
-                    if self.cell_states_to_model is None:
-                        cos_sims_dict = defaultdict(list)
-                    else:
-                        cos_sims_dict = {
-                            state: defaultdict(list)
-                            for state in pu.get_possible_states(
-                                self.cell_states_to_model
-                            )
-                        }
-
-                    if self.emb_mode == "cell_and_gene":
-                        stored_gene_embs_dict = defaultdict(list)
-
-                    torch.cuda.empty_cache()
-
-            pu.write_perturbation_dictionary(
-                cos_sims_dict,
-                f"{output_path_prefix}_dict_cell_embs_{h}batch{pickle_batch}",
-            )
-
-            if self.emb_mode == "cell_and_gene":
-                pu.write_perturbation_dictionary(
-                    stored_gene_embs_dict,
-                    f"{output_path_prefix}_dict_gene_embs_{h}batch{pickle_batch}",
-                )
-
-            pickle_batch = -1
-            if self.cell_states_to_model is None:
-                cos_sims_dict = defaultdict(list)
-            else:
-                cos_sims_dict = {
-                    state: defaultdict(list)
-                    for state in pu.get_possible_states(self.cell_states_to_model)
-                }
-
-            if self.emb_mode == "cell_and_gene":
-                stored_gene_embs_dict = defaultdict(list)
-
-            # clear memory between cells
-            del perturbation_batch
-            del full_original_emb
-            if self.cell_states_to_model is not None:
-                del original_cell_emb
-            torch.cuda.empty_cache()
-
-    def isp_perturb_all_special(
-        self,
-        model,
-        filtered_input_data: Dataset,
-        layer_to_quant: int,
-        output_path_prefix: str,
-    ):
-        pickle_batch = -1
-        if self.cell_states_to_model is None:
-            cos_sims_dict = defaultdict(list)
-        else:
-            cos_sims_dict = {
-                state: defaultdict(list)
-                for state in pu.get_possible_states(self.cell_states_to_model)
-            }
-
-        if self.emb_mode == "cls_and_gene":
-            stored_gene_embs_dict = defaultdict(list)
-
-        num_inds_perturbed = 1 + self.combos
-        for h in trange(len(filtered_input_data)):
-            example_cell = filtered_input_data.select([h])
-
-            # get original example cell cls and/or gene embs for comparison
-            if self.emb_mode == "cls":
-                original_cls_emb = get_embs(
-                    model,
-                    example_cell,
-                    "cls",
-                    layer_to_quant,
-                    self.pad_token_id,
-                    self.forward_batch_size,
-                    self.token_gene_dict,
-                    summary_stat=None,
-                    silent=True,
-                )
-            elif self.emb_mode == "cls_and_gene":
-                full_original_emb = get_embs(
-                    model,
-                    example_cell,
-                    "gene",
-                    layer_to_quant,
-                    self.pad_token_id,
-                    self.forward_batch_size,
-                    self.token_gene_dict,
-                    summary_stat=None,
-                    silent=True,
-                )
-                original_cls_emb = full_original_emb[:, 0, :].clone().detach()
-
-            # gene_list is used to assign cos sims back to genes
-            gene_list = example_cell["input_ids"][0][:]
-
-            # need to remove special tokens
-            for token in [self.cls_token_id, self.eos_token_id]:
-                gene_list.remove(token)
-            # need to remove the anchor gene
-            if self.anchor_token is not None:
-                for token in self.anchor_token:
-                    gene_list.remove(token)
-            # index 0 is not overexpressed so remove
-            if self.perturb_type == "overexpress":
-                gene_list = gene_list[num_inds_perturbed:]
-            # remove perturbed index for gene list dict
-            perturbed_gene_dict = {
-                gene: gene_list[:i] + gene_list[i + 1 :]
-                for i, gene in enumerate(gene_list)
-            }
-
-            perturbation_batch, indices_to_perturb = pu.make_perturbation_batch_special(
-                example_cell,
-                self.perturb_type,
-                self.tokens_to_perturb,
-                self.anchor_token,
-                self.combos,
-                self.nproc,
-            )
-
-            ispall_total_batch_length = len(perturbation_batch)
-            for i in trange(
-                0, ispall_total_batch_length, self.forward_batch_size, leave=False
-            ):
-                ispall_max_range = min(
-                    i + self.forward_batch_size, ispall_total_batch_length
-                )
-                perturbation_minibatch = perturbation_batch.select(
-                    [i for i in range(i, ispall_max_range)]
-                )
-                indices_to_perturb_mini = indices_to_perturb[i:ispall_max_range]
-                gene_list_mini = gene_list[
-                    i:ispall_max_range
-                ]  # only perturbed genes from this minibatch
-
-                ##### CLS Embedding Mode #####
-                if self.emb_mode == "cls":
-                    # Extract cls embeddings from perturbed cells
-                    perturbation_cls_emb = get_embs(
-                        model,
-                        perturbation_minibatch,
-                        "cls",
-                        layer_to_quant,
-                        self.pad_token_id,
-                        self.forward_batch_size,
-                        self.token_gene_dict,
-                        summary_stat=None,
-                        silent=True,
-                    )
-
-                    # Calculate cosine similarities
-                    cls_cos_sims = pu.quant_cos_sims(
-                        perturbation_cls_emb,
-                        original_cls_emb,
-                        self.cell_states_to_model,
-                        self.state_embs_dict,
-                        emb_mode="cell",
-                    )
-
-                    if self.cell_states_to_model is None:
-                        cos_sims_dict = self.update_perturbation_dictionary(
-                            cos_sims_dict,
-                            cls_cos_sims,
-                            gene_list_mini,
-                        )
-                    else:
-                        for state in cos_sims_dict.keys():
-                            cos_sims_dict[state] = self.update_perturbation_dictionary(
-                                cos_sims_dict[state],
-                                cls_cos_sims[state],
-                                gene_list_mini,
-                            )
-
-                    del perturbation_minibatch
-                    del perturbation_cls_emb
-                    del cls_cos_sims
-
-                ##### CLS and Gene Embedding Mode #####
-                elif self.emb_mode == "cls_and_gene":
-                    full_perturbation_emb = get_embs(
-                        model,
-                        perturbation_minibatch,
-                        "gene",
-                        layer_to_quant,
-                        self.pad_token_id,
-                        self.forward_batch_size,
-                        self.token_gene_dict,
-                        summary_stat=None,
-                        silent=True,
-                    )
-
-                    # need to remove overexpressed gene and cls/eos to quantify cosine shifts
-                    if self.perturb_type == "overexpress":
-                        perturbation_emb = (
-                            full_perturbation_emb[:, 1 + num_inds_perturbed : -1, :]
-                            .clone()
-                            .detach()
-                        )
-                    elif self.perturb_type == "delete":
-                        perturbation_emb = (
-                            full_perturbation_emb[:, 1:-1, :].clone().detach()
-                        )
-
-                    original_emb_minibatch = pu.make_comparison_batch(
-                        full_original_emb, indices_to_perturb_mini, perturb_group=False
-                    )
-
-                    original_emb_minibatch = (
-                        original_emb_minibatch[:, 1:-1, :].clone().detach()
-                    )
-                    gene_cos_sims = pu.quant_cos_sims(
-                        perturbation_emb,
-                        original_emb_minibatch,
-                        self.cell_states_to_model,
-                        self.state_embs_dict,
-                        emb_mode="gene",
-                    )
-
-                    for perturbation_i, perturbed_gene in enumerate(gene_list_mini):
-                        for gene_j, affected_gene in enumerate(
-                            perturbed_gene_dict[perturbed_gene]
-                        ):
-                            try:
-                                stored_gene_embs_dict[
-                                    (perturbed_gene, affected_gene)
-                                ].append(gene_cos_sims[perturbation_i, gene_j].item())
-                            except KeyError:
-                                stored_gene_embs_dict[
-                                    (perturbed_gene, affected_gene)
-                                ] = gene_cos_sims[perturbation_i, gene_j].item()
-
-                    # get cls emb
-                    perturbation_cls_emb = (
-                        full_perturbation_emb[:, 0, :].clone().detach()
-                    )
-
-                    cls_cos_sims = pu.quant_cos_sims(
-                        perturbation_cls_emb,
-                        original_cls_emb,
-                        self.cell_states_to_model,
-                        self.state_embs_dict,
-                        emb_mode="cell",
-                    )
-
-                    if self.cell_states_to_model is None:
-                        cos_sims_dict = self.update_perturbation_dictionary(
-                            cos_sims_dict,
-                            cls_cos_sims,
-                            gene_list_mini,
-                        )
-                    else:
-                        for state in cos_sims_dict.keys():
-                            cos_sims_dict[state] = self.update_perturbation_dictionary(
-                                cos_sims_dict[state],
-                                cls_cos_sims[state],
-                                gene_list_mini,
-                            )
-
-                    del perturbation_minibatch
-                    del original_emb_minibatch
-                    del full_perturbation_emb
-                    del perturbation_emb
-                    del perturbation_cls_emb
-                    del cls_cos_sims
-                    del gene_cos_sims
-
-                # save dict to disk every self.clear_mem_ncells/10 (default 100) simulated cells
-                if i % max(1, self.clear_mem_ncells / 10) == 0:
-                    pu.write_perturbation_dictionary(
-                        cos_sims_dict,
-                        f"{output_path_prefix}_dict_cell_embs_{h}batch{pickle_batch}",
-                    )
-                    if self.emb_mode == "cls_and_gene":
-                        pu.write_perturbation_dictionary(
-                            stored_gene_embs_dict,
-                            f"{output_path_prefix}_dict_gene_embs_{h}batch{pickle_batch}",
-                        )
-
-                # reset and clear memory every self.clear_mem_ncells (default 1000) simulated cells or at the end of the example cell
-                if i % self.clear_mem_ncells == 0:
-                    pickle_batch += 1
-                    if self.cell_states_to_model is None:
-                        cos_sims_dict = defaultdict(list)
-                    else:
-                        cos_sims_dict = {
-                            state: defaultdict(list)
-                            for state in pu.get_possible_states(
-                                self.cell_states_to_model
-                            )
-                        }
-
-                    if self.emb_mode == "cls_and_gene":
-                        stored_gene_embs_dict = defaultdict(list)
-
-                    torch.cuda.empty_cache()
-
-            pu.write_perturbation_dictionary(
-                cos_sims_dict,
-                f"{output_path_prefix}_dict_cell_embs_{h}batch{pickle_batch}",
-            )
-
-            if self.emb_mode == "cls_and_gene":
-                pu.write_perturbation_dictionary(
-                    stored_gene_embs_dict,
-                    f"{output_path_prefix}_dict_gene_embs_{h}batch{pickle_batch}",
-                )
-
-            pickle_batch = -1
-            if self.cell_states_to_model is None:
-                cos_sims_dict = defaultdict(list)
-            else:
-                cos_sims_dict = {
-                    state: defaultdict(list)
-                    for state in pu.get_possible_states(self.cell_states_to_model)
-                }
-
-            if self.emb_mode == "cls_and_gene":
-                stored_gene_embs_dict = defaultdict(list)
-
-            # clear memory between cells
-            del perturbation_batch
-            del original_cls_emb
-            if self.emb_mode == "cls_and_gene":
-                del full_original_emb
-            torch.cuda.empty_cache()
-
-    def update_perturbation_dictionary(
-        self,
-        cos_sims_dict: defaultdict,
-        cos_sims_data: torch.Tensor,
-        gene_list=None,
-    ):
-        if gene_list is not None and cos_sims_data.shape[0] != len(gene_list):
-            logger.error(
-                f"len(cos_sims_data.shape[0]) != len(gene_list). \n \
-                            {cos_sims_data.shape[0]=}.\n \
-                            {len(gene_list)=}."
-            )
-            raise
-
-        if self.perturb_group is True:
-            if len(self.tokens_to_perturb) > 1:
-                perturbed_genes = tuple(self.tokens_to_perturb)
-            else:
-                perturbed_genes = self.tokens_to_perturb[0]
-
-            # if cell embeddings, can just append
-            # shape will be (batch size, 1)
-            cos_sims_data = torch.squeeze(cos_sims_data).tolist()
-
-            # handle case of single cell left
-            if not isinstance(cos_sims_data, list):
-                cos_sims_data = [cos_sims_data]
-
-            cos_sims_dict[(perturbed_genes, "cell_emb")] += cos_sims_data
-
-        else:
-            for i, cos in enumerate(cos_sims_data.tolist()):
-                cos_sims_dict[(gene_list[i], "cell_emb")].append(cos)
-
-        return cos_sims_dict

geneformer/in_silico_perturber_stats.py

@@ -1,1104 +0,0 @@
-"""
-Geneformer in silico perturber stats generator.
-
-**Usage:**
-
-.. code-block :: python
-
-    >>> from geneformer import InSilicoPerturberStats
-    >>> ispstats = InSilicoPerturberStats(mode="goal_state_shift",
-    ...    cell_states_to_model={"state_key": "disease",
-    ...                          "start_state": "dcm",
-    ...                          "goal_state": "nf",
-    ...                          "alt_states": ["hcm", "other1", "other2"]})
-    >>> ispstats.get_stats("path/to/input_data",
-    ...                    None,
-    ...                    "path/to/output_directory",
-    ...                    "output_prefix")
-
-**Description:**
-
-| Aggregates data or calculates stats for in silico perturbations based on type of statistics specified in InSilicoPerturberStats.
-| Input data is raw in silico perturbation results in the form of dictionaries outputted by ``in_silico_perturber``.
-
-"""
-
-
-import logging
-import os
-import pickle
-import random
-from pathlib import Path
-
-import numpy as np
-import pandas as pd
-import statsmodels.stats.multitest as smt
-from scipy.stats import ranksums
-from sklearn.mixture import GaussianMixture
-from tqdm.auto import tqdm, trange
-
-from . import ENSEMBL_DICTIONARY_FILE, TOKEN_DICTIONARY_FILE
-from .perturber_utils import flatten_list, validate_cell_states_to_model
-
-logger = logging.getLogger(__name__)
-
-
-# invert dictionary keys/values
-def invert_dict(dictionary):
-    return {v: k for k, v in dictionary.items()}
-
-
-def read_dict(cos_sims_dict, cell_or_gene_emb, anchor_token):
-    if cell_or_gene_emb == "cell":
-        cell_emb_dict = {
-            k: v for k, v in cos_sims_dict.items() if v and "cell_emb" in k
-        }
-        return [cell_emb_dict]
-    elif cell_or_gene_emb == "gene":
-        if anchor_token is None:
-            gene_emb_dict = {k: v for k, v in cos_sims_dict.items() if v}
-        else:
-            gene_emb_dict = {
-                k: v for k, v in cos_sims_dict.items() if v and anchor_token == k[0]
-            }
-    return [gene_emb_dict]
-
-
-# read raw dictionary files
-def read_dictionaries(
-    input_data_directory,
-    cell_or_gene_emb,
-    anchor_token,
-    cell_states_to_model,
-    pickle_suffix,
-):
-    file_found = False
-    file_path_list = []
-    if cell_states_to_model is None:
-        dict_list = []
-    else:
-        validate_cell_states_to_model(cell_states_to_model)
-        cell_states_to_model_valid = {
-            state: value
-            for state, value in cell_states_to_model.items()
-            if state != "state_key"
-            and cell_states_to_model[state] is not None
-            and cell_states_to_model[state] != []
-        }
-        cell_states_list = []
-        # flatten all state values into list
-        for state in cell_states_to_model_valid:
-            value = cell_states_to_model_valid[state]
-            if isinstance(value, list):
-                cell_states_list += value
-            else:
-                cell_states_list.append(value)
-        state_dict = {state_value: dict() for state_value in cell_states_list}
-    for file in os.listdir(input_data_directory):
-        # process only files with given suffix (e.g. "_raw.pickle")
-        if file.endswith(pickle_suffix):
-            file_found = True
-            file_path_list += [f"{input_data_directory}/{file}"]
-    for file_path in tqdm(file_path_list):
-        with open(file_path, "rb") as fp:
-            cos_sims_dict = pickle.load(fp)
-            if cell_states_to_model is None:
-                dict_list += read_dict(cos_sims_dict, cell_or_gene_emb, anchor_token)
-            else:
-                for state_value in cell_states_list:
-                    new_dict = read_dict(
-                        cos_sims_dict[state_value], cell_or_gene_emb, anchor_token
-                    )[0]
-                    for key in new_dict:
-                        try:
-                            state_dict[state_value][key] += new_dict[key]
-                        except KeyError:
-                            state_dict[state_value][key] = new_dict[key]
-
-    if not file_found:
-        logger.error(
-            "No raw data for processing found within provided directory. "
-            "Please ensure data files end with '{pickle_suffix}'."
-        )
-        raise
-    if cell_states_to_model is None:
-        return dict_list
-    else:
-        return state_dict
-
-
-# get complete gene list
-def get_gene_list(dict_list, mode):
-    if mode == "cell":
-        position = 0
-    elif mode == "gene":
-        position = 1
-    gene_set = set()
-    if isinstance(dict_list, list):
-        for dict_i in dict_list:
-            gene_set.update([k[position] for k, v in dict_i.items() if v])
-    elif isinstance(dict_list, dict):
-        for state, dict_i in dict_list.items():
-            gene_set.update([k[position] for k, v in dict_i.items() if v])
-    else:
-        logger.error(
-            "dict_list should be a list, or if modeling shift to goal states, a dict. "
-            f"{type(dict_list)} is not the correct format."
-        )
-        raise
-    gene_list = list(gene_set)
-    if mode == "gene":
-        gene_list.remove("cell_emb")
-    gene_list.sort()
-    return gene_list
-
-
-def token_tuple_to_ensembl_ids(token_tuple, gene_token_id_dict):
-    try:
-        return tuple([gene_token_id_dict.get(i, np.nan) for i in token_tuple])
-    except TypeError:
-        return gene_token_id_dict.get(token_tuple, np.nan)
-
-
-def n_detections(token, dict_list, mode, anchor_token):
-    cos_sim_megalist = []
-    for dict_i in dict_list:
-        if mode == "cell":
-            cos_sim_megalist += dict_i.get((token, "cell_emb"), [])
-        elif mode == "gene":
-            cos_sim_megalist += dict_i.get((anchor_token, token), [])
-    return len(cos_sim_megalist)
-
-
-def get_fdr(pvalues):
-    return list(smt.multipletests(pvalues, alpha=0.05, method="fdr_bh")[1])
-
-
-def get_impact_component(test_value, gaussian_mixture_model):
-    impact_border = gaussian_mixture_model.means_[0][0]
-    nonimpact_border = gaussian_mixture_model.means_[1][0]
-    if test_value > nonimpact_border:
-        impact_component = 0
-    elif test_value < impact_border:
-        impact_component = 1
-    else:
-        impact_component_raw = gaussian_mixture_model.predict([[test_value]])[0]
-        if impact_component_raw == 1:
-            impact_component = 0
-        elif impact_component_raw == 0:
-            impact_component = 1
-    return impact_component
-
-
-# aggregate data for single perturbation in multiple cells
-def isp_aggregate_grouped_perturb(cos_sims_df, dict_list, genes_perturbed):
-    names = ["Cosine_sim", "Gene"]
-    cos_sims_full_dfs = []
-    if isinstance(genes_perturbed, list):
-        if len(genes_perturbed) > 1:
-            gene_ids_df = cos_sims_df.loc[
-                np.isin(
-                    [set(idx) for idx in cos_sims_df["Ensembl_ID"]],
-                    set(genes_perturbed),
-                ),
-                :,
-            ]
-        else:
-            gene_ids_df = cos_sims_df.loc[
-                np.isin(cos_sims_df["Ensembl_ID"], genes_perturbed), :
-            ]
-    else:
-        logger.error(
-            "aggregate_data is for perturbation of single gene or single group of genes. genes_to_perturb should be formatted as list."
-        )
-        raise
-
-    if gene_ids_df.empty:
-        logger.error("genes_to_perturb not found in data.")
-        raise
-
-    tokens = gene_ids_df["Gene"]
-    symbols = gene_ids_df["Gene_name"]
-
-    for token, symbol in zip(tokens, symbols):
-        cos_shift_data = []
-        for dict_i in dict_list:
-            cos_shift_data += dict_i.get((token, "cell_emb"), [])
-
-        df = pd.DataFrame(columns=names)
-        df["Cosine_sim"] = cos_shift_data
-        df["Gene"] = symbol
-        cos_sims_full_dfs.append(df)
-
-    return pd.concat(cos_sims_full_dfs)
-
-
-def find(variable, x):
-    try:
-        if x in variable:  # Test if variable is iterable and contains x
-            return True
-        elif x == variable:
-            return True
-    except (ValueError, TypeError):
-        return x == variable  # Test if variable is x if non-iterable
-
-
-def isp_aggregate_gene_shifts(
-    cos_sims_df, dict_list, gene_token_id_dict, gene_id_name_dict, token_dtype
-):
-    cos_shift_data = dict()
-    for i in trange(cos_sims_df.shape[0]):
-        token = cos_sims_df["Gene"][i]
-        for dict_i in dict_list:
-            if token_dtype == "nontuple":
-                affected_pairs = [k for k, v in dict_i.items() if k[0] == token]
-            else:
-                affected_pairs = [k for k, v in dict_i.items() if find(k[0], token)]
-            for key in affected_pairs:
-                if key in cos_shift_data.keys():
-                    cos_shift_data[key] += dict_i.get(key, [])
-                else:
-                    cos_shift_data[key] = dict_i.get(key, [])
-
-    cos_data_mean = {
-        k: [np.mean(v), np.std(v), len(v)] for k, v in cos_shift_data.items()
-    }
-    cos_sims_full_df = pd.DataFrame()
-    cos_sims_full_df["Perturbed"] = [k[0] for k, v in cos_data_mean.items()]
-    cos_sims_full_df["Gene_name"] = [
-        cos_sims_df[cos_sims_df["Gene"] == k[0]]["Gene_name"].item()
-        for k, v in cos_data_mean.items()
-    ]
-    cos_sims_full_df["Ensembl_ID"] = [
-        cos_sims_df[cos_sims_df["Gene"] == k[0]]["Ensembl_ID"].item()
-        for k, v in cos_data_mean.items()
-    ]
-
-    cos_sims_full_df["Affected"] = [k[1] for k, v in cos_data_mean.items()]
-    cos_sims_full_df["Affected_gene_name"] = [
-        gene_id_name_dict.get(gene_token_id_dict.get(token, np.nan), np.nan)
-        for token in cos_sims_full_df["Affected"]
-    ]
-    cos_sims_full_df["Affected_Ensembl_ID"] = [
-        gene_token_id_dict.get(token, np.nan) for token in cos_sims_full_df["Affected"]
-    ]
-    cos_sims_full_df["Cosine_sim_mean"] = [v[0] for k, v in cos_data_mean.items()]
-    cos_sims_full_df["Cosine_sim_stdev"] = [v[1] for k, v in cos_data_mean.items()]
-    cos_sims_full_df["N_Detections"] = [v[2] for k, v in cos_data_mean.items()]
-
-    specific_val = "cell_emb"
-    cos_sims_full_df["temp"] = list(cos_sims_full_df["Affected"] == specific_val)
-    # reorder so cell embs are at the top and all are subordered by magnitude of cosine sim
-    cos_sims_full_df = cos_sims_full_df.sort_values(
-        by=(["temp", "Cosine_sim_mean"]), ascending=[False, True]
-    ).drop("temp", axis=1)
-
-    return cos_sims_full_df
-
-
-# stats comparing cos sim shifts towards goal state of test perturbations vs random perturbations
-def isp_stats_to_goal_state(
-    cos_sims_df, result_dict, cell_states_to_model, genes_perturbed
-):
-    if (
-        ("alt_states" not in cell_states_to_model.keys())
-        or (len(cell_states_to_model["alt_states"]) == 0)
-        or (cell_states_to_model["alt_states"] == [None])
-    ):
-        alt_end_state_exists = False
-    elif (len(cell_states_to_model["alt_states"]) > 0) and (
-        cell_states_to_model["alt_states"] != [None]
-    ):
-        alt_end_state_exists = True
-
-    # for single perturbation in multiple cells, there are no random perturbations to compare to
-    if genes_perturbed != "all":
-        cos_sims_full_df = pd.DataFrame()
-
-        cos_shift_data_end = []
-        token = cos_sims_df["Gene"][0]
-        cos_shift_data_end += result_dict[cell_states_to_model["goal_state"]].get(
-            (token, "cell_emb"), []
-        )
-        cos_sims_full_df["Shift_to_goal_end"] = [np.mean(cos_shift_data_end)]
-        if alt_end_state_exists is True:
-            for alt_state in cell_states_to_model["alt_states"]:
-                cos_shift_data_alt_state = []
-                cos_shift_data_alt_state += result_dict.get(alt_state).get(
-                    (token, "cell_emb"), []
-                )
-                cos_sims_full_df[f"Shift_to_alt_end_{alt_state}"] = [
-                    np.mean(cos_shift_data_alt_state)
-                ]
-
-        # sort by shift to desired state
-        cos_sims_full_df = cos_sims_full_df.sort_values(
-            by=["Shift_to_goal_end"], ascending=[False]
-        )
-        return cos_sims_full_df
-
-    elif genes_perturbed == "all":
-        goal_end_random_megalist = []
-        if alt_end_state_exists is True:
-            alt_end_state_random_dict = {
-                alt_state: [] for alt_state in cell_states_to_model["alt_states"]
-            }
-        for i in trange(cos_sims_df.shape[0]):
-            token = cos_sims_df["Gene"][i]
-            goal_end_random_megalist += result_dict[
-                cell_states_to_model["goal_state"]
-            ].get((token, "cell_emb"), [])
-            if alt_end_state_exists is True:
-                for alt_state in cell_states_to_model["alt_states"]:
-                    alt_end_state_random_dict[alt_state] += result_dict[alt_state].get(
-                        (token, "cell_emb"), []
-                    )
-
-        # downsample to improve speed of ranksums
-        if len(goal_end_random_megalist) > 100_000:
-            random.seed(42)
-            goal_end_random_megalist = random.sample(
-                goal_end_random_megalist, k=100_000
-            )
-        if alt_end_state_exists is True:
-            for alt_state in cell_states_to_model["alt_states"]:
-                if len(alt_end_state_random_dict[alt_state]) > 100_000:
-                    random.seed(42)
-                    alt_end_state_random_dict[alt_state] = random.sample(
-                        alt_end_state_random_dict[alt_state], k=100_000
-                    )
-
-        names = [
-            "Gene",
-            "Gene_name",
-            "Ensembl_ID",
-            "Shift_to_goal_end",
-            "Goal_end_vs_random_pval",
-        ]
-        if alt_end_state_exists is True:
-            [
-                names.append(f"Shift_to_alt_end_{alt_state}")
-                for alt_state in cell_states_to_model["alt_states"]
-            ]
-            names.append(names.pop(names.index("Goal_end_vs_random_pval")))
-            [
-                names.append(f"Alt_end_vs_random_pval_{alt_state}")
-                for alt_state in cell_states_to_model["alt_states"]
-            ]
-        cos_sims_full_df = pd.DataFrame(columns=names)
-
-        n_detections_dict = dict()
-        for i in trange(cos_sims_df.shape[0]):
-            token = cos_sims_df["Gene"][i]
-            name = cos_sims_df["Gene_name"][i]
-            ensembl_id = cos_sims_df["Ensembl_ID"][i]
-            goal_end_cos_sim_megalist = result_dict[
-                cell_states_to_model["goal_state"]
-            ].get((token, "cell_emb"), [])
-            n_detections_dict[token] = len(goal_end_cos_sim_megalist)
-            mean_goal_end = np.mean(goal_end_cos_sim_megalist)
-            pval_goal_end = ranksums(
-                goal_end_random_megalist, goal_end_cos_sim_megalist
-            ).pvalue
-
-            if alt_end_state_exists is True:
-                alt_end_state_dict = {
-                    alt_state: [] for alt_state in cell_states_to_model["alt_states"]
-                }
-                for alt_state in cell_states_to_model["alt_states"]:
-                    alt_end_state_dict[alt_state] = result_dict[alt_state].get(
-                        (token, "cell_emb"), []
-                    )
-                    alt_end_state_dict[f"{alt_state}_mean"] = np.mean(
-                        alt_end_state_dict[alt_state]
-                    )
-                    alt_end_state_dict[f"{alt_state}_pval"] = ranksums(
-                        alt_end_state_random_dict[alt_state],
-                        alt_end_state_dict[alt_state],
-                    ).pvalue
-
-            results_dict = dict()
-            results_dict["Gene"] = token
-            results_dict["Gene_name"] = name
-            results_dict["Ensembl_ID"] = ensembl_id
-            results_dict["Shift_to_goal_end"] = mean_goal_end
-            results_dict["Goal_end_vs_random_pval"] = pval_goal_end
-            if alt_end_state_exists is True:
-                for alt_state in cell_states_to_model["alt_states"]:
-                    results_dict[f"Shift_to_alt_end_{alt_state}"] = alt_end_state_dict[
-                        f"{alt_state}_mean"
-                    ]
-                    results_dict[
-                        f"Alt_end_vs_random_pval_{alt_state}"
-                    ] = alt_end_state_dict[f"{alt_state}_pval"]
-
-            cos_sims_df_i = pd.DataFrame(results_dict, index=[i])
-            cos_sims_full_df = pd.concat([cos_sims_full_df, cos_sims_df_i])
-
-        cos_sims_full_df["Goal_end_FDR"] = get_fdr(
-            list(cos_sims_full_df["Goal_end_vs_random_pval"])
-        )
-        if alt_end_state_exists is True:
-            for alt_state in cell_states_to_model["alt_states"]:
-                cos_sims_full_df[f"Alt_end_FDR_{alt_state}"] = get_fdr(
-                    list(cos_sims_full_df[f"Alt_end_vs_random_pval_{alt_state}"])
-                )
-
-        # quantify number of detections of each gene
-        cos_sims_full_df["N_Detections"] = [
-            n_detections_dict[token] for token in cos_sims_full_df["Gene"]
-        ]
-
-        # sort by shift to desired state
-        cos_sims_full_df["Sig"] = [
-            1 if fdr < 0.05 else 0 for fdr in cos_sims_full_df["Goal_end_FDR"]
-        ]
-        cos_sims_full_df = cos_sims_full_df.sort_values(
-            by=["Sig", "Shift_to_goal_end", "Goal_end_FDR"],
-            ascending=[False, False, True],
-        )
-
-        return cos_sims_full_df
-
-
-# stats comparing cos sim shifts of test perturbations vs null distribution
-def isp_stats_vs_null(cos_sims_df, dict_list, null_dict_list):
-    cos_sims_full_df = cos_sims_df.copy()
-
-    cos_sims_full_df["Test_avg_shift"] = np.zeros(cos_sims_df.shape[0], dtype=float)
-    cos_sims_full_df["Null_avg_shift"] = np.zeros(cos_sims_df.shape[0], dtype=float)
-    cos_sims_full_df["Test_vs_null_avg_shift"] = np.zeros(
-        cos_sims_df.shape[0], dtype=float
-    )
-    cos_sims_full_df["Test_vs_null_pval"] = np.zeros(cos_sims_df.shape[0], dtype=float)
-    cos_sims_full_df["Test_vs_null_FDR"] = np.zeros(cos_sims_df.shape[0], dtype=float)
-    cos_sims_full_df["N_Detections_test"] = np.zeros(
-        cos_sims_df.shape[0], dtype="uint32"
-    )
-    cos_sims_full_df["N_Detections_null"] = np.zeros(
-        cos_sims_df.shape[0], dtype="uint32"
-    )
-
-    for i in trange(cos_sims_df.shape[0]):
-        token = cos_sims_df["Gene"][i]
-        test_shifts = []
-        null_shifts = []
-
-        for dict_i in dict_list:
-            test_shifts += dict_i.get((token, "cell_emb"), [])
-
-        for dict_i in null_dict_list:
-            null_shifts += dict_i.get((token, "cell_emb"), [])
-
-        cos_sims_full_df.loc[i, "Test_avg_shift"] = np.mean(test_shifts)
-        cos_sims_full_df.loc[i, "Null_avg_shift"] = np.mean(null_shifts)
-        cos_sims_full_df.loc[i, "Test_vs_null_avg_shift"] = np.mean(
-            test_shifts
-        ) - np.mean(null_shifts)
-        cos_sims_full_df.loc[i, "Test_vs_null_pval"] = ranksums(
-            test_shifts, null_shifts, nan_policy="omit"
-        ).pvalue
-        # remove nan values
-        cos_sims_full_df.Test_vs_null_pval = np.where(
-            np.isnan(cos_sims_full_df.Test_vs_null_pval),
-            1,
-            cos_sims_full_df.Test_vs_null_pval,
-        )
-        cos_sims_full_df.loc[i, "N_Detections_test"] = len(test_shifts)
-        cos_sims_full_df.loc[i, "N_Detections_null"] = len(null_shifts)
-
-    cos_sims_full_df["Test_vs_null_FDR"] = get_fdr(
-        cos_sims_full_df["Test_vs_null_pval"]
-    )
-
-    cos_sims_full_df["Sig"] = [
-        1 if fdr < 0.05 else 0 for fdr in cos_sims_full_df["Test_vs_null_FDR"]
-    ]
-    cos_sims_full_df = cos_sims_full_df.sort_values(
-        by=["Sig", "Test_vs_null_avg_shift", "Test_vs_null_FDR"],
-        ascending=[False, False, True],
-    )
-    return cos_sims_full_df
-
-
-# stats for identifying perturbations with largest effect within a given set of cells
-# fits a mixture model to 2 components (impact vs. non-impact) and
-# reports the most likely component for each test perturbation
-# Note: because assumes given perturbation has a consistent effect in the cells tested,
-# we recommend only using the mixture model strategy with uniform cell populations
-def isp_stats_mixture_model(cos_sims_df, dict_list, combos, anchor_token):
-    names = ["Gene", "Gene_name", "Ensembl_ID"]
-
-    if combos == 0:
-        names += ["Test_avg_shift"]
-    elif combos == 1:
-        names += [
-            "Anchor_shift",
-            "Test_token_shift",
-            "Sum_of_indiv_shifts",
-            "Combo_shift",
-            "Combo_minus_sum_shift",
-        ]
-
-    names += ["Impact_component", "Impact_component_percent"]
-
-    cos_sims_full_df = pd.DataFrame(columns=names)
-    avg_values = []
-    gene_names = []
-
-    for i in trange(cos_sims_df.shape[0]):
-        token = cos_sims_df["Gene"][i]
-        name = cos_sims_df["Gene_name"][i]
-        ensembl_id = cos_sims_df["Ensembl_ID"][i]
-        cos_shift_data = []
-
-        for dict_i in dict_list:
-            if (combos == 0) and (anchor_token is not None):
-                cos_shift_data += dict_i.get((anchor_token, token), [])
-            else:
-                cos_shift_data += dict_i.get((token, "cell_emb"), [])
-
-        # Extract values for current gene
-        if combos == 0:
-            test_values = cos_shift_data
-        elif combos == 1:
-            test_values = []
-            for tup in cos_shift_data:
-                test_values.append(tup[2])
-
-        if len(test_values) > 0:
-            avg_value = np.mean(test_values)
-            avg_values.append(avg_value)
-            gene_names.append(name)
-
-    # fit Gaussian mixture model to dataset of mean for each gene
-    avg_values_to_fit = np.array(avg_values).reshape(-1, 1)
-    gm = GaussianMixture(n_components=2, random_state=0).fit(avg_values_to_fit)
-
-    for i in trange(cos_sims_df.shape[0]):
-        token = cos_sims_df["Gene"][i]
-        name = cos_sims_df["Gene_name"][i]
-        ensembl_id = cos_sims_df["Ensembl_ID"][i]
-        cos_shift_data = []
-
-        for dict_i in dict_list:
-            if (combos == 0) and (anchor_token is not None):
-                cos_shift_data += dict_i.get((anchor_token, token), [])
-            else:
-                cos_shift_data += dict_i.get((token, "cell_emb"), [])
-
-        if combos == 0:
-            mean_test = np.mean(cos_shift_data)
-            impact_components = [
-                get_impact_component(value, gm) for value in cos_shift_data
-            ]
-        elif combos == 1:
-            anchor_cos_sim_megalist = [
-                anchor for anchor, token, combo in cos_shift_data
-            ]
-            token_cos_sim_megalist = [token for anchor, token, combo in cos_shift_data]
-            anchor_plus_token_cos_sim_megalist = [
-                1 - ((1 - anchor) + (1 - token))
-                for anchor, token, combo in cos_shift_data
-            ]
-            combo_anchor_token_cos_sim_megalist = [
-                combo for anchor, token, combo in cos_shift_data
-            ]
-            combo_minus_sum_cos_sim_megalist = [
-                combo - (1 - ((1 - anchor) + (1 - token)))
-                for anchor, token, combo in cos_shift_data
-            ]
-
-            mean_anchor = np.mean(anchor_cos_sim_megalist)
-            mean_token = np.mean(token_cos_sim_megalist)
-            mean_sum = np.mean(anchor_plus_token_cos_sim_megalist)
-            mean_test = np.mean(combo_anchor_token_cos_sim_megalist)
-            mean_combo_minus_sum = np.mean(combo_minus_sum_cos_sim_megalist)
-
-            impact_components = [
-                get_impact_component(value, gm)
-                for value in combo_anchor_token_cos_sim_megalist
-            ]
-
-        impact_component = get_impact_component(mean_test, gm)
-        impact_component_percent = np.mean(impact_components) * 100
-
-        data_i = [token, name, ensembl_id]
-        if combos == 0:
-            data_i += [mean_test]
-        elif combos == 1:
-            data_i += [
-                mean_anchor,
-                mean_token,
-                mean_sum,
-                mean_test,
-                mean_combo_minus_sum,
-            ]
-        data_i += [impact_component, impact_component_percent]
-
-        cos_sims_df_i = pd.DataFrame(dict(zip(names, data_i)), index=[i])
-        cos_sims_full_df = pd.concat([cos_sims_full_df, cos_sims_df_i])
-
-    # quantify number of detections of each gene
-    if anchor_token is None:
-        cos_sims_full_df["N_Detections"] = [
-            n_detections(i, dict_list, "cell", anchor_token)
-            for i in cos_sims_full_df["Gene"]
-        ]
-    else:
-        cos_sims_full_df["N_Detections"] = [
-            n_detections(i, dict_list, "gene", anchor_token)
-            for i in cos_sims_full_df["Gene"]
-        ]
-
-    if combos == 0:
-        cos_sims_full_df = cos_sims_full_df.sort_values(
-            by=["Impact_component", "Test_avg_shift"], ascending=[False, True]
-        )
-    elif combos == 1:
-        cos_sims_full_df = cos_sims_full_df.sort_values(
-            by=["Impact_component", "Combo_minus_sum_shift"], ascending=[False, True]
-        )
-    return cos_sims_full_df
-
-
-class InSilicoPerturberStats:
-    valid_option_dict = {
-        "mode": {
-            "goal_state_shift",
-            "vs_null",
-            "mixture_model",
-            "aggregate_data",
-            "aggregate_gene_shifts",
-        },
-        "genes_perturbed": {"all", list},
-        "combos": {0, 1},
-        "anchor_gene": {None, str},
-        "cell_states_to_model": {None, dict},
-        "pickle_suffix": {None, str},
-    }
-
-    def __init__(
-        self,
-        mode="mixture_model",
-        genes_perturbed="all",
-        combos=0,
-        anchor_gene=None,
-        cell_states_to_model=None,
-        pickle_suffix="_raw.pickle",
-        token_dictionary_file=TOKEN_DICTIONARY_FILE,
-        gene_name_id_dictionary_file=ENSEMBL_DICTIONARY_FILE,
-    ):
-        """
-        Initialize in silico perturber stats generator.
-
-        **Parameters:**
-
-        mode : {"goal_state_shift", "vs_null", "mixture_model", "aggregate_data", "aggregate_gene_shifts"}
-            | Type of stats.
-            | "goal_state_shift": perturbation vs. random for desired cell state shift
-            | "vs_null": perturbation vs. null from provided null distribution dataset
-            | "mixture_model": perturbation in impact vs. no impact component of mixture model (no goal direction)
-            | "aggregate_data": aggregates cosine shifts for single perturbation in multiple cells
-            | "aggregate_gene_shifts": aggregates cosine shifts of genes in response to perturbation(s)
-        genes_perturbed : "all", list
-            | Genes perturbed in isp experiment.
-            | Default is assuming genes_to_perturb in isp experiment was "all" (each gene in each cell).
-            | Otherwise, may provide a list of ENSEMBL IDs of genes perturbed as a group all together.
-        combos : {0,1,2}
-            | Whether genex perturbed in isp experiment were perturbed individually (0), in pairs (1), or in triplets (2).
-        anchor_gene : None, str
-            | ENSEMBL ID of gene to use as anchor in combination perturbations or in testing effect on downstream genes.
-            | For example, if combos=1 and anchor_gene="ENSG00000136574":
-            |    analyzes data for anchor gene perturbed in combination with each other gene.
-            | However, if combos=0 and anchor_gene="ENSG00000136574":
-            |    analyzes data for the effect of anchor gene's perturbation on the embedding of each other gene.
-        cell_states_to_model: None, dict
-            | Cell states to model if testing perturbations that achieve goal state change.
-            | Four-item dictionary with keys: state_key, start_state, goal_state, and alt_states
-            | state_key: key specifying name of column in .dataset that defines the start/goal states
-            | start_state: value in the state_key column that specifies the start state
-            | goal_state: value in the state_key column taht specifies the goal end state
-            | alt_states: list of values in the state_key column that specify the alternate end states
-            | For example: {"state_key": "disease",
-            |               "start_state": "dcm",
-            |               "goal_state": "nf",
-            |               "alt_states": ["hcm", "other1", "other2"]}
-        token_dictionary_file : Path
-            | Path to pickle file containing token dictionary (Ensembl ID:token).
-        gene_name_id_dictionary_file : Path
-            | Path to pickle file containing gene name to ID dictionary (gene name:Ensembl ID).
-        """
-
-        self.mode = mode
-        self.genes_perturbed = genes_perturbed
-        self.combos = combos
-        self.anchor_gene = anchor_gene
-        self.cell_states_to_model = cell_states_to_model
-        self.pickle_suffix = pickle_suffix
-
-        self.validate_options()
-
-        # load token dictionary (Ensembl IDs:token)
-        with open(token_dictionary_file, "rb") as f:
-            self.gene_token_dict = pickle.load(f)
-
-        # load gene name dictionary (gene name:Ensembl ID)
-        with open(gene_name_id_dictionary_file, "rb") as f:
-            self.gene_name_id_dict = pickle.load(f)
-
-        if anchor_gene is None:
-            self.anchor_token = None
-        else:
-            self.anchor_token = self.gene_token_dict[self.anchor_gene]
-
-    def validate_options(self):
-        for attr_name, valid_options in self.valid_option_dict.items():
-            attr_value = self.__dict__[attr_name]
-            if type(attr_value) not in {list, dict}:
-                if attr_name in {"anchor_gene"}:
-                    continue
-                elif attr_value in valid_options:
-                    continue
-            valid_type = False
-            for option in valid_options:
-                if (option in [str, int, list, dict]) and isinstance(
-                    attr_value, option
-                ):
-                    valid_type = True
-                    break
-            if not valid_type:
-                logger.error(
-                    f"Invalid option for {attr_name}. "
-                    f"Valid options for {attr_name}: {valid_options}"
-                )
-                raise
-
-        if self.cell_states_to_model is not None:
-            if len(self.cell_states_to_model.items()) == 1:
-                logger.warning(
-                    "The single value dictionary for cell_states_to_model will be "
-                    "replaced with a dictionary with named keys for start, goal, and alternate states. "
-                    "Please specify state_key, start_state, goal_state, and alt_states "
-                    "in the cell_states_to_model dictionary for future use. "
-                    "For example, cell_states_to_model={"
-                    "'state_key': 'disease', "
-                    "'start_state': 'dcm', "
-                    "'goal_state': 'nf', "
-                    "'alt_states': ['hcm', 'other1', 'other2']}"
-                )
-                for key, value in self.cell_states_to_model.items():
-                    if (len(value) == 3) and isinstance(value, tuple):
-                        if (
-                            isinstance(value[0], list)
-                            and isinstance(value[1], list)
-                            and isinstance(value[2], list)
-                        ):
-                            if len(value[0]) == 1 and len(value[1]) == 1:
-                                all_values = value[0] + value[1] + value[2]
-                                if len(all_values) == len(set(all_values)):
-                                    continue
-                # reformat to the new named key format
-                state_values = flatten_list(list(self.cell_states_to_model.values()))
-                self.cell_states_to_model = {
-                    "state_key": list(self.cell_states_to_model.keys())[0],
-                    "start_state": state_values[0][0],
-                    "goal_state": state_values[1][0],
-                    "alt_states": state_values[2:][0],
-                }
-            elif set(self.cell_states_to_model.keys()) == {
-                "state_key",
-                "start_state",
-                "goal_state",
-                "alt_states",
-            }:
-                if (
-                    (self.cell_states_to_model["state_key"] is None)
-                    or (self.cell_states_to_model["start_state"] is None)
-                    or (self.cell_states_to_model["goal_state"] is None)
-                ):
-                    logger.error(
-                        "Please specify 'state_key', 'start_state', and 'goal_state' in cell_states_to_model."
-                    )
-                    raise
-
-                if (
-                    self.cell_states_to_model["start_state"]
-                    == self.cell_states_to_model["goal_state"]
-                ):
-                    logger.error("All states must be unique.")
-                    raise
-
-                if self.cell_states_to_model["alt_states"] is not None:
-                    if not isinstance(self.cell_states_to_model["alt_states"], list):
-                        logger.error(
-                            "self.cell_states_to_model['alt_states'] must be a list (even if it is one element)."
-                        )
-                        raise
-                    if len(self.cell_states_to_model["alt_states"]) != len(
-                        set(self.cell_states_to_model["alt_states"])
-                    ):
-                        logger.error("All states must be unique.")
-                        raise
-
-            elif set(self.cell_states_to_model.keys()) == {
-                "state_key",
-                "start_state",
-                "goal_state",
-            }:
-                self.cell_states_to_model["alt_states"] = []
-            else:
-                logger.error(
-                    "cell_states_to_model must only have the following four keys: "
-                    "'state_key', 'start_state', 'goal_state', 'alt_states'."
-                    "For example, cell_states_to_model={"
-                    "'state_key': 'disease', "
-                    "'start_state': 'dcm', "
-                    "'goal_state': 'nf', "
-                    "'alt_states': ['hcm', 'other1', 'other2']}"
-                )
-                raise
-
-            if self.anchor_gene is not None:
-                self.anchor_gene = None
-                logger.warning(
-                    "anchor_gene set to None. "
-                    "Currently, anchor gene not available "
-                    "when modeling multiple cell states."
-                )
-
-        if self.combos > 0:
-            if self.anchor_gene is None:
-                logger.error(
-                    "Currently, stats are only supported for combination "
-                    "in silico perturbation run with anchor gene. Please add "
-                    "anchor gene when using with combos > 0. "
-                )
-                raise
-
-        if (self.mode == "mixture_model") and (self.genes_perturbed != "all"):
-            logger.error(
-                "Mixture model mode requires multiple gene perturbations to fit model "
-                "so is incompatible with a single grouped perturbation."
-            )
-            raise
-        if (self.mode == "aggregate_data") and (self.genes_perturbed == "all"):
-            logger.error(
-                "Simple data aggregation mode is for single perturbation in multiple cells "
-                "so is incompatible with a genes_perturbed being 'all'."
-            )
-            raise
-
-    def get_stats(
-        self,
-        input_data_directory,
-        null_dist_data_directory,
-        output_directory,
-        output_prefix,
-        null_dict_list=None,
-    ):
-        """
-        Get stats for in silico perturbation data and save as results in output_directory.
-
-        **Parameters:**
-
-        input_data_directory : Path
-            | Path to directory containing cos_sim dictionary inputs
-        null_dist_data_directory : Path
-            | Path to directory containing null distribution cos_sim dictionary inputs
-        output_directory : Path
-            | Path to directory where perturbation data will be saved as .csv
-        output_prefix : str
-            | Prefix for output .csv
-        null_dict_list: list[dict]
-            | List of loaded null distribution dictionary if more than one comparison vs. the null is to be performed
-
-        **Outputs:**
-
-        Definition of possible columns in .csv output file.
-
-        | Of note, not all columns will be present in all output files.
-        | Some columns are specific to particular perturbation modes.
-
-        | "Gene": gene token
-        | "Gene_name": gene name
-        | "Ensembl_ID": gene Ensembl ID
-        | "N_Detections": number of cells in which each gene or gene combination was detected in the input dataset
-        | "Sig": 1 if FDR<0.05, otherwise 0
-
-        | "Shift_to_goal_end": cosine shift from start state towards goal end state in response to given perturbation
-        | "Shift_to_alt_end": cosine shift from start state towards alternate end state in response to given perturbation
-        | "Goal_end_vs_random_pval": pvalue of cosine shift from start state towards goal end state by Wilcoxon
-        |     pvalue compares shift caused by perturbing given gene compared to random genes
-        | "Alt_end_vs_random_pval": pvalue of cosine shift from start state towards alternate end state by Wilcoxon
-        |     pvalue compares shift caused by perturbing given gene compared to random genes
-        | "Goal_end_FDR": Benjamini-Hochberg correction of "Goal_end_vs_random_pval"
-        | "Alt_end_FDR": Benjamini-Hochberg correction of "Alt_end_vs_random_pval"
-
-        | "Test_avg_shift": cosine shift in response to given perturbation in cells from test distribution
-        | "Null_avg_shift": cosine shift in response to given perturbation in cells from null distribution (e.g. random cells)
-        | "Test_vs_null_avg_shift": difference in cosine shift in cells from test vs. null distribution
-        |     (i.e. "Test_avg_shift" minus "Null_avg_shift")
-        | "Test_vs_null_pval": pvalue of cosine shift in test vs. null distribution
-        | "Test_vs_null_FDR": Benjamini-Hochberg correction of "Test_vs_null_pval"
-        | "N_Detections_test": "N_Detections" in cells from test distribution
-        | "N_Detections_null": "N_Detections" in cells from null distribution
-
-        | "Anchor_shift": cosine shift in response to given perturbation of anchor gene
-        | "Test_token_shift": cosine shift in response to given perturbation of test gene
-        | "Sum_of_indiv_shifts": sum of cosine shifts in response to individually perturbing test and anchor genes
-        | "Combo_shift": cosine shift in response to given perturbation of both anchor and test gene(s) in combination
-        | "Combo_minus_sum_shift": difference of cosine shifts in response combo perturbation vs. sum of individual perturbations
-        |     (i.e. "Combo_shift" minus "Sum_of_indiv_shifts")
-        | "Impact_component": whether the given perturbation was modeled to be within the impact component by the mixture model
-        |     1: within impact component; 0: not within impact component
-        | "Impact_component_percent": percent of cells in which given perturbation was modeled to be within impact component
-
-        | In case of aggregating data / gene shifts:
-        | "Perturbed": ID(s) of gene(s) being perturbed
-        | "Affected": ID of affected gene or "cell_emb" indicating the impact on the cell embedding as a whole
-        | "Cosine_sim_mean": mean of cosine similarity of cell or affected gene in original vs. perturbed
-        | "Cosine_sim_stdev": standard deviation of cosine similarity of cell or affected gene in original vs. perturbed
-        """
-
-        if self.mode not in [
-            "goal_state_shift",
-            "vs_null",
-            "mixture_model",
-            "aggregate_data",
-            "aggregate_gene_shifts",
-        ]:
-            logger.error(
-                "Currently, only modes available are stats for goal_state_shift, "
-                "vs_null (comparing to null distribution), "
-                "mixture_model (fitting mixture model for perturbations with or without impact), "
-                "and aggregating data for single perturbations or for gene embedding shifts."
-            )
-            raise
-
-        self.gene_token_id_dict = invert_dict(self.gene_token_dict)
-        self.gene_id_name_dict = invert_dict(self.gene_name_id_dict)
-
-        # obtain total gene list
-        if (self.combos == 0) and (self.anchor_token is not None):
-            # cos sim data for effect of gene perturbation on the embedding of each other gene
-            dict_list = read_dictionaries(
-                input_data_directory,
-                "gene",
-                self.anchor_token,
-                self.cell_states_to_model,
-                self.pickle_suffix,
-            )
-            gene_list = get_gene_list(dict_list, "gene")
-        elif (
-            (self.combos == 0)
-            and (self.anchor_token is None)
-            and (self.mode == "aggregate_gene_shifts")
-        ):
-            dict_list = read_dictionaries(
-                input_data_directory,
-                "gene",
-                self.anchor_token,
-                self.cell_states_to_model,
-                self.pickle_suffix,
-            )
-            gene_list = get_gene_list(dict_list, "cell")
-        else:
-            # cos sim data for effect of gene perturbation on the embedding of each cell
-            dict_list = read_dictionaries(
-                input_data_directory,
-                "cell",
-                self.anchor_token,
-                self.cell_states_to_model,
-                self.pickle_suffix,
-            )
-            gene_list = get_gene_list(dict_list, "cell")
-
-        # initiate results dataframe
-        cos_sims_df_initial = pd.DataFrame(
-            {
-                "Gene": gene_list,
-                "Gene_name": [self.token_to_gene_name(item) for item in gene_list],
-                "Ensembl_ID": [
-                    token_tuple_to_ensembl_ids(genes, self.gene_token_id_dict)
-                    if self.genes_perturbed != "all"
-                    else self.gene_token_id_dict[genes[1]]
-                    if isinstance(genes, tuple)
-                    else self.gene_token_id_dict[genes]
-                    for genes in gene_list
-                ],
-            },
-            index=[i for i in range(len(gene_list))],
-        )
-
-        if self.mode == "goal_state_shift":
-            cos_sims_df = isp_stats_to_goal_state(
-                cos_sims_df_initial,
-                dict_list,
-                self.cell_states_to_model,
-                self.genes_perturbed,
-            )
-
-        elif self.mode == "vs_null":
-            if null_dict_list is None:
-                null_dict_list = read_dictionaries(
-                    null_dist_data_directory,
-                    "cell",
-                    self.anchor_token,
-                    self.cell_states_to_model,
-                    self.pickle_suffix,
-                )
-            cos_sims_df = isp_stats_vs_null(
-                cos_sims_df_initial, dict_list, null_dict_list
-            )
-
-        elif self.mode == "mixture_model":
-            cos_sims_df = isp_stats_mixture_model(
-                cos_sims_df_initial, dict_list, self.combos, self.anchor_token
-            )
-
-        elif self.mode == "aggregate_data":
-            cos_sims_df = isp_aggregate_grouped_perturb(
-                cos_sims_df_initial, dict_list, self.genes_perturbed
-            )
-
-        elif self.mode == "aggregate_gene_shifts":
-            if (self.genes_perturbed == "all") and (self.combos == 0):
-                tuple_types = [
-                    True if isinstance(genes, tuple) else False for genes in gene_list
-                ]
-                if all(tuple_types):
-                    token_dtype = "tuple"
-                elif not any(tuple_types):
-                    token_dtype = "nontuple"
-                else:
-                    token_dtype = "mix"
-            else:
-                token_dtype = "mix"
-
-            cos_sims_df = isp_aggregate_gene_shifts(
-                cos_sims_df_initial,
-                dict_list,
-                self.gene_token_id_dict,
-                self.gene_id_name_dict,
-                token_dtype,
-            )
-
-        # save perturbation stats to output_path
-        output_path = (Path(output_directory) / output_prefix).with_suffix(".csv")
-        cos_sims_df.to_csv(output_path)
-
-    def token_to_gene_name(self, item):
-        if np.issubdtype(type(item), np.integer):
-            return self.gene_id_name_dict.get(
-                self.gene_token_id_dict.get(item, np.nan), np.nan
-            )
-        if isinstance(item, tuple):
-            return tuple(
-                [
-                    self.gene_id_name_dict.get(
-                        self.gene_token_id_dict.get(i, np.nan), np.nan
-                    )
-                    for i in item
-                ]
-            )

geneformer/mtl/__init__.py

@@ -1 +0,0 @@
-# ruff: noqa: F401

geneformer/mtl/collators.py

@@ -1,76 +0,0 @@
-# imports
-import torch
-import pickle
-from ..collator_for_classification import DataCollatorForGeneClassification
-from .. import TOKEN_DICTIONARY_FILE
-
-"""Geneformer collator for multi-task cell classification."""
-
-class DataCollatorForMultitaskCellClassification(DataCollatorForGeneClassification):
-    class_type = "cell"
-
-    @staticmethod
-    def load_token_dictionary():
-        with open(TOKEN_DICTIONARY_FILE, 'rb') as f:
-            return pickle.load(f)
-
-    def __init__(self, *args, **kwargs) -> None:
-        # Load the token dictionary
-        token_dictionary = self.load_token_dictionary()
-        # Use the loaded token dictionary
-        super().__init__(token_dictionary=token_dictionary, *args, **kwargs)
-
-    def _prepare_batch(self, features):
-        # Process inputs as usual
-        batch = self.tokenizer.pad(
-            features,
-            class_type=self.class_type,
-            padding=self.padding,
-            max_length=self.max_length,
-            pad_to_multiple_of=self.pad_to_multiple_of,
-            return_tensors="pt",
-        )
-
-        # Check if labels are present
-        if "label" in features[0]:
-            # Initialize labels dictionary for all tasks
-            labels = {task: [] for task in features[0]["label"].keys()}
-            # Populate labels for each task
-            for feature in features:
-                for task, label in feature["label"].items():
-                    labels[task].append(label)
-
-            # Convert label lists to tensors, handling dictionaries appropriately
-            for task in labels:
-                if isinstance(labels[task][0], (list, torch.Tensor)):
-                    dtype = torch.long
-                    labels[task] = torch.tensor(labels[task], dtype=dtype)
-                elif isinstance(labels[task][0], dict):
-                    # Handle dict specifically if needed
-                    pass  # Resolve nested data structure
-
-            # Update the batch to include task-specific labels
-            batch["labels"] = labels
-        else:
-            # If no labels are present, create empty labels for all tasks
-            batch["labels"] = {
-                task: torch.tensor([], dtype=torch.long)
-                for task in features[0]["input_ids"].keys()
-            }
-
-        return batch
-
-    def __call__(self, features):
-        batch = self._prepare_batch(features)
-        for k, v in batch.items():
-            if torch.is_tensor(v):
-                batch[k] = v.clone().detach()
-            elif isinstance(v, dict):
-                # Assuming nested structure needs conversion
-                batch[k] = {
-                    task: torch.tensor(labels, dtype=torch.int64)
-                    for task, labels in v.items()
-                }
-            else:
-                batch[k] = torch.tensor(v, dtype=torch.int64)
-        return batch

geneformer/mtl/data.py

@@ -1,162 +0,0 @@
-import os
-from .collators import DataCollatorForMultitaskCellClassification
-from .imports import *
-
-def validate_columns(dataset, required_columns, dataset_type):
-    """Ensures required columns are present in the dataset."""
-    missing_columns = [col for col in required_columns if col not in dataset.column_names]
-    if missing_columns:
-        raise KeyError(
-            f"Missing columns in {dataset_type} dataset: {missing_columns}. "
-            f"Available columns: {dataset.column_names}"
-        )
-
-
-def create_label_mappings(dataset, task_to_column):
-    """Creates label mappings for the dataset."""
-    task_label_mappings = {}
-    num_labels_list = []
-    for task, column in task_to_column.items():
-        unique_values = sorted(set(dataset[column]))
-        mapping = {label: idx for idx, label in enumerate(unique_values)}
-        task_label_mappings[task] = mapping
-        num_labels_list.append(len(unique_values))
-    return task_label_mappings, num_labels_list
-
-
-def save_label_mappings(mappings, path):
-    """Saves label mappings to a pickle file."""
-    with open(path, "wb") as f:
-        pickle.dump(mappings, f)
-
-
-def load_label_mappings(path):
-    """Loads label mappings from a pickle file."""
-    with open(path, "rb") as f:
-        return pickle.load(f)
-
-
-def transform_dataset(dataset, task_to_column, task_label_mappings, config, is_test):
-    """Transforms the dataset to the required format."""
-    transformed_dataset = []
-    cell_id_mapping = {}
-
-    for idx, record in enumerate(dataset):
-        transformed_record = {
-            "input_ids": torch.tensor(record["input_ids"], dtype=torch.long),
-            "cell_id": idx,  # Index-based cell ID
-        }
-
-        if not is_test:
-            label_dict = {
-                task: task_label_mappings[task][record[column]]
-                for task, column in task_to_column.items()
-            }
-        else:
-            label_dict = {task: -1 for task in config["task_names"]}
-
-        transformed_record["label"] = label_dict
-        transformed_dataset.append(transformed_record)
-        cell_id_mapping[idx] = record.get("unique_cell_id", idx)
-
-    return transformed_dataset, cell_id_mapping
-
-
-def load_and_preprocess_data(dataset_path, config, is_test=False, dataset_type=""):
-    """Main function to load and preprocess data."""
-    try:
-        dataset = load_from_disk(dataset_path)
-
-        # Setup task and column mappings
-        task_names = [f"task{i+1}" for i in range(len(config["task_columns"]))]
-        task_to_column = dict(zip(task_names, config["task_columns"]))
-        config["task_names"] = task_names
-
-        label_mappings_path = os.path.join(
-            config["results_dir"],
-            f"task_label_mappings{'_val' if dataset_type == 'validation' else ''}.pkl"
-        )
-
-        if not is_test:
-            validate_columns(dataset, task_to_column.values(), dataset_type)
-
-            # Create and save label mappings
-            task_label_mappings, num_labels_list = create_label_mappings(dataset, task_to_column)
-            save_label_mappings(task_label_mappings, label_mappings_path)
-        else:
-            # Load existing mappings for test data
-            task_label_mappings = load_label_mappings(label_mappings_path)
-            num_labels_list = [len(mapping) for mapping in task_label_mappings.values()]
-
-        # Transform dataset
-        transformed_dataset, cell_id_mapping = transform_dataset(
-            dataset, task_to_column, task_label_mappings, config, is_test
-        )
-
-        return transformed_dataset, cell_id_mapping, num_labels_list
-
-    except KeyError as e:
-        raise ValueError(f"Configuration error or dataset key missing: {e}")
-    except Exception as e:
-        raise RuntimeError(f"Error during data loading or preprocessing: {e}")
-
-
-def preload_and_process_data(config):
-    """Preloads and preprocesses train and validation datasets."""
-    # Process train data and save mappings
-    train_data = load_and_preprocess_data(config["train_path"], config, dataset_type="train")
-
-    # Process validation data and save mappings
-    val_data = load_and_preprocess_data(config["val_path"], config, dataset_type="validation")
-
-    # Validate that the mappings match
-    validate_label_mappings(config)
-
-    return (*train_data, *val_data[:2])  # Return train and val data along with mappings
-
-
-def validate_label_mappings(config):
-    """Ensures train and validation label mappings are consistent."""
-    train_mappings_path = os.path.join(config["results_dir"], "task_label_mappings.pkl")
-    val_mappings_path = os.path.join(config["results_dir"], "task_label_mappings_val.pkl")
-    train_mappings = load_label_mappings(train_mappings_path)
-    val_mappings = load_label_mappings(val_mappings_path)
-
-    for task_name in config["task_names"]:
-        if train_mappings[task_name] != val_mappings[task_name]:
-            raise ValueError(
-                f"Mismatch in label mappings for task '{task_name}'.\n"
-                f"Train Mapping: {train_mappings[task_name]}\n"
-                f"Validation Mapping: {val_mappings[task_name]}"
-            )
-
-
-def get_data_loader(preprocessed_dataset, batch_size):
-    """Creates a DataLoader with optimal settings."""
-    return DataLoader(
-        preprocessed_dataset,
-        batch_size=batch_size,
-        shuffle=True,
-        collate_fn=DataCollatorForMultitaskCellClassification(),
-        num_workers=os.cpu_count(),
-        pin_memory=True,
-    )
-
-
-def preload_data(config):
-    """Preprocesses train and validation data for trials."""
-    train_loader = get_data_loader(*preload_and_process_data(config)[:2], config["batch_size"])
-    val_loader = get_data_loader(*preload_and_process_data(config)[2:4], config["batch_size"])
-    return train_loader, val_loader
-
-
-def load_and_preprocess_test_data(config):
-    """Loads and preprocesses test data."""
-    return load_and_preprocess_data(config["test_path"], config, is_test=True)
-
-
-def prepare_test_loader(config):
-    """Prepares DataLoader for test data."""
-    test_dataset, cell_id_mapping, num_labels_list = load_and_preprocess_test_data(config)
-    test_loader = get_data_loader(test_dataset, config["batch_size"])
-    return test_loader, cell_id_mapping, num_labels_list

geneformer/mtl/eval_utils.py

@@ -1,88 +0,0 @@
-import pandas as pd
-
-from .imports import *  # noqa # isort:skip
-from .data import prepare_test_loader  # noqa # isort:skip
-from .model import GeneformerMultiTask
-
-
-def evaluate_test_dataset(model, device, test_loader, cell_id_mapping, config):
-    task_pred_labels = {task_name: [] for task_name in config["task_names"]}
-    task_pred_probs = {task_name: [] for task_name in config["task_names"]}
-    cell_ids = []
-
-    # # Load task label mappings from pickle file
-    # with open(f"{config['results_dir']}/task_label_mappings.pkl", "rb") as f:
-    #     task_label_mappings = pickle.load(f)
-
-    model.eval()
-    with torch.no_grad():
-        for batch in test_loader:
-            input_ids = batch["input_ids"].to(device)
-            attention_mask = batch["attention_mask"].to(device)
-            _, logits, _ = model(input_ids, attention_mask)
-            for sample_idx in range(len(batch["input_ids"])):
-                cell_id = cell_id_mapping[batch["cell_id"][sample_idx].item()]
-                cell_ids.append(cell_id)
-                for i, task_name in enumerate(config["task_names"]):
-                    pred_label = torch.argmax(logits[i][sample_idx], dim=-1).item()
-                    pred_prob = (
-                        torch.softmax(logits[i][sample_idx], dim=-1).cpu().numpy()
-                    )
-                    task_pred_labels[task_name].append(pred_label)
-                    task_pred_probs[task_name].append(pred_prob)
-
-    # Save test predictions with cell IDs and probabilities to CSV
-    test_results_dir = config["results_dir"]
-    os.makedirs(test_results_dir, exist_ok=True)
-    test_preds_file = os.path.join(test_results_dir, "test_preds.csv")
-
-    rows = []
-    for sample_idx in range(len(cell_ids)):
-        row = {"Cell ID": cell_ids[sample_idx]}
-        for task_name in config["task_names"]:
-            row[f"{task_name} Prediction"] = task_pred_labels[task_name][sample_idx]
-            row[f"{task_name} Probabilities"] = ",".join(
-                map(str, task_pred_probs[task_name][sample_idx])
-            )
-        rows.append(row)
-
-    df = pd.DataFrame(rows)
-    df.to_csv(test_preds_file, index=False)
-    print(f"Test predictions saved to {test_preds_file}")
-
-
-def load_and_evaluate_test_model(config):
-    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    test_loader, cell_id_mapping, num_labels_list = prepare_test_loader(config)
-    model_directory = os.path.join(config["model_save_path"], "GeneformerMultiTask")
-    hyperparams_path = os.path.join(model_directory, "hyperparameters.json")
-
-    # Load the saved best hyperparameters
-    with open(hyperparams_path, "r") as f:
-        best_hyperparams = json.load(f)
-
-    # Extract the task weights if present, otherwise set to None
-    task_weights = best_hyperparams.get("task_weights", None)
-    normalized_task_weights = task_weights if task_weights else []
-
-    # Print the loaded hyperparameters
-    print("Loaded hyperparameters:")
-    for param, value in best_hyperparams.items():
-        if param == "task_weights":
-            print(f"normalized_task_weights: {value}")
-        else:
-            print(f"{param}: {value}")
-
-    best_model_path = os.path.join(model_directory, "pytorch_model.bin")
-    best_model = GeneformerMultiTask(
-        config["pretrained_path"],
-        num_labels_list,
-        dropout_rate=best_hyperparams["dropout_rate"],
-        use_task_weights=config["use_task_weights"],
-        task_weights=normalized_task_weights,
-    )
-    best_model.load_state_dict(torch.load(best_model_path))
-    best_model.to(device)
-
-    evaluate_test_dataset(best_model, device, test_loader, cell_id_mapping, config)
-    print("Evaluation completed.")

geneformer/mtl/imports.py

@@ -1,43 +0,0 @@
-import functools
-import gc
-import json
-import os
-import pickle
-import sys
-import warnings
-from enum import Enum
-from itertools import chain
-from typing import Dict, List, Optional, Union
-
-import numpy as np
-import optuna
-import pandas as pd
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-import torch.optim as optim
-from datasets import load_from_disk
-from sklearn.metrics import accuracy_score, f1_score, roc_auc_score, roc_curve
-from sklearn.model_selection import train_test_split
-from sklearn.preprocessing import LabelEncoder
-from torch.utils.data import DataLoader
-from transformers import (
-    AdamW,
-    BatchEncoding,
-    BertConfig,
-    BertModel,
-    DataCollatorForTokenClassification,
-    SpecialTokensMixin,
-    get_cosine_schedule_with_warmup,
-    get_linear_schedule_with_warmup,
-    get_scheduler,
-)
-from transformers.utils import logging, to_py_obj
-
-from .collators import DataCollatorForMultitaskCellClassification
-
-# local modules
-from .data import get_data_loader, preload_and_process_data
-from .model import GeneformerMultiTask
-from .optuna_utils import create_optuna_study
-from .utils import save_model

geneformer/mtl/model.py

@@ -1,121 +0,0 @@
-import torch
-import torch.nn as nn
-from transformers import BertConfig, BertModel
-
-
-class AttentionPool(nn.Module):
-    """Attention-based pooling layer."""
-
-    def __init__(self, hidden_size):
-        super(AttentionPool, self).__init__()
-        self.attention_weights = nn.Parameter(torch.randn(hidden_size, 1))
-        nn.init.xavier_uniform_(
-            self.attention_weights
-        )  # https://pytorch.org/docs/stable/nn.init.html
-
-    def forward(self, hidden_states):
-        attention_scores = torch.matmul(hidden_states, self.attention_weights)
-        attention_scores = torch.softmax(attention_scores, dim=1)
-        pooled_output = torch.sum(hidden_states * attention_scores, dim=1)
-        return pooled_output
-
-
-class GeneformerMultiTask(nn.Module):
-    def __init__(
-        self,
-        pretrained_path,
-        num_labels_list,
-        dropout_rate=0.1,
-        use_task_weights=False,
-        task_weights=None,
-        max_layers_to_freeze=0,
-        use_attention_pooling=False,
-    ):
-        super(GeneformerMultiTask, self).__init__()
-        self.config = BertConfig.from_pretrained(pretrained_path)
-        self.bert = BertModel(self.config)
-        self.num_labels_list = num_labels_list
-        self.use_task_weights = use_task_weights
-        self.dropout = nn.Dropout(dropout_rate)
-        self.use_attention_pooling = use_attention_pooling
-
-        if use_task_weights and (
-            task_weights is None or len(task_weights) != len(num_labels_list)
-        ):
-            raise ValueError(
-                "Task weights must be defined and match the number of tasks when 'use_task_weights' is True."
-            )
-        self.task_weights = (
-            task_weights if use_task_weights else [1.0] * len(num_labels_list)
-        )
-
-        # Freeze the specified initial layers
-        for layer in self.bert.encoder.layer[:max_layers_to_freeze]:
-            for param in layer.parameters():
-                param.requires_grad = False
-
-        self.attention_pool = (
-            AttentionPool(self.config.hidden_size) if use_attention_pooling else None
-        )
-
-        self.classification_heads = nn.ModuleList(
-            [
-                nn.Linear(self.config.hidden_size, num_labels)
-                for num_labels in num_labels_list
-            ]
-        )
-        # initialization of the classification heads: https://pytorch.org/docs/stable/nn.init.html
-        for head in self.classification_heads:
-            nn.init.xavier_uniform_(head.weight)
-            nn.init.zeros_(head.bias)
-
-    def forward(self, input_ids, attention_mask, labels=None):
-        try:
-            outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)
-        except Exception as e:
-            raise RuntimeError(f"Error during BERT forward pass: {e}")
-
-        sequence_output = outputs.last_hidden_state
-
-        try:
-            pooled_output = (
-                self.attention_pool(sequence_output)
-                if self.use_attention_pooling
-                else sequence_output[:, 0, :]
-            )
-            pooled_output = self.dropout(pooled_output)
-        except Exception as e:
-            raise RuntimeError(f"Error during pooling and dropout: {e}")
-
-        total_loss = 0
-        logits = []
-        losses = []
-
-        for task_id, (head, num_labels) in enumerate(
-            zip(self.classification_heads, self.num_labels_list)
-        ):
-            try:
-                task_logits = head(pooled_output)
-            except Exception as e:
-                raise RuntimeError(
-                    f"Error during forward pass of classification head {task_id}: {e}"
-                )
-
-            logits.append(task_logits)
-
-            if labels is not None:
-                try:
-                    loss_fct = nn.CrossEntropyLoss()
-                    task_loss = loss_fct(
-                        task_logits.view(-1, num_labels), labels[task_id].view(-1)
-                    )
-                    if self.use_task_weights:
-                        task_loss *= self.task_weights[task_id]
-                    total_loss += task_loss
-                    losses.append(task_loss.item())
-                except Exception as e:
-                    raise RuntimeError(
-                        f"Error during loss computation for task {task_id}: {e}"
-                    )
-
-        return total_loss, logits, losses if labels is not None else logits

geneformer/mtl/optuna_utils.py

@@ -1,27 +0,0 @@
-import optuna
-from optuna.integration import TensorBoardCallback
-
-
-def save_trial_callback(study, trial, trials_result_path):
-    with open(trials_result_path, "a") as f:
-        f.write(
-            f"Trial {trial.number}: Value (F1 Macro): {trial.value}, Params: {trial.params}\n"
-        )
-
-
-def create_optuna_study(objective, n_trials, trials_result_path, tensorboard_log_dir):
-    study = optuna.create_study(direction="maximize")
-
-    # init TensorBoard callback
-    tensorboard_callback = TensorBoardCallback(
-        dirname=tensorboard_log_dir, metric_name="F1 Macro"
-    )
-
-    # callback and TensorBoard callback
-    callbacks = [
-        lambda study, trial: save_trial_callback(study, trial, trials_result_path),
-        tensorboard_callback,
-    ]
-
-    study.optimize(objective, n_trials=n_trials, callbacks=callbacks)
-    return study

geneformer/mtl/train.py

@@ -1,380 +0,0 @@
-import os
-import random
-
-import numpy as np
-import pandas as pd
-import torch
-from torch.utils.tensorboard import SummaryWriter
-from tqdm import tqdm
-
-from .imports import *
-from .model import GeneformerMultiTask
-from .utils import calculate_task_specific_metrics, get_layer_freeze_range
-
-
-def set_seed(seed):
-    random.seed(seed)
-    np.random.seed(seed)
-    torch.manual_seed(seed)
-    torch.cuda.manual_seed_all(seed)
-    torch.backends.cudnn.deterministic = True
-    torch.backends.cudnn.benchmark = False
-
-
-def initialize_wandb(config):
-    if config.get("use_wandb", False):
-        import wandb
-
-        wandb.init(project=config["wandb_project"], config=config)
-        print("Weights & Biases (wandb) initialized and will be used for logging.")
-    else:
-        print(
-            "Weights & Biases (wandb) is not enabled. Logging will use other methods."
-        )
-
-
-def create_model(config, num_labels_list, device):
-    model = GeneformerMultiTask(
-        config["pretrained_path"],
-        num_labels_list,
-        dropout_rate=config["dropout_rate"],
-        use_task_weights=config["use_task_weights"],
-        task_weights=config["task_weights"],
-        max_layers_to_freeze=config["max_layers_to_freeze"],
-        use_attention_pooling=config["use_attention_pooling"],
-    )
-    if config["use_data_parallel"]:
-        model = nn.DataParallel(model)
-    return model.to(device)
-
-
-def setup_optimizer_and_scheduler(model, config, total_steps):
-    optimizer = AdamW(
-        model.parameters(),
-        lr=config["learning_rate"],
-        weight_decay=config["weight_decay"],
-    )
-    warmup_steps = int(config["warmup_ratio"] * total_steps)
-
-    if config["lr_scheduler_type"] == "linear":
-        scheduler = get_linear_schedule_with_warmup(
-            optimizer, num_warmup_steps=warmup_steps, num_training_steps=total_steps
-        )
-    elif config["lr_scheduler_type"] == "cosine":
-        scheduler = get_cosine_schedule_with_warmup(
-            optimizer,
-            num_warmup_steps=warmup_steps,
-            num_training_steps=total_steps,
-            num_cycles=0.5,
-        )
-
-    return optimizer, scheduler
-
-
-def train_epoch(
-    model, train_loader, optimizer, scheduler, device, config, writer, epoch
-):
-    model.train()
-    progress_bar = tqdm(train_loader, desc=f"Epoch {epoch+1}/{config['epochs']}")
-    for batch_idx, batch in enumerate(progress_bar):
-        optimizer.zero_grad()
-        input_ids = batch["input_ids"].to(device)
-        attention_mask = batch["attention_mask"].to(device)
-        labels = [
-            batch["labels"][task_name].to(device) for task_name in config["task_names"]
-        ]
-
-        loss, _, _ = model(input_ids, attention_mask, labels)
-        loss.backward()
-
-        if config["gradient_clipping"]:
-            torch.nn.utils.clip_grad_norm_(model.parameters(), config["max_grad_norm"])
-
-        optimizer.step()
-        scheduler.step()
-
-        writer.add_scalar(
-            "Training Loss", loss.item(), epoch * len(train_loader) + batch_idx
-        )
-        if config.get("use_wandb", False):
-            import wandb
-
-            wandb.log({"Training Loss": loss.item()})
-
-        # Update progress bar
-        progress_bar.set_postfix({"loss": f"{loss.item():.4f}"})
-
-    return loss.item()  # Return the last batch loss
-
-
-def validate_model(model, val_loader, device, config):
-    model.eval()
-    val_loss = 0.0
-    task_true_labels = {task_name: [] for task_name in config["task_names"]}
-    task_pred_labels = {task_name: [] for task_name in config["task_names"]}
-    task_pred_probs = {task_name: [] for task_name in config["task_names"]}
-
-    with torch.no_grad():
-        for batch in val_loader:
-            input_ids = batch["input_ids"].to(device)
-            attention_mask = batch["attention_mask"].to(device)
-            labels = [
-                batch["labels"][task_name].to(device)
-                for task_name in config["task_names"]
-            ]
-            loss, logits, _ = model(input_ids, attention_mask, labels)
-            val_loss += loss.item()
-
-            for sample_idx in range(len(batch["input_ids"])):
-                for i, task_name in enumerate(config["task_names"]):
-                    true_label = batch["labels"][task_name][sample_idx].item()
-                    pred_label = torch.argmax(logits[i][sample_idx], dim=-1).item()
-                    pred_prob = (
-                        torch.softmax(logits[i][sample_idx], dim=-1).cpu().numpy()
-                    )
-                    task_true_labels[task_name].append(true_label)
-                    task_pred_labels[task_name].append(pred_label)
-                    task_pred_probs[task_name].append(pred_prob)
-
-    val_loss /= len(val_loader)
-    return val_loss, task_true_labels, task_pred_labels, task_pred_probs
-
-
-def log_metrics(task_metrics, val_loss, config, writer, epochs):
-    for task_name, metrics in task_metrics.items():
-        print(
-            f"{task_name} - Validation F1 Macro: {metrics['f1']:.4f}, Validation Accuracy: {metrics['accuracy']:.4f}"
-        )
-        if config.get("use_wandb", False):
-            import wandb
-
-            wandb.log(
-                {
-                    f"{task_name} Validation F1 Macro": metrics["f1"],
-                    f"{task_name} Validation Accuracy": metrics["accuracy"],
-                }
-            )
-
-    writer.add_scalar("Validation Loss", val_loss, epochs)
-    for task_name, metrics in task_metrics.items():
-        writer.add_scalar(f"{task_name} - Validation F1 Macro", metrics["f1"], epochs)
-        writer.add_scalar(
-            f"{task_name} - Validation Accuracy", metrics["accuracy"], epochs
-        )
-
-
-def save_validation_predictions(
-    val_cell_id_mapping,
-    task_true_labels,
-    task_pred_labels,
-    task_pred_probs,
-    config,
-    trial_number=None,
-):
-    if trial_number is not None:
-        trial_results_dir = os.path.join(config["results_dir"], f"trial_{trial_number}")
-        os.makedirs(trial_results_dir, exist_ok=True)
-        val_preds_file = os.path.join(trial_results_dir, "val_preds.csv")
-    else:
-        val_preds_file = os.path.join(config["results_dir"], "manual_run_val_preds.csv")
-
-    rows = []
-    for sample_idx in range(len(val_cell_id_mapping)):
-        row = {"Cell ID": val_cell_id_mapping[sample_idx]}
-        for task_name in config["task_names"]:
-            row[f"{task_name} True"] = task_true_labels[task_name][sample_idx]
-            row[f"{task_name} Pred"] = task_pred_labels[task_name][sample_idx]
-            row[f"{task_name} Probabilities"] = ",".join(
-                map(str, task_pred_probs[task_name][sample_idx])
-            )
-        rows.append(row)
-
-    df = pd.DataFrame(rows)
-    df.to_csv(val_preds_file, index=False)
-    print(f"Validation predictions saved to {val_preds_file}")
-
-
-def train_model(
-    config,
-    device,
-    train_loader,
-    val_loader,
-    train_cell_id_mapping,
-    val_cell_id_mapping,
-    num_labels_list,
-):
-    set_seed(config["seed"])
-    initialize_wandb(config)
-
-    model = create_model(config, num_labels_list, device)
-    total_steps = len(train_loader) * config["epochs"]
-    optimizer, scheduler = setup_optimizer_and_scheduler(model, config, total_steps)
-
-    log_dir = os.path.join(config["tensorboard_log_dir"], "manual_run")
-    writer = SummaryWriter(log_dir=log_dir)
-
-    epoch_progress = tqdm(range(config["epochs"]), desc="Training Progress")
-    for epoch in epoch_progress:
-        last_loss = train_epoch(
-            model, train_loader, optimizer, scheduler, device, config, writer, epoch
-        )
-        epoch_progress.set_postfix({"last_loss": f"{last_loss:.4f}"})
-
-    val_loss, task_true_labels, task_pred_labels, task_pred_probs = validate_model(
-        model, val_loader, device, config
-    )
-    task_metrics = calculate_task_specific_metrics(task_true_labels, task_pred_labels)
-
-    log_metrics(task_metrics, val_loss, config, writer, config["epochs"])
-    writer.close()
-
-    save_validation_predictions(
-        val_cell_id_mapping, task_true_labels, task_pred_labels, task_pred_probs, config
-    )
-
-    if config.get("use_wandb", False):
-        import wandb
-
-        wandb.finish()
-
-    print(f"\nFinal Validation Loss: {val_loss:.4f}")
-    return val_loss, model  # Return both the validation loss and the trained model
-
-
-def objective(
-    trial,
-    train_loader,
-    val_loader,
-    train_cell_id_mapping,
-    val_cell_id_mapping,
-    num_labels_list,
-    config,
-    device,
-):
-    set_seed(config["seed"])  # Set the seed before each trial
-    initialize_wandb(config)
-
-    # Hyperparameters
-    config["learning_rate"] = trial.suggest_float(
-        "learning_rate",
-        config["hyperparameters"]["learning_rate"]["low"],
-        config["hyperparameters"]["learning_rate"]["high"],
-        log=config["hyperparameters"]["learning_rate"]["log"],
-    )
-    config["warmup_ratio"] = trial.suggest_float(
-        "warmup_ratio",
-        config["hyperparameters"]["warmup_ratio"]["low"],
-        config["hyperparameters"]["warmup_ratio"]["high"],
-    )
-    config["weight_decay"] = trial.suggest_float(
-        "weight_decay",
-        config["hyperparameters"]["weight_decay"]["low"],
-        config["hyperparameters"]["weight_decay"]["high"],
-    )
-    config["dropout_rate"] = trial.suggest_float(
-        "dropout_rate",
-        config["hyperparameters"]["dropout_rate"]["low"],
-        config["hyperparameters"]["dropout_rate"]["high"],
-    )
-    config["lr_scheduler_type"] = trial.suggest_categorical(
-        "lr_scheduler_type", config["hyperparameters"]["lr_scheduler_type"]["choices"]
-    )
-    config["use_attention_pooling"] = trial.suggest_categorical(
-        "use_attention_pooling", [False]
-    )
-
-    if config["use_task_weights"]:
-        config["task_weights"] = [
-            trial.suggest_float(
-                f"task_weight_{i}",
-                config["hyperparameters"]["task_weights"]["low"],
-                config["hyperparameters"]["task_weights"]["high"],
-            )
-            for i in range(len(num_labels_list))
-        ]
-        weight_sum = sum(config["task_weights"])
-        config["task_weights"] = [
-            weight / weight_sum for weight in config["task_weights"]
-        ]
-    else:
-        config["task_weights"] = None
-
-    # Dynamic range for max_layers_to_freeze
-    freeze_range = get_layer_freeze_range(config["pretrained_path"])
-    config["max_layers_to_freeze"] = trial.suggest_int(
-        "max_layers_to_freeze",
-        freeze_range["min"],
-        freeze_range["max"]
-    )
-
-    model = create_model(config, num_labels_list, device)
-    total_steps = len(train_loader) * config["epochs"]
-    optimizer, scheduler = setup_optimizer_and_scheduler(model, config, total_steps)
-
-    log_dir = os.path.join(config["tensorboard_log_dir"], f"trial_{trial.number}")
-    writer = SummaryWriter(log_dir=log_dir)
-
-    for epoch in range(config["epochs"]):
-        train_epoch(
-            model, train_loader, optimizer, scheduler, device, config, writer, epoch
-        )
-
-    val_loss, task_true_labels, task_pred_labels, task_pred_probs = validate_model(
-        model, val_loader, device, config
-    )
-    task_metrics = calculate_task_specific_metrics(task_true_labels, task_pred_labels)
-
-    log_metrics(task_metrics, val_loss, config, writer, config["epochs"])
-    writer.close()
-
-    save_validation_predictions(
-        val_cell_id_mapping,
-        task_true_labels,
-        task_pred_labels,
-        task_pred_probs,
-        config,
-        trial.number,
-    )
-
-    trial.set_user_attr("model_state_dict", model.state_dict())
-    trial.set_user_attr("task_weights", config["task_weights"])
-
-    trial.report(val_loss, config["epochs"])
-
-    if trial.should_prune():
-        raise optuna.TrialPruned()
-
-    if config.get("use_wandb", False):
-        import wandb
-
-        wandb.log(
-            {
-                "trial_number": trial.number,
-                "val_loss": val_loss,
-                **{
-                    f"{task_name}_f1": metrics["f1"]
-                    for task_name, metrics in task_metrics.items()
-                },
-                **{
-                    f"{task_name}_accuracy": metrics["accuracy"]
-                    for task_name, metrics in task_metrics.items()
-                },
-                **{
-                    k: v
-                    for k, v in config.items()
-                    if k
-                    in [
-                        "learning_rate",
-                        "warmup_ratio",
-                        "weight_decay",
-                        "dropout_rate",
-                        "lr_scheduler_type",
-                        "use_attention_pooling",
-                        "max_layers_to_freeze",
-                    ]
-                },
-            }
-        )
-        wandb.finish()
-
-    return val_loss

geneformer/mtl/train_utils.py

@@ -1,161 +0,0 @@
-import random
-
-from .data import get_data_loader, preload_and_process_data
-from .imports import *
-from .model import GeneformerMultiTask
-from .train import objective, train_model
-from .utils import save_model
-
-
-def set_seed(seed):
-    random.seed(seed)
-    np.random.seed(seed)
-    torch.manual_seed(seed)
-    torch.cuda.manual_seed_all(seed)
-    torch.backends.cudnn.deterministic = True
-    torch.backends.cudnn.benchmark = False
-
-
-def run_manual_tuning(config):
-    # Set seed for reproducibility
-    set_seed(config["seed"])
-
-    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    (
-        train_dataset,
-        train_cell_id_mapping,
-        val_dataset,
-        val_cell_id_mapping,
-        num_labels_list,
-    ) = preload_and_process_data(config)
-    train_loader = get_data_loader(train_dataset, config["batch_size"])
-    val_loader = get_data_loader(val_dataset, config["batch_size"])
-
-    # Print the manual hyperparameters being used
-    print("\nManual hyperparameters being used:")
-    for key, value in config["manual_hyperparameters"].items():
-        print(f"{key}: {value}")
-    print()  # Add an empty line for better readability
-
-    # Use the manual hyperparameters
-    for key, value in config["manual_hyperparameters"].items():
-        config[key] = value
-
-    # Train the model
-    val_loss, trained_model = train_model(
-        config,
-        device,
-        train_loader,
-        val_loader,
-        train_cell_id_mapping,
-        val_cell_id_mapping,
-        num_labels_list,
-    )
-
-    print(f"\nValidation loss with manual hyperparameters: {val_loss}")
-
-    # Save the trained model
-    model_save_directory = os.path.join(
-        config["model_save_path"], "GeneformerMultiTask"
-    )
-    save_model(trained_model, model_save_directory)
-
-    # Save the hyperparameters
-    hyperparams_to_save = {
-        **config["manual_hyperparameters"],
-        "dropout_rate": config["dropout_rate"],
-        "use_task_weights": config["use_task_weights"],
-        "task_weights": config["task_weights"],
-        "max_layers_to_freeze": config["max_layers_to_freeze"],
-        "use_attention_pooling": config["use_attention_pooling"],
-    }
-    hyperparams_path = os.path.join(model_save_directory, "hyperparameters.json")
-    with open(hyperparams_path, "w") as f:
-        json.dump(hyperparams_to_save, f)
-    print(f"Manual hyperparameters saved to {hyperparams_path}")
-
-    return val_loss
-
-
-def run_optuna_study(config):
-    # Set seed for reproducibility
-    set_seed(config["seed"])
-
-    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    (
-        train_dataset,
-        train_cell_id_mapping,
-        val_dataset,
-        val_cell_id_mapping,
-        num_labels_list,
-    ) = preload_and_process_data(config)
-    train_loader = get_data_loader(train_dataset, config["batch_size"])
-    val_loader = get_data_loader(val_dataset, config["batch_size"])
-
-    if config["use_manual_hyperparameters"]:
-        train_model(
-            config,
-            device,
-            train_loader,
-            val_loader,
-            train_cell_id_mapping,
-            val_cell_id_mapping,
-            num_labels_list,
-        )
-    else:
-        objective_with_config_and_data = functools.partial(
-            objective,
-            train_loader=train_loader,
-            val_loader=val_loader,
-            train_cell_id_mapping=train_cell_id_mapping,
-            val_cell_id_mapping=val_cell_id_mapping,
-            num_labels_list=num_labels_list,
-            config=config,
-            device=device,
-        )
-
-        study = optuna.create_study(
-            direction="minimize",  # Minimize validation loss
-            study_name=config["study_name"],
-            # storage=config["storage"],
-            load_if_exists=True,
-        )
-
-        study.optimize(objective_with_config_and_data, n_trials=config["n_trials"])
-
-        # After finding the best trial
-        best_params = study.best_trial.params
-        best_task_weights = study.best_trial.user_attrs["task_weights"]
-        print("Saving the best model and its hyperparameters...")
-
-        # Saving model as before
-        best_model = GeneformerMultiTask(
-            config["pretrained_path"],
-            num_labels_list,
-            dropout_rate=best_params["dropout_rate"],
-            use_task_weights=config["use_task_weights"],
-            task_weights=best_task_weights,
-        )
-
-        # Get the best model state dictionary
-        best_model_state_dict = study.best_trial.user_attrs["model_state_dict"]
-
-        # Remove the "module." prefix from the state dictionary keys if present
-        best_model_state_dict = {
-            k.replace("module.", ""): v for k, v in best_model_state_dict.items()
-        }
-
-        # Load the modified state dictionary into the model, skipping unexpected keys
-        best_model.load_state_dict(best_model_state_dict, strict=False)
-
-        model_save_directory = os.path.join(
-            config["model_save_path"], "GeneformerMultiTask"
-        )
-        save_model(best_model, model_save_directory)
-
-        # Additionally, save the best hyperparameters and task weights
-        hyperparams_path = os.path.join(model_save_directory, "hyperparameters.json")
-
-        with open(hyperparams_path, "w") as f:
-            json.dump({**best_params, "task_weights": best_task_weights}, f)
-        print(f"Best hyperparameters and task weights saved to {hyperparams_path}")

geneformer/mtl/utils.py

@@ -1,129 +0,0 @@
-import os
-import shutil
-
-from sklearn.metrics import accuracy_score, f1_score
-from sklearn.preprocessing import LabelEncoder
-from transformers import AutoConfig, BertConfig, BertModel
-
-from .imports import *
-
-
-def save_model(model, model_save_directory):
-    if not os.path.exists(model_save_directory):
-        os.makedirs(model_save_directory)
-
-    # Get the state dict
-    if isinstance(model, nn.DataParallel):
-        model_state_dict = (
-            model.module.state_dict()
-        )  # Use model.module to access the underlying model
-    else:
-        model_state_dict = model.state_dict()
-
-    # Remove the "module." prefix from the keys if present
-    model_state_dict = {
-        k.replace("module.", ""): v for k, v in model_state_dict.items()
-    }
-
-    model_save_path = os.path.join(model_save_directory, "pytorch_model.bin")
-    torch.save(model_state_dict, model_save_path)
-
-    # Save the model configuration
-    if isinstance(model, nn.DataParallel):
-        model.module.config.to_json_file(
-            os.path.join(model_save_directory, "config.json")
-        )
-    else:
-        model.config.to_json_file(os.path.join(model_save_directory, "config.json"))
-
-    print(f"Model and configuration saved to {model_save_directory}")
-
-
-def calculate_task_specific_metrics(task_true_labels, task_pred_labels):
-    task_metrics = {}
-    for task_name in task_true_labels.keys():
-        true_labels = task_true_labels[task_name]
-        pred_labels = task_pred_labels[task_name]
-        f1 = f1_score(true_labels, pred_labels, average="macro")
-        accuracy = accuracy_score(true_labels, pred_labels)
-        task_metrics[task_name] = {"f1": f1, "accuracy": accuracy}
-    return task_metrics
-
-
-def calculate_combined_f1(combined_labels, combined_preds):
-    # Initialize the LabelEncoder
-    le = LabelEncoder()
-
-    # Fit and transform combined labels and predictions to numerical values
-    le.fit(combined_labels + combined_preds)
-    encoded_true_labels = le.transform(combined_labels)
-    encoded_pred_labels = le.transform(combined_preds)
-
-    # Print out the mapping for sanity check
-    print("\nLabel Encoder Mapping:")
-    for index, class_label in enumerate(le.classes_):
-        print(f"'{class_label}': {index}")
-
-    # Calculate accuracy
-    accuracy = accuracy_score(encoded_true_labels, encoded_pred_labels)
-
-    # Calculate F1 Macro score
-    f1 = f1_score(encoded_true_labels, encoded_pred_labels, average="macro")
-
-    return f1, accuracy
-
-
-# def save_model_without_heads(original_model_save_directory):
-#     # Create a new directory for the model without heads
-#     new_model_save_directory = original_model_save_directory + "_No_Heads"
-#     if not os.path.exists(new_model_save_directory):
-#         os.makedirs(new_model_save_directory)
-
-#     # Load the model state dictionary
-#     model_state_dict = torch.load(
-#         os.path.join(original_model_save_directory, "pytorch_model.bin")
-#     )
-
-#     # Initialize a new BERT model without the classification heads
-#     config = BertConfig.from_pretrained(
-#         os.path.join(original_model_save_directory, "config.json")
-#     )
-#     model_without_heads = BertModel(config)
-
-#     # Filter the state dict to exclude classification heads
-#     model_without_heads_state_dict = {
-#         k: v
-#         for k, v in model_state_dict.items()
-#         if not k.startswith("classification_heads")
-#     }
-
-#     # Load the filtered state dict into the model
-#     model_without_heads.load_state_dict(model_without_heads_state_dict, strict=False)
-
-#     # Save the model without heads
-#     model_save_path = os.path.join(new_model_save_directory, "pytorch_model.bin")
-#     torch.save(model_without_heads.state_dict(), model_save_path)
-
-#     # Copy the configuration file
-#     shutil.copy(
-#         os.path.join(original_model_save_directory, "config.json"),
-#         new_model_save_directory,
-#     )
-
-#     print(f"Model without classification heads saved to {new_model_save_directory}")
-
-
-def get_layer_freeze_range(pretrained_path):
-    """
-    Dynamically determines the number of layers to freeze based on the model depth from its configuration.
-    Args:
-        pretrained_path (str): Path to the pretrained model directory or model identifier.
-    Returns:
-        dict: A dictionary with 'min' and 'max' keys indicating the range of layers to freeze.
-    """
-    if pretrained_path:
-        config = AutoConfig.from_pretrained(pretrained_path)
-        total_layers = config.num_hidden_layers
-        return {"min": 0, "max": total_layers - 1}
-    else:
-        return {"min": 0, "max": 0}

geneformer/mtl_classifier.py

@@ -1,363 +0,0 @@
-"""
-Geneformer multi-task cell classifier.
-
-**Input data:**
-
-| Single-cell transcriptomes as Geneformer rank value encodings with cell state labels for each task in Geneformer .dataset format (generated from single-cell RNAseq data by tokenizer.py). Must contain "unique_cell_id" column for logging.
-
-**Usage:**
-
-.. code-block :: python
-
-    >>> from geneformer import MTLClassifier
-    >>> mc = MTLClassifier(task_columns = ["task1", "task2"],
-    ...                 study_name = "mtl",
-    ...                 pretrained_path = "/path/pretrained/model",
-    ...                 train_path = "/path/train/set",
-    ...                 val_path = "/path/eval/set",
-    ...                 test_path = "/path/test/set",
-    ...                 model_save_path = "/results/directory/save_path",
-    ...                 trials_result_path = "/results/directory/results.txt",
-    ...                 results_dir = "/results/directory",
-    ...                 tensorboard_log_dir = "/results/tblogdir",
-    ...                 hyperparameters = hyperparameters)
-    >>> mc.run_optuna_study()
-    >>> mc.load_and_evaluate_test_model()
-    >>> mc.save_model_without_heads()
-"""
-
-import logging
-import os
-
-from .mtl import eval_utils, train_utils, utils
-
-logger = logging.getLogger(__name__)
-
-
-class MTLClassifier:
-    valid_option_dict = {
-        "task_columns": {list},
-        "train_path": {None, str},
-        "val_path": {None, str},
-        "test_path": {None, str},
-        "pretrained_path": {None, str},
-        "model_save_path": {None, str},
-        "results_dir": {None, str},
-        "batch_size": {None, int},
-        "n_trials": {None, int},
-        "study_name": {None, str},
-        "max_layers_to_freeze": {None, dict},
-        "epochs": {None, int},
-        "tensorboard_log_dir": {None, str},
-        "use_data_parallel": {None, bool},
-        "use_attention_pooling": {None, bool},
-        "use_task_weights": {None, bool},
-        "hyperparameters": {None, dict},
-        "manual_hyperparameters": {None, dict},
-        "use_manual_hyperparameters": {None, bool},
-        "use_wandb": {None, bool},
-        "wandb_project": {None, str},
-        "gradient_clipping": {None, bool},
-        "max_grad_norm": {None, int, float},
-        "seed": {None, int},
-        "trials_result_path": {None, str},
-    }
-
-    def __init__(
-        self,
-        task_columns=None,
-        train_path=None,
-        val_path=None,
-        test_path=None,
-        pretrained_path=None,
-        model_save_path=None,
-        results_dir=None,
-        trials_result_path=None,
-        batch_size=4,
-        n_trials=15,
-        study_name="mtl",
-        max_layers_to_freeze=None,
-        epochs=1,
-        tensorboard_log_dir="/results/tblogdir",
-        use_data_parallel=False,
-        use_attention_pooling=True,
-        use_task_weights=True,
-        hyperparameters=None,  # Default is None
-        manual_hyperparameters=None,  # Default is None
-        use_manual_hyperparameters=False,  # Default is False
-        use_wandb=False,
-        wandb_project=None,
-        gradient_clipping=False,
-        max_grad_norm=None,
-        seed=42,  # Default seed value
-    ):
-        """
-        Initialize Geneformer multi-task classifier.
-        
-        **Parameters:**
-        
-        task_columns : list
-            | List of tasks for cell state classification
-            | Input data columns are labeled with corresponding task names
-        study_name : None, str
-            | Study name for labeling output files
-        pretrained_path : None, str
-            | Path to pretrained model
-        train_path : None, str
-            | Path to training dataset with task columns and "unique_cell_id" column
-        val_path : None, str
-            | Path to validation dataset with task columns and "unique_cell_id" column
-        test_path : None, str
-            | Path to test dataset with task columns and "unique_cell_id" column
-        model_save_path : None, str
-            | Path to directory to save output model (either full model or model without heads)
-        trials_result_path : None, str
-            | Path to directory to save hyperparameter tuning trial results
-        results_dir : None, str
-            | Path to directory to save results
-        tensorboard_log_dir : None, str
-            | Path to directory for Tensorboard logging results
-        use_data_parallel : None, bool
-            | Whether to use data parallelization
-        use_attention_pooling : None, bool
-            | Whether to use attention pooling
-        use_task_weights : None, bool
-            | Whether to use task weights
-        batch_size : None, int
-            | Batch size to use
-        n_trials : None, int
-            | Number of trials for hyperparameter tuning
-        epochs : None, int
-            | Number of epochs for training
-        max_layers_to_freeze : None, dict
-            | Dictionary with keys "min" and "max" indicating the min and max layers to freeze from fine-tuning (int)
-            | 0: no layers will be frozen; 2: first two layers will be frozen; etc.
-        hyperparameters : None, dict
-            | Dictionary of categorical max and min for each hyperparameter for tuning
-            | For example:
-            | {"learning_rate": {"type":"float", "low":"1e-5", "high":"1e-3", "log":True}, "task_weights": {...}, ...}
-        manual_hyperparameters : None, dict
-            | Dictionary of manually set value for each hyperparameter
-            | For example:
-            | {"learning_rate": 0.001, "task_weights": [1, 1], ...}
-        use_manual_hyperparameters : None, bool
-            | Whether to use manually set hyperparameters
-        use_wandb : None, bool
-            | Whether to use Weights & Biases for logging
-        wandb_project : None, str
-            | Weights & Biases project name
-        gradient_clipping : None, bool
-            | Whether to use gradient clipping
-        max_grad_norm : None, int, float
-            | Maximum norm for gradient clipping
-        seed : None, int
-            | Random seed
-        """
-
-        self.task_columns = task_columns
-        self.train_path = train_path
-        self.val_path = val_path
-        self.test_path = test_path
-        self.pretrained_path = pretrained_path
-        self.model_save_path = model_save_path
-        self.results_dir = results_dir
-        self.trials_result_path = trials_result_path
-        self.batch_size = batch_size
-        self.n_trials = n_trials
-        self.study_name = study_name
-
-        if max_layers_to_freeze is None:
-            # Dynamically determine the range of layers to freeze
-            layer_freeze_range = utils.get_layer_freeze_range(pretrained_path)
-            self.max_layers_to_freeze = {"min": 1, "max": layer_freeze_range["max"]}
-        else:
-            self.max_layers_to_freeze = max_layers_to_freeze
-
-        self.epochs = epochs
-        self.tensorboard_log_dir = tensorboard_log_dir
-        self.use_data_parallel = use_data_parallel
-        self.use_attention_pooling = use_attention_pooling
-        self.use_task_weights = use_task_weights
-        self.hyperparameters = (
-            hyperparameters
-            if hyperparameters is not None
-            else {
-                "learning_rate": {
-                    "type": "float",
-                    "low": 1e-5,
-                    "high": 1e-3,
-                    "log": True,
-                },
-                "warmup_ratio": {"type": "float", "low": 0.005, "high": 0.01},
-                "weight_decay": {"type": "float", "low": 0.01, "high": 0.1},
-                "dropout_rate": {"type": "float", "low": 0.0, "high": 0.7},
-                "lr_scheduler_type": {"type": "categorical", "choices": ["cosine"]},
-                "task_weights": {"type": "float", "low": 0.1, "high": 2.0},
-            }
-        )
-        self.manual_hyperparameters = (
-            manual_hyperparameters
-            if manual_hyperparameters is not None
-            else {
-                "learning_rate": 0.001,
-                "warmup_ratio": 0.01,
-                "weight_decay": 0.1,
-                "dropout_rate": 0.1,
-                "lr_scheduler_type": "cosine",
-                "use_attention_pooling": False,
-                "task_weights": [1, 1],
-                "max_layers_to_freeze": 2,
-            }
-        )
-        self.use_manual_hyperparameters = use_manual_hyperparameters
-        self.use_wandb = use_wandb
-        self.wandb_project = wandb_project
-        self.gradient_clipping = gradient_clipping
-        self.max_grad_norm = max_grad_norm
-        self.seed = seed
-
-        if self.use_manual_hyperparameters:
-            logger.warning(
-                "Hyperparameter tuning is highly recommended for optimal results."
-            )
-
-        self.validate_options()
-
-        # set up output directories
-        if self.results_dir is not None:
-            self.trials_results_path = f"{self.results_dir}/results.txt".replace(
-                "//", "/"
-            )
-
-        for output_dir in [self.model_save_path, self.results_dir]:
-            if not os.path.exists(output_dir):
-                os.makedirs(output_dir)
-
-        self.config = {
-            key: value
-            for key, value in self.__dict__.items()
-            if key in self.valid_option_dict
-        }
-
-    def validate_options(self):
-        # confirm arguments are within valid options and compatible with each other
-        for attr_name, valid_options in self.valid_option_dict.items():
-            attr_value = self.__dict__[attr_name]
-            if not isinstance(attr_value, (list, dict)):
-                if attr_value in valid_options:
-                    continue
-            valid_type = False
-            for option in valid_options:
-                if (option in [int, float, list, dict, bool, str]) and isinstance(
-                    attr_value, option
-                ):
-                    valid_type = True
-                    break
-            if valid_type:
-                continue
-            logger.error(
-                f"Invalid option for {attr_name}. "
-                f"Valid options for {attr_name}: {valid_options}"
-            )
-            raise ValueError(
-                f"Invalid option for {attr_name}. Valid options for {attr_name}: {valid_options}"
-            )
-
-    def run_manual_tuning(self):
-        """
-        Manual hyperparameter tuning and multi-task fine-tuning of pretrained model.
-        """
-        required_variable_names = [
-            "train_path",
-            "val_path",
-            "pretrained_path",
-            "model_save_path",
-            "results_dir",
-        ]
-        required_variables = [
-            self.train_path,
-            self.val_path,
-            self.pretrained_path,
-            self.model_save_path,
-            self.results_dir,
-        ]
-        req_var_dict = dict(zip(required_variable_names, required_variables))
-        self.validate_additional_options(req_var_dict)
-
-        if not self.use_manual_hyperparameters:
-            raise ValueError(
-                "Manual hyperparameters are not enabled. Set use_manual_hyperparameters to True."
-            )
-
-        # Ensure manual_hyperparameters are set in the config
-        self.config["manual_hyperparameters"] = self.manual_hyperparameters
-        self.config["use_manual_hyperparameters"] = True
-
-        train_utils.run_manual_tuning(self.config)
-
-    def validate_additional_options(self, req_var_dict):
-        missing_variable = False
-        for variable_name, variable in req_var_dict.items():
-            if variable is None:
-                logger.warning(
-                    f"Please provide value to MTLClassifier for required variable {variable_name}"
-                )
-                missing_variable = True
-        if missing_variable is True:
-            raise ValueError("Missing required variables for MTLClassifier")
-
-    def run_optuna_study(
-        self,
-    ):
-        """
-        Hyperparameter optimization and/or multi-task fine-tuning of pretrained model.
-        """
-
-        required_variable_names = [
-            "train_path",
-            "val_path",
-            "pretrained_path",
-            "model_save_path",
-            "results_dir",
-        ]
-        required_variables = [
-            self.train_path,
-            self.val_path,
-            self.pretrained_path,
-            self.model_save_path,
-            self.results_dir,
-        ]
-        req_var_dict = dict(zip(required_variable_names, required_variables))
-        self.validate_additional_options(req_var_dict)
-
-        train_utils.run_optuna_study(self.config)
-
-    def load_and_evaluate_test_model(
-        self,
-    ):
-        """
-        Loads previously fine-tuned multi-task model and evaluates on test data.
-        """
-
-        required_variable_names = ["test_path", "model_save_path", "results_dir"]
-        required_variables = [self.test_path, self.model_save_path, self.results_dir]
-        req_var_dict = dict(zip(required_variable_names, required_variables))
-        self.validate_additional_options(req_var_dict)
-
-        eval_utils.load_and_evaluate_test_model(self.config)
-
-    # def save_model_without_heads(
-    #     self,
-    # ):
-    #     """
-    #     Save previously fine-tuned multi-task model without classification heads.
-    #     """
-
-    #     required_variable_names = ["model_save_path"]
-    #     required_variables = [self.model_save_path]
-    #     req_var_dict = dict(zip(required_variable_names, required_variables))
-    #     self.validate_additional_options(req_var_dict)
-
-    #     utils.save_model_without_heads(
-    #         os.path.join(self.model_save_path, "GeneformerMultiTask")
-    #     )

geneformer/perturber_utils.py

@@ -1,919 +0,0 @@
-import itertools as it
-import logging
-import pickle
-from collections import defaultdict
-from pathlib import Path
-from typing import List
-
-import numpy as np
-import pandas as pd
-import torch
-from datasets import Dataset, load_from_disk
-from peft import LoraConfig, get_peft_model
-from transformers import (
-    BertForMaskedLM,
-    BertForSequenceClassification,
-    BertForTokenClassification,
-    BitsAndBytesConfig,
-)
-
-GENE_MEDIAN_FILE = Path(__file__).parent / "gene_median_dictionary.pkl"
-TOKEN_DICTIONARY_FILE = Path(__file__).parent / "token_dictionary.pkl"
-ENSEMBL_DICTIONARY_FILE = Path(__file__).parent / "gene_name_id_dict.pkl"
-
-
-logger = logging.getLogger(__name__)
-
-
-# load data and filter by defined criteria
-def load_and_filter(filter_data, nproc, input_data_file):
-    data = load_from_disk(input_data_file)
-    if filter_data is not None:
-        data = filter_by_dict(data, filter_data, nproc)
-    return data
-
-
-def filter_by_dict(data, filter_data, nproc):
-    for key, value in filter_data.items():
-
-        def filter_data_by_criteria(example):
-            return example[key] in value
-
-        data = data.filter(filter_data_by_criteria, num_proc=nproc)
-    if len(data) == 0:
-        logger.error("No cells remain after filtering. Check filtering criteria.")
-        raise
-    return data
-
-
-def filter_data_by_tokens(filtered_input_data, tokens, nproc):
-    def if_has_tokens(example):
-        return len(set(example["input_ids"]).intersection(tokens)) == len(tokens)
-
-    filtered_input_data = filtered_input_data.filter(if_has_tokens, num_proc=nproc)
-    return filtered_input_data
-
-
-def logging_filtered_data_len(filtered_input_data, filtered_tokens_categ):
-    if len(filtered_input_data) == 0:
-        logger.error(f"No cells in dataset contain {filtered_tokens_categ}.")
-        raise
-    else:
-        logger.info(f"# cells with {filtered_tokens_categ}: {len(filtered_input_data)}")
-
-
-def filter_data_by_tokens_and_log(
-    filtered_input_data, tokens, nproc, filtered_tokens_categ
-):
-    # filter for cells with anchor gene
-    filtered_input_data = filter_data_by_tokens(filtered_input_data, tokens, nproc)
-    # logging length of filtered data
-    logging_filtered_data_len(filtered_input_data, filtered_tokens_categ)
-
-    return filtered_input_data
-
-
-def filter_data_by_start_state(filtered_input_data, cell_states_to_model, nproc):
-    # confirm that start state is valid to prevent futile filtering
-    state_key = cell_states_to_model["state_key"]
-    state_values = filtered_input_data[state_key]
-    start_state = cell_states_to_model["start_state"]
-    if start_state not in state_values:
-        logger.error(
-            f"Start state {start_state} is not present "
-            f"in the dataset's {state_key} attribute."
-        )
-        raise
-
-    # filter for start state cells
-    def filter_for_origin(example):
-        return example[state_key] in [start_state]
-
-    filtered_input_data = filtered_input_data.filter(filter_for_origin, num_proc=nproc)
-    return filtered_input_data
-
-
-def slice_by_inds_to_perturb(filtered_input_data, cell_inds_to_perturb):
-    if cell_inds_to_perturb["start"] >= len(filtered_input_data):
-        logger.error(
-            "cell_inds_to_perturb['start'] is larger than the filtered dataset."
-        )
-        raise
-    if cell_inds_to_perturb["end"] > len(filtered_input_data):
-        logger.warning(
-            "cell_inds_to_perturb['end'] is larger than the filtered dataset. \
-                       Setting to the end of the filtered dataset."
-        )
-        cell_inds_to_perturb["end"] = len(filtered_input_data)
-    filtered_input_data = filtered_input_data.select(
-        [i for i in range(cell_inds_to_perturb["start"], cell_inds_to_perturb["end"])]
-    )
-    return filtered_input_data
-
-
-# load model to GPU
-def load_model(model_type, num_classes, model_directory, mode, quantize=False):
-    if model_type == "MTLCellClassifier-Quantized":
-        model_type = "MTLCellClassifier"
-        quantize = True
-
-    output_hidden_states = (mode == "eval")
-
-    # Quantization logic
-    if quantize:
-        if model_type == "MTLCellClassifier":
-            quantize_config = BitsAndBytesConfig(load_in_8bit=True)
-            peft_config = None
-        else:
-            quantize_config = BitsAndBytesConfig(
-                load_in_4bit=True,
-                bnb_4bit_use_double_quant=True,
-                bnb_4bit_quant_type="nf4",
-                bnb_4bit_compute_dtype=torch.bfloat16,
-            )
-            peft_config = LoraConfig(
-                lora_alpha=128,
-                lora_dropout=0.1,
-                r=64,
-                bias="none",
-                task_type="TokenClassification",
-            )
-    else:
-        quantize_config = None
-        peft_config = None
-
-    # Model class selection
-    model_classes = {
-        "Pretrained": BertForMaskedLM,
-        "GeneClassifier": BertForTokenClassification,
-        "CellClassifier": BertForSequenceClassification,
-        "MTLCellClassifier": BertForMaskedLM
-    }
-
-    model_class = model_classes.get(model_type)
-    if not model_class:
-        raise ValueError(f"Unknown model type: {model_type}")
-
-    # Model loading
-    model_args = {
-        "pretrained_model_name_or_path": model_directory,
-        "output_hidden_states": output_hidden_states,
-        "output_attentions": False,
-    }
-
-    if model_type != "Pretrained":
-        model_args["num_labels"] = num_classes
-
-    if quantize_config:
-        model_args["quantization_config"] = quantize_config
-
-    # Load the model
-    model = model_class.from_pretrained(**model_args)
-
-    if mode == "eval":
-        model.eval()
-
-    # Handle device placement and PEFT
-    if not quantize:
-        # Only move non-quantized models
-        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        model = model.to(device)
-    elif peft_config:
-        # Apply PEFT for quantized models (except MTLCellClassifier)
-        model.enable_input_require_grads()
-        model = get_peft_model(model, peft_config)
-
-    return model
-
-def quant_layers(model):
-    layer_nums = []
-    for name, parameter in model.named_parameters():
-        if "layer" in name:
-            layer_nums += [int(name.split("layer.")[1].split(".")[0])]
-    return int(max(layer_nums)) + 1
-
-
-def get_model_emb_dims(model):
-    return model.config.hidden_size
-
-
-def get_model_input_size(model):
-    return model.config.max_position_embeddings
-
-
-def flatten_list(megalist):
-    return [item for sublist in megalist for item in sublist]
-
-
-def measure_length(example):
-    example["length"] = len(example["input_ids"])
-    return example
-
-
-def downsample_and_sort(data, max_ncells):
-    num_cells = len(data)
-    # if max number of cells is defined, then shuffle and subsample to this max number
-    if max_ncells is not None:
-        if num_cells > max_ncells:
-            data = data.shuffle(seed=42)
-            num_cells = max_ncells
-    data_subset = data.select([i for i in range(num_cells)])
-    # sort dataset with largest cell first to encounter any memory errors earlier
-    data_sorted = data_subset.sort("length", reverse=True)
-    return data_sorted
-
-
-def get_possible_states(cell_states_to_model):
-    possible_states = []
-    for key in ["start_state", "goal_state"]:
-        possible_states += [cell_states_to_model[key]]
-    possible_states += cell_states_to_model.get("alt_states", [])
-    return possible_states
-
-
-def forward_pass_single_cell(model, example_cell, layer_to_quant):
-    example_cell.set_format(type="torch")
-    input_data = example_cell["input_ids"]
-    with torch.no_grad():
-        outputs = model(input_ids=input_data.to("cuda"))
-    emb = torch.squeeze(outputs.hidden_states[layer_to_quant])
-    del outputs
-    return emb
-
-
-def perturb_emb_by_index(emb, indices):
-    mask = torch.ones(emb.numel(), dtype=torch.bool)
-    mask[indices] = False
-    return emb[mask]
-
-
-def delete_indices(example):
-    indices = example["perturb_index"]
-    if any(isinstance(el, list) for el in indices):
-        indices = flatten_list(indices)
-    for index in sorted(indices, reverse=True):
-        del example["input_ids"][index]
-
-    example["length"] = len(example["input_ids"])
-    return example
-
-
-# for genes_to_perturb = "all" where only genes within cell are overexpressed
-def overexpress_indices(example):
-    indices = example["perturb_index"]
-    if any(isinstance(el, list) for el in indices):
-        indices = flatten_list(indices)
-    insert_pos = 0
-    for index in sorted(indices, reverse=False):
-        example["input_ids"].insert(insert_pos, example["input_ids"].pop(index))
-        insert_pos += 1
-    example["length"] = len(example["input_ids"])
-    return example
-
-
-# if CLS token present, move to 1st rather than 0th position
-def overexpress_indices_special(example):
-    indices = example["perturb_index"]
-    if any(isinstance(el, list) for el in indices):
-        indices = flatten_list(indices)
-    insert_pos = 1  # Insert starting after CLS token
-    for index in sorted(indices, reverse=False):
-        example["input_ids"].insert(insert_pos, example["input_ids"].pop(index))
-        insert_pos += 1
-    example["length"] = len(example["input_ids"])
-    return example
-
-
-# for genes_to_perturb = list of genes to overexpress that are not necessarily expressed in cell
-def overexpress_tokens(example, max_len, special_token):
-    # -100 indicates tokens to overexpress are not present in rank value encoding
-    if example["perturb_index"] != [-100]:
-        example = delete_indices(example)
-    if special_token:
-        [
-            example["input_ids"].insert(1, token)
-            for token in example["tokens_to_perturb"][::-1]
-        ]
-    else:
-        [
-            example["input_ids"].insert(0, token)
-            for token in example["tokens_to_perturb"][::-1]
-        ]
-
-    # truncate to max input size, must also truncate original emb to be comparable
-    if len(example["input_ids"]) > max_len:
-        if special_token:
-            example["input_ids"] = example["input_ids"][0 : max_len - 1] + [
-                example["input_ids"][-1]
-            ]
-        else:
-            example["input_ids"] = example["input_ids"][0:max_len]
-    example["length"] = len(example["input_ids"])
-    return example
-
-
-def calc_n_overflow(max_len, example_len, tokens_to_perturb, indices_to_perturb):
-    n_to_add = len(tokens_to_perturb) - len(indices_to_perturb)
-    n_overflow = example_len + n_to_add - max_len
-    return n_overflow
-
-
-def truncate_by_n_overflow(example):
-    new_max_len = example["length"] - example["n_overflow"]
-    example["input_ids"] = example["input_ids"][0:new_max_len]
-    example["length"] = len(example["input_ids"])
-    return example
-
-
-def truncate_by_n_overflow_special(example):
-    if example["n_overflow"] > 0:
-        new_max_len = example["length"] - example["n_overflow"]
-        example["input_ids"] = example["input_ids"][0 : new_max_len - 1] + [
-            example["input_ids"][-1]
-        ]
-        example["length"] = len(example["input_ids"])
-    return example
-
-
-def remove_indices_from_emb(emb, indices_to_remove, gene_dim):
-    # indices_to_remove is list of indices to remove
-    indices_to_keep = [
-        i for i in range(emb.size()[gene_dim]) if i not in indices_to_remove
-    ]
-    num_dims = emb.dim()
-    emb_slice = [
-        slice(None) if dim != gene_dim else indices_to_keep for dim in range(num_dims)
-    ]
-    sliced_emb = emb[emb_slice]
-    return sliced_emb
-
-
-def remove_indices_from_emb_batch(emb_batch, list_of_indices_to_remove, gene_dim):
-    output_batch_list = [
-        remove_indices_from_emb(emb_batch[i, :, :], idxes, gene_dim - 1)
-        for i, idxes in enumerate(list_of_indices_to_remove)
-    ]
-    # add padding given genes are sometimes added that are or are not in original cell
-    batch_max = max([emb.size()[gene_dim - 1] for emb in output_batch_list])
-    output_batch_list_padded = [
-        pad_xd_tensor(emb, 0.000, batch_max, gene_dim - 1) for emb in output_batch_list
-    ]
-    return torch.stack(output_batch_list_padded)
-
-
-# removes perturbed indices
-# need to handle the various cases where a set of genes is overexpressed
-def remove_perturbed_indices_set(
-    emb,
-    perturb_type: str,
-    indices_to_perturb: List[List],
-    tokens_to_perturb: List[List],
-    original_lengths: List[int],
-    input_ids=None,
-):
-    if perturb_type == "overexpress":
-        num_perturbed = len(tokens_to_perturb)
-        if num_perturbed == 1:
-            indices_to_perturb_orig = [
-                idx if idx != [-100] else [None] for idx in indices_to_perturb
-            ]
-            if all(v is [None] for v in indices_to_perturb_orig):
-                return emb
-        else:
-            indices_to_perturb_orig = []
-
-            for idx_list in indices_to_perturb:
-                indices_to_perturb_orig.append(
-                    [idx if idx != [-100] else [None] for idx in idx_list]
-                )
-
-    else:
-        indices_to_perturb_orig = indices_to_perturb
-
-    emb = remove_indices_from_emb_batch(emb, indices_to_perturb_orig, gene_dim=1)
-
-    return emb
-
-
-def make_perturbation_batch(
-    example_cell, perturb_type, tokens_to_perturb, anchor_token, combo_lvl, num_proc
-) -> tuple[Dataset, List[int]]:
-    if combo_lvl == 0 and tokens_to_perturb == "all":
-        if perturb_type in ["overexpress", "activate"]:
-            range_start = 1
-        elif perturb_type in ["delete", "inhibit"]:
-            range_start = 0
-        indices_to_perturb = [
-            [i] for i in range(range_start, example_cell["length"][0])
-        ]
-    # elif combo_lvl > 0 and anchor_token is None:
-    ## to implement
-    elif combo_lvl > 0 and (anchor_token is not None):
-        example_input_ids = example_cell["input_ids"][0]
-        anchor_index = example_input_ids.index(anchor_token[0])
-        indices_to_perturb = [
-            sorted([anchor_index, i]) if i != anchor_index else None
-            for i in range(example_cell["length"][0])
-        ]
-        indices_to_perturb = [item for item in indices_to_perturb if item is not None]
-    else:
-        example_input_ids = example_cell["input_ids"][0]
-        indices_to_perturb = [
-            [example_input_ids.index(token)] if token in example_input_ids else None
-            for token in tokens_to_perturb
-        ]
-        indices_to_perturb = [item for item in indices_to_perturb if item is not None]
-
-    # create all permutations of combo_lvl of modifiers from tokens_to_perturb
-    if combo_lvl > 0 and (anchor_token is None):
-        if tokens_to_perturb != "all":
-            if len(tokens_to_perturb) == combo_lvl + 1:
-                indices_to_perturb = [
-                    list(x) for x in it.combinations(indices_to_perturb, combo_lvl + 1)
-                ]
-        else:
-            all_indices = [[i] for i in range(example_cell["length"][0])]
-            all_indices = [
-                index for index in all_indices if index not in indices_to_perturb
-            ]
-            indices_to_perturb = [
-                [[j for i in indices_to_perturb for j in i], x] for x in all_indices
-            ]
-
-    length = len(indices_to_perturb)
-    perturbation_dataset = Dataset.from_dict(
-        {
-            "input_ids": example_cell["input_ids"] * length,
-            "perturb_index": indices_to_perturb,
-        }
-    )
-
-    if length < 400:
-        num_proc_i = 1
-    else:
-        num_proc_i = num_proc
-
-    if perturb_type == "delete":
-        perturbation_dataset = perturbation_dataset.map(
-            delete_indices, num_proc=num_proc_i
-        )
-    elif perturb_type == "overexpress":
-        perturbation_dataset = perturbation_dataset.map(
-            overexpress_indices, num_proc=num_proc_i
-        )
-
-    perturbation_dataset = perturbation_dataset.map(measure_length, num_proc=num_proc_i)
-
-    return perturbation_dataset, indices_to_perturb
-
-
-def make_perturbation_batch_special(
-    example_cell, perturb_type, tokens_to_perturb, anchor_token, combo_lvl, num_proc
-) -> tuple[Dataset, List[int]]:
-    if combo_lvl == 0 and tokens_to_perturb == "all":
-        if perturb_type in ["overexpress", "activate"]:
-            range_start = 1
-        elif perturb_type in ["delete", "inhibit"]:
-            range_start = 0
-        range_start += 1  # Starting after the CLS token
-        indices_to_perturb = [
-            [i]
-            for i in range(
-                range_start, example_cell["length"][0] - 1
-            )  # And excluding the EOS token
-        ]
-
-    # elif combo_lvl > 0 and anchor_token is None:
-    ## to implement
-    elif combo_lvl > 0 and (anchor_token is not None):
-        example_input_ids = example_cell["input_ids"][0]
-        anchor_index = example_input_ids.index(anchor_token[0])
-        indices_to_perturb = [
-            sorted([anchor_index, i]) if i != anchor_index else None
-            for i in range(
-                1, example_cell["length"][0] - 1
-            )  # Exclude CLS and EOS tokens
-        ]
-        indices_to_perturb = [item for item in indices_to_perturb if item is not None]
-    else:
-        example_input_ids = example_cell["input_ids"][0]
-        indices_to_perturb = [
-            [example_input_ids.index(token)] if token in example_input_ids else None
-            for token in tokens_to_perturb
-        ]
-        indices_to_perturb = [item for item in indices_to_perturb if item is not None]
-
-    # create all permutations of combo_lvl of modifiers from tokens_to_perturb
-    if combo_lvl > 0 and (anchor_token is None):
-        if tokens_to_perturb != "all":
-            if len(tokens_to_perturb) == combo_lvl + 1:
-                indices_to_perturb = [
-                    list(x) for x in it.combinations(indices_to_perturb, combo_lvl + 1)
-                ]
-        else:
-            all_indices = [
-                [i] for i in range(1, example_cell["length"][0] - 1)
-            ]  # Exclude CLS and EOS tokens
-            all_indices = [
-                index for index in all_indices if index not in indices_to_perturb
-            ]
-            indices_to_perturb = [
-                [[j for i in indices_to_perturb for j in i], x] for x in all_indices
-            ]
-
-    length = len(indices_to_perturb)
-    perturbation_dataset = Dataset.from_dict(
-        {
-            "input_ids": example_cell["input_ids"] * length,
-            "perturb_index": indices_to_perturb,
-        }
-    )
-
-    if length < 400:
-        num_proc_i = 1
-    else:
-        num_proc_i = num_proc
-
-    if perturb_type == "delete":
-        perturbation_dataset = perturbation_dataset.map(
-            delete_indices, num_proc=num_proc_i
-        )
-    elif perturb_type == "overexpress":
-        perturbation_dataset = perturbation_dataset.map(
-            overexpress_indices_special, num_proc=num_proc_i
-        )
-
-    perturbation_dataset = perturbation_dataset.map(measure_length, num_proc=num_proc_i)
-
-    return perturbation_dataset, indices_to_perturb
-
-
-# original cell emb removing the activated/overexpressed/inhibited gene emb
-# so that only non-perturbed gene embeddings are compared to each other
-# in original or perturbed context
-def make_comparison_batch(original_emb_batch, indices_to_perturb, perturb_group):
-    all_embs_list = []
-
-    # if making comparison batch for multiple perturbations in single cell
-    if perturb_group is False:
-        # squeeze if single cell
-        if original_emb_batch.ndim == 3 and original_emb_batch.size()[0] == 1:
-            original_emb_batch = torch.squeeze(original_emb_batch)
-        original_emb_list = [original_emb_batch] * len(indices_to_perturb)
-    # if making comparison batch for single perturbation in multiple cells
-    elif perturb_group is True:
-        original_emb_list = original_emb_batch
-
-    for original_emb, indices in zip(original_emb_list, indices_to_perturb):
-        if indices == [-100]:
-            all_embs_list += [original_emb[:]]
-            continue
-
-        emb_list = []
-        start = 0
-        if any(isinstance(el, list) for el in indices):
-            indices = flatten_list(indices)
-
-        # removes indices that were perturbed from the original embedding
-        for i in sorted(indices):
-            emb_list += [original_emb[start:i]]
-            start = i + 1
-
-        emb_list += [original_emb[start:]]
-        all_embs_list += [torch.cat(emb_list)]
-
-    len_set = set([emb.size()[0] for emb in all_embs_list])
-    if len(len_set) > 1:
-        max_len = max(len_set)
-        all_embs_list = [pad_2d_tensor(emb, None, max_len, 0) for emb in all_embs_list]
-    return torch.stack(all_embs_list)
-
-
-def pad_list(input_ids, pad_token_id, max_len):
-    input_ids = np.pad(
-        input_ids,
-        (0, max_len - len(input_ids)),
-        mode="constant",
-        constant_values=pad_token_id,
-    )
-    return input_ids
-
-
-def pad_xd_tensor(tensor, pad_token_id, max_len, dim):
-    padding_length = max_len - tensor.size()[dim]
-    # Construct a padding configuration where all padding values are 0, except for the padding dimension
-    # 2 * number of dimensions (padding before and after for every dimension)
-    pad_config = [0] * 2 * tensor.dim()
-    # Set the padding after the desired dimension to the calculated padding length
-    pad_config[-2 * dim - 1] = padding_length
-    return torch.nn.functional.pad(
-        tensor, pad=pad_config, mode="constant", value=pad_token_id
-    )
-
-
-def pad_tensor(tensor, pad_token_id, max_len):
-    tensor = torch.nn.functional.pad(
-        tensor, pad=(0, max_len - tensor.numel()), mode="constant", value=pad_token_id
-    )
-
-    return tensor
-
-
-def pad_2d_tensor(tensor, pad_token_id, max_len, dim):
-    if dim == 0:
-        pad = (0, 0, 0, max_len - tensor.size()[dim])
-    elif dim == 1:
-        pad = (0, max_len - tensor.size()[dim], 0, 0)
-    tensor = torch.nn.functional.pad(
-        tensor, pad=pad, mode="constant", value=pad_token_id
-    )
-    return tensor
-
-
-def pad_3d_tensor(tensor, pad_token_id, max_len, dim):
-    if dim == 0:
-        raise Exception("dim 0 usually does not need to be padded.")
-    if dim == 1:
-        pad = (0, 0, 0, max_len - tensor.size()[dim])
-    elif dim == 2:
-        pad = (0, max_len - tensor.size()[dim], 0, 0)
-    tensor = torch.nn.functional.pad(
-        tensor, pad=pad, mode="constant", value=pad_token_id
-    )
-    return tensor
-
-
-def pad_or_truncate_encoding(encoding, pad_token_id, max_len):
-    if isinstance(encoding, torch.Tensor):
-        encoding_len = encoding.size()[0]
-    elif isinstance(encoding, list):
-        encoding_len = len(encoding)
-    if encoding_len > max_len:
-        encoding = encoding[0:max_len]
-    elif encoding_len < max_len:
-        if isinstance(encoding, torch.Tensor):
-            encoding = pad_tensor(encoding, pad_token_id, max_len)
-        elif isinstance(encoding, list):
-            encoding = pad_list(encoding, pad_token_id, max_len)
-    return encoding
-
-
-# pad list of tensors and convert to tensor
-def pad_tensor_list(
-    tensor_list,
-    dynamic_or_constant,
-    pad_token_id,
-    model_input_size,
-    dim=None,
-    padding_func=None,
-):
-    # determine maximum tensor length
-    if dynamic_or_constant == "dynamic":
-        max_len = max([tensor.squeeze().numel() for tensor in tensor_list])
-    elif isinstance(dynamic_or_constant, int):
-        max_len = dynamic_or_constant
-    else:
-        max_len = model_input_size
-        logger.warning(
-            "If padding style is constant, must provide integer value. "
-            f"Setting padding to max input size {model_input_size}."
-        )
-
-    # pad all tensors to maximum length
-    if dim is None:
-        tensor_list = [
-            pad_tensor(tensor, pad_token_id, max_len) for tensor in tensor_list
-        ]
-    else:
-        tensor_list = [
-            padding_func(tensor, pad_token_id, max_len, dim) for tensor in tensor_list
-        ]
-    # return stacked tensors
-    if padding_func != pad_3d_tensor:
-        return torch.stack(tensor_list)
-    else:
-        return torch.cat(tensor_list, 0)
-
-
-def gen_attention_mask(minibatch_encoding, max_len=None):
-    if max_len is None:
-        max_len = max(minibatch_encoding["length"])
-    original_lens = minibatch_encoding["length"]
-    attention_mask = [
-        [1] * original_len + [0] * (max_len - original_len)
-        if original_len <= max_len
-        else [1] * max_len
-        for original_len in original_lens
-    ]
-    return torch.tensor(attention_mask, device="cuda")
-
-
-# get cell embeddings excluding padding
-def mean_nonpadding_embs(embs, original_lens, dim=1):
-    # create a mask tensor based on padding lengths
-    mask = torch.arange(embs.size(dim), device=embs.device) < original_lens.unsqueeze(1)
-    if embs.dim() == 3:
-        # fill the masked positions in embs with zeros
-        masked_embs = embs.masked_fill(~mask.unsqueeze(2), 0.0)
-
-        # compute the mean across the non-padding dimensions
-        mean_embs = masked_embs.sum(dim) / original_lens.view(-1, 1).float()
-
-    elif embs.dim() == 2:
-        masked_embs = embs.masked_fill(~mask, 0.0)
-        mean_embs = masked_embs.sum(dim) / original_lens.float()
-    return mean_embs
-
-
-# get cell embeddings when there is no padding
-def compute_nonpadded_cell_embedding(embs, cell_emb_style):
-    if cell_emb_style == "mean_pool":
-        return torch.mean(embs, dim=embs.ndim - 2)
-
-
-# quantify shifts for a set of genes
-def quant_cos_sims(
-    perturbation_emb,
-    original_emb,
-    cell_states_to_model,
-    state_embs_dict,
-    emb_mode="gene",
-):
-    if emb_mode == "gene":
-        cos = torch.nn.CosineSimilarity(dim=2)
-    elif emb_mode == "cell":
-        cos = torch.nn.CosineSimilarity(dim=1)
-
-    # if emb_mode == "gene", can only calculate gene cos sims
-    # against original cell
-    if cell_states_to_model is None or emb_mode == "gene":
-        cos_sims = cos(perturbation_emb, original_emb).to("cuda")
-
-    elif cell_states_to_model is not None and emb_mode == "cell":
-        possible_states = get_possible_states(cell_states_to_model)
-        cos_sims = dict(zip(possible_states, [[] for _ in range(len(possible_states))]))
-        for state in possible_states:
-            cos_sims[state] = cos_sim_shift(
-                original_emb,
-                perturbation_emb,
-                state_embs_dict[state].to("cuda"),  # required to move to cuda here
-                cos,
-            )
-
-    return cos_sims
-
-
-# calculate cos sim shift of perturbation with respect to origin and alternative cell
-def cos_sim_shift(original_emb, perturbed_emb, end_emb, cos):
-    origin_v_end = cos(original_emb, end_emb)
-    perturb_v_end = cos(perturbed_emb, end_emb)
-
-    return perturb_v_end - origin_v_end
-
-
-def concatenate_cos_sims(cos_sims):
-    if isinstance(cos_sims, list):
-        return torch.cat(cos_sims)
-    else:
-        for state in cos_sims.keys():
-            cos_sims[state] = torch.cat(cos_sims[state])
-        return cos_sims
-
-
-def write_perturbation_dictionary(cos_sims_dict: defaultdict, output_path_prefix: str):
-    with open(f"{output_path_prefix}_raw.pickle", "wb") as fp:
-        pickle.dump(cos_sims_dict, fp)
-
-
-def tensor_list_to_pd(tensor_list):
-    tensor = torch.cat(tensor_list).cpu().numpy()
-    df = pd.DataFrame(tensor)
-    return df
-
-
-def validate_cell_states_to_model(cell_states_to_model):
-    if cell_states_to_model is not None:
-        if len(cell_states_to_model.items()) == 1:
-            logger.warning(
-                "The single value dictionary for cell_states_to_model will be "
-                "replaced with a dictionary with named keys for start, goal, and alternate states. "
-                "Please specify state_key, start_state, goal_state, and alt_states "
-                "in the cell_states_to_model dictionary for future use. "
-                "For example, cell_states_to_model={"
-                "'state_key': 'disease', "
-                "'start_state': 'dcm', "
-                "'goal_state': 'nf', "
-                "'alt_states': ['hcm', 'other1', 'other2']}"
-            )
-            for key, value in cell_states_to_model.items():
-                if (len(value) == 3) and isinstance(value, tuple):
-                    if (
-                        isinstance(value[0], list)
-                        and isinstance(value[1], list)
-                        and isinstance(value[2], list)
-                    ):
-                        if len(value[0]) == 1 and len(value[1]) == 1:
-                            all_values = value[0] + value[1] + value[2]
-                            if len(all_values) == len(set(all_values)):
-                                continue
-            # reformat to the new named key format
-            state_values = flatten_list(list(cell_states_to_model.values()))
-
-            cell_states_to_model = {
-                "state_key": list(cell_states_to_model.keys())[0],
-                "start_state": state_values[0][0],
-                "goal_state": state_values[1][0],
-                "alt_states": state_values[2:][0],
-            }
-        elif set(cell_states_to_model.keys()).issuperset(
-            {"state_key", "start_state", "goal_state"}
-        ):
-            if (
-                (cell_states_to_model["state_key"] is None)
-                or (cell_states_to_model["start_state"] is None)
-                or (cell_states_to_model["goal_state"] is None)
-            ):
-                logger.error(
-                    "Please specify 'state_key', 'start_state', and 'goal_state' in cell_states_to_model."
-                )
-                raise
-
-            if (
-                cell_states_to_model["start_state"]
-                == cell_states_to_model["goal_state"]
-            ):
-                logger.error("All states must be unique.")
-                raise
-
-            if "alt_states" in set(cell_states_to_model.keys()):
-                if cell_states_to_model["alt_states"] is not None:
-                    if not isinstance(cell_states_to_model["alt_states"], list):
-                        logger.error(
-                            "cell_states_to_model['alt_states'] must be a list (even if it is one element)."
-                        )
-                        raise
-                    if len(cell_states_to_model["alt_states"]) != len(
-                        set(cell_states_to_model["alt_states"])
-                    ):
-                        logger.error("All states must be unique.")
-                        raise
-            else:
-                cell_states_to_model["alt_states"] = []
-
-        else:
-            logger.error(
-                "cell_states_to_model must only have the following four keys: "
-                "'state_key', 'start_state', 'goal_state', 'alt_states'."
-                "For example, cell_states_to_model={"
-                "'state_key': 'disease', "
-                "'start_state': 'dcm', "
-                "'goal_state': 'nf', "
-                "'alt_states': ['hcm', 'other1', 'other2']}"
-            )
-            raise
-
-
-class GeneIdHandler:
-    def __init__(self, raise_errors=False):
-        def invert_dict(dict_obj):
-            return {v: k for k, v in dict_obj.items()}
-
-        self.raise_errors = raise_errors
-
-        with open(TOKEN_DICTIONARY_FILE, "rb") as f:
-            self.gene_token_dict = pickle.load(f)
-            self.token_gene_dict = invert_dict(self.gene_token_dict)
-
-        with open(ENSEMBL_DICTIONARY_FILE, "rb") as f:
-            self.id_gene_dict = pickle.load(f)
-            self.gene_id_dict = invert_dict(self.id_gene_dict)
-
-    def ens_to_token(self, ens_id):
-        if not self.raise_errors:
-            return self.gene_token_dict.get(ens_id, ens_id)
-        else:
-            return self.gene_token_dict[ens_id]
-
-    def token_to_ens(self, token):
-        if not self.raise_errors:
-            return self.token_gene_dict.get(token, token)
-        else:
-            return self.token_gene_dict[token]
-
-    def ens_to_symbol(self, ens_id):
-        if not self.raise_errors:
-            return self.gene_id_dict.get(ens_id, ens_id)
-        else:
-            return self.gene_id_dict[ens_id]
-
-    def symbol_to_ens(self, symbol):
-        if not self.raise_errors:
-            return self.id_gene_dict.get(symbol, symbol)
-        else:
-            return self.id_gene_dict[symbol]
-
-    def token_to_symbol(self, token):
-        return self.ens_to_symbol(self.token_to_ens(token))
-
-    def symbol_to_token(self, symbol):
-        return self.ens_to_token(self.symbol_to_ens(symbol))

geneformer/pretrainer.py

@@ -1,640 +0,0 @@
-"""
-Geneformer precollator and pretrainer.
-
-Huggingface data collator and trainer modified to accommodate single-cell transcriptomics data.
-"""
-import collections
-import math
-import pickle
-import warnings
-from enum import Enum
-from typing import Dict, List, Optional, Union
-
-import numpy as np
-import torch
-from datasets import Dataset
-from packaging import version
-from torch.utils.data.sampler import RandomSampler
-from transformers import (
-    BatchEncoding,
-    DataCollatorForLanguageModeling,
-    SpecialTokensMixin,
-    Trainer,
-)
-from transformers.file_utils import is_datasets_available, is_sagemaker_dp_enabled
-from transformers.trainer_pt_utils import (
-    LengthGroupedSampler,
-)
-from transformers.utils import is_tf_available, is_torch_available, logging, to_py_obj
-from transformers.utils.generic import _is_tensorflow, _is_torch
-
-logger = logging.get_logger(__name__)
-EncodedInput = List[int]
-VERY_LARGE_INTEGER = int(
-    1e30
-)  # This is used to set the max input length for a model with infinite size input
-LARGE_INTEGER = int(
-    1e20
-)  # This is used when we need something big but slightly smaller than VERY_LARGE_INTEGER
-
-if is_sagemaker_dp_enabled():
-    import smdistributed.dataparallel.torch.distributed as dist
-else:
-    import torch.distributed as dist
-
-_is_torch_generator_available = False
-if version.parse(torch.__version__) >= version.parse("1.6"):
-    _is_torch_generator_available = True
-
-
-class ExplicitEnum(Enum):
-    """
-    Enum with more explicit error message for missing values.
-    """
-
-    @classmethod
-    def _missing_(cls, value):
-        raise ValueError(
-            "%r is not a valid %s, please select one of %s"
-            % (value, cls.__name__, str(list(cls._value2member_map_.keys())))
-        )
-
-
-class TruncationStrategy(ExplicitEnum):
-    """
-    Possible values for the ``truncation`` argument in :meth:`PreTrainedTokenizerBase.__call__`. Useful for
-    tab-completion in an IDE.
-    """
-
-    ONLY_FIRST = "only_first"
-    ONLY_SECOND = "only_second"
-    LONGEST_FIRST = "longest_first"
-    DO_NOT_TRUNCATE = "do_not_truncate"
-
-
-class PaddingStrategy(ExplicitEnum):
-    """
-    Possible values for the ``padding`` argument in :meth:`PreTrainedTokenizerBase.__call__`. Useful for tab-completion
-    in an IDE.
-    """
-
-    LONGEST = "longest"
-    MAX_LENGTH = "max_length"
-    DO_NOT_PAD = "do_not_pad"
-
-
-class TensorType(ExplicitEnum):
-    """
-    Possible values for the ``return_tensors`` argument in :meth:`PreTrainedTokenizerBase.__call__`. Useful for
-    tab-completion in an IDE.
-    """
-
-    PYTORCH = "pt"
-    TENSORFLOW = "tf"
-    NUMPY = "np"
-    JAX = "jax"
-
-
-class GeneformerPreCollator(SpecialTokensMixin):
-    def __init__(self, *args, **kwargs) -> None:
-        super().__init__(mask_token="<mask>", pad_token="<pad>")
-
-        self.token_dictionary = kwargs.get("token_dictionary")
-        self.padding_side = "right"
-        self.model_input_names = ["input_ids"]
-
-    def convert_ids_to_tokens(self, value):
-        return self.token_dictionary.get(value)
-
-    def _get_padding_truncation_strategies(
-        self,
-        padding=False,
-        truncation=False,
-        max_length=None,
-        pad_to_multiple_of=None,
-        verbose=True,
-        **kwargs,
-    ):
-        """
-        Find the correct padding/truncation strategy with backward compatibility for old arguments (truncation_strategy
-        and pad_to_max_length) and behaviors.
-        """
-        old_truncation_strategy = kwargs.pop("truncation_strategy", "do_not_truncate")
-        old_pad_to_max_length = kwargs.pop("pad_to_max_length", False)
-
-        # Backward compatibility for previous behavior, maybe we should deprecate it:
-        # If you only set max_length, it activates truncation for max_length
-        if max_length is not None and padding is False and truncation is False:
-            if verbose:
-                if not self.deprecation_warnings.get(
-                    "Truncation-not-explicitly-activated", False
-                ):
-                    logger.warning(
-                        "Truncation was not explicitly activated but `max_length` is provided a specific value, "
-                        "please use `truncation=True` to explicitly truncate examples to max length. "
-                        "Defaulting to 'longest_first' truncation strategy. "
-                        "If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy "
-                        "more precisely by providing a specific strategy to `truncation`."
-                    )
-                self.deprecation_warnings["Truncation-not-explicitly-activated"] = True
-            truncation = "longest_first"
-
-        # Get padding strategy
-        if padding is False and old_pad_to_max_length:
-            if verbose:
-                warnings.warn(
-                    "The `pad_to_max_length` argument is deprecated and will be removed in a future version, "
-                    "use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or "
-                    "use `padding='max_length'` to pad to a max length. In this case, you can give a specific "
-                    "length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the "
-                    "maximal input size of the model (e.g. 512 for Bert).",
-                    FutureWarning,
-                )
-            if max_length is None:
-                padding_strategy = PaddingStrategy.LONGEST
-            else:
-                padding_strategy = PaddingStrategy.MAX_LENGTH
-        elif padding is not False:
-            if padding is True:
-                padding_strategy = (
-                    PaddingStrategy.LONGEST
-                )  # Default to pad to the longest sequence in the batch
-            elif not isinstance(padding, PaddingStrategy):
-                padding_strategy = PaddingStrategy(padding)
-            elif isinstance(padding, PaddingStrategy):
-                padding_strategy = padding
-        else:
-            padding_strategy = PaddingStrategy.DO_NOT_PAD
-
-        # Get truncation strategy
-        if truncation is False and old_truncation_strategy != "do_not_truncate":
-            if verbose:
-                warnings.warn(
-                    "The `truncation_strategy` argument is deprecated and will be removed in a future version, "
-                    "use `truncation=True` to truncate examples to a max length. You can give a specific "
-                    "length with `max_length` (e.g. `max_length=45`) or leave max_length to None to truncate to the "
-                    "maximal input size of the model (e.g. 512 for Bert). "
-                    " If you have pairs of inputs, you can give a specific truncation strategy selected among "
-                    "`truncation='only_first'` (will only truncate the first sentence in the pairs) "
-                    "`truncation='only_second'` (will only truncate the second sentence in the pairs) "
-                    "or `truncation='longest_first'` (will iteratively remove tokens from the longest sentence in the pairs).",
-                    FutureWarning,
-                )
-            truncation_strategy = TruncationStrategy(old_truncation_strategy)
-        elif truncation is not False:
-            if truncation is True:
-                truncation_strategy = (
-                    TruncationStrategy.LONGEST_FIRST
-                )  # Default to truncate the longest sequences in pairs of inputs
-            elif not isinstance(truncation, TruncationStrategy):
-                truncation_strategy = TruncationStrategy(truncation)
-            elif isinstance(truncation, TruncationStrategy):
-                truncation_strategy = truncation
-        else:
-            truncation_strategy = TruncationStrategy.DO_NOT_TRUNCATE
-
-        # Set max length if needed
-        if max_length is None:
-            if padding_strategy == PaddingStrategy.MAX_LENGTH:
-                if self.model_max_length > LARGE_INTEGER:
-                    if verbose:
-                        if not self.deprecation_warnings.get(
-                            "Asking-to-pad-to-max_length", False
-                        ):
-                            logger.warning(
-                                "Asking to pad to max_length but no maximum length is provided and the model has no predefined maximum length. "
-                                "Default to no padding."
-                            )
-                        self.deprecation_warnings["Asking-to-pad-to-max_length"] = True
-                    padding_strategy = PaddingStrategy.DO_NOT_PAD
-                else:
-                    max_length = self.model_max_length
-
-            if truncation_strategy != TruncationStrategy.DO_NOT_TRUNCATE:
-                if self.model_max_length > LARGE_INTEGER:
-                    if verbose:
-                        if not self.deprecation_warnings.get(
-                            "Asking-to-truncate-to-max_length", False
-                        ):
-                            logger.warning(
-                                "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. "
-                                "Default to no truncation."
-                            )
-                        self.deprecation_warnings[
-                            "Asking-to-truncate-to-max_length"
-                        ] = True
-                    truncation_strategy = TruncationStrategy.DO_NOT_TRUNCATE
-                else:
-                    max_length = self.model_max_length
-
-        # Test if we have a padding token
-        if padding_strategy != PaddingStrategy.DO_NOT_PAD and (
-            not self.pad_token or self.pad_token_id < 0
-        ):
-            raise ValueError(
-                "Asking to pad but the tokenizer does not have a padding token. "
-                "Please select a token to use as `pad_token` `(tokenizer.pad_token = tokenizer.eos_token e.g.)` "
-                "or add a new pad token via `tokenizer.add_special_tokens({'pad_token': '[PAD]'})`."
-            )
-
-        # Check that we will truncate to a multiple of pad_to_multiple_of if both are provided
-        if (
-            truncation_strategy != TruncationStrategy.DO_NOT_TRUNCATE
-            and padding_strategy != PaddingStrategy.DO_NOT_PAD
-            and pad_to_multiple_of is not None
-            and max_length is not None
-            and (max_length % pad_to_multiple_of != 0)
-        ):
-            raise ValueError(
-                f"Truncation and padding are both activated but "
-                f"truncation length ({max_length}) is not a multiple of pad_to_multiple_of ({pad_to_multiple_of})."
-            )
-
-        return padding_strategy, truncation_strategy, max_length, kwargs
-
-    def pad(
-        self,
-        encoded_inputs: Union[
-            BatchEncoding,
-            List[BatchEncoding],
-            Dict[str, EncodedInput],
-            Dict[str, List[EncodedInput]],
-            List[Dict[str, EncodedInput]],
-        ],
-        padding: Union[bool, str, PaddingStrategy] = True,
-        max_length: Optional[int] = None,
-        pad_to_multiple_of: Optional[int] = None,
-        return_attention_mask: Optional[bool] = True,
-        return_tensors: Optional[Union[str, TensorType]] = None,
-        verbose: bool = True,
-    ) -> BatchEncoding:
-        """
-        Pad a single encoded input or a batch of encoded inputs up to predefined length or to the max sequence length
-        in the batch.
-
-        Padding side (left/right) padding token ids are defined at the tokenizer level (with ``self.padding_side``,
-        ``self.pad_token_id`` and ``self.pad_token_type_id``)
-
-        .. note::
-
-            If the ``encoded_inputs`` passed are dictionary of numpy arrays, PyTorch tensors or TensorFlow tensors, the
-            result will use the same type unless you provide a different tensor type with ``return_tensors``. In the
-            case of PyTorch tensors, you will lose the specific device of your tensors however.
-
-        Args:
-            encoded_inputs (:class:`~transformers.BatchEncoding`, list of :class:`~transformers.BatchEncoding`, :obj:`Dict[str, List[int]]`, :obj:`Dict[str, List[List[int]]` or :obj:`List[Dict[str, List[int]]]`):
-                Tokenized inputs. Can represent one input (:class:`~transformers.BatchEncoding` or :obj:`Dict[str,
-                List[int]]`) or a batch of tokenized inputs (list of :class:`~transformers.BatchEncoding`, `Dict[str,
-                List[List[int]]]` or `List[Dict[str, List[int]]]`) so you can use this method during preprocessing as
-                well as in a PyTorch Dataloader collate function.
-
-                Instead of :obj:`List[int]` you can have tensors (numpy arrays, PyTorch tensors or TensorFlow tensors),
-                see the note above for the return type.
-            padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`):
-                 Select a strategy to pad the returned sequences (according to the model's padding side and padding
-                 index) among:
-
-                * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a
-                  single sequence if provided).
-                * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the
-                  maximum acceptable input length for the model if that argument is not provided.
-                * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
-                  different lengths).
-            max_length (:obj:`int`, `optional`):
-                Maximum length of the returned list and optionally padding length (see above).
-            pad_to_multiple_of (:obj:`int`, `optional`):
-                If set will pad the sequence to a multiple of the provided value.
-
-                This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
-                >= 7.5 (Volta).
-            return_attention_mask (:obj:`bool`, `optional`):
-                Whether to return the attention mask. If left to the default, will return the attention mask according
-                to the specific tokenizer's default, defined by the :obj:`return_outputs` attribute.
-
-                `What are attention masks? <../glossary.html#attention-mask>`__
-            return_tensors (:obj:`str` or :class:`~transformers.tokenization_utils_base.TensorType`, `optional`):
-                If set, will return tensors instead of list of python integers. Acceptable values are:
-
-                * :obj:`'tf'`: Return TensorFlow :obj:`tf.constant` objects.
-                * :obj:`'pt'`: Return PyTorch :obj:`torch.Tensor` objects.
-                * :obj:`'np'`: Return Numpy :obj:`np.ndarray` objects.
-            verbose (:obj:`bool`, `optional`, defaults to :obj:`True`):
-                Whether or not to print more information and warnings.
-        """
-        # If we have a list of dicts, let's convert it in a dict of lists
-        # We do this to allow using this method as a collate_fn function in PyTorch Dataloader
-        if isinstance(encoded_inputs, (list, tuple)) and isinstance(
-            encoded_inputs[0], (dict, BatchEncoding)
-        ):
-            encoded_inputs = {
-                key: [example[key] for example in encoded_inputs]
-                for key in encoded_inputs[0].keys()
-            }
-
-        # The model's main input name, usually `input_ids`, has be passed for padding
-        if self.model_input_names[0] not in encoded_inputs:
-            raise ValueError(
-                "You should supply an encoding or a list of encodings to this method"
-                f"that includes {self.model_input_names[0]}, but you provided {list(encoded_inputs.keys())}"
-            )
-
-        required_input = encoded_inputs[self.model_input_names[0]]
-
-        if not required_input:
-            if return_attention_mask:
-                encoded_inputs["attention_mask"] = []
-            return encoded_inputs
-
-        # If we have PyTorch/TF/NumPy tensors/arrays as inputs, we cast them as python objects
-        # and rebuild them afterwards if no return_tensors is specified
-        # Note that we lose the specific device the tensor may be on for PyTorch
-
-        first_element = required_input[0]
-        if isinstance(first_element, (list, tuple)):
-            # first_element might be an empty list/tuple in some edge cases so we grab the first non empty element.
-            index = 0
-            while len(required_input[index]) == 0:
-                index += 1
-            if index < len(required_input):
-                first_element = required_input[index][0]
-        # At this state, if `first_element` is still a list/tuple, it's an empty one so there is nothing to do.
-        if not isinstance(first_element, (int, list, tuple)):
-            if is_tf_available() and _is_tensorflow(first_element):
-                return_tensors = "tf" if return_tensors is None else return_tensors
-            elif is_torch_available() and _is_torch(first_element):
-                return_tensors = "pt" if return_tensors is None else return_tensors
-            elif isinstance(first_element, np.ndarray):
-                return_tensors = "np" if return_tensors is None else return_tensors
-            else:
-                raise ValueError(
-                    f"type of {first_element} unknown: {type(first_element)}. "
-                    f"Should be one of a python, numpy, pytorch or tensorflow object."
-                )
-
-            for key, value in encoded_inputs.items():
-                encoded_inputs[key] = to_py_obj(value)
-
-        # Convert padding_strategy in PaddingStrategy
-        padding_strategy, _, max_length, _ = self._get_padding_truncation_strategies(
-            padding=padding, max_length=max_length, verbose=verbose
-        )
-
-        required_input = encoded_inputs[self.model_input_names[0]]
-        if required_input and not isinstance(required_input[0], (list, tuple)):
-            encoded_inputs = self._pad(
-                encoded_inputs,
-                max_length=max_length,
-                padding_strategy=padding_strategy,
-                pad_to_multiple_of=pad_to_multiple_of,
-                return_attention_mask=return_attention_mask,
-            )
-            return BatchEncoding(encoded_inputs, tensor_type=return_tensors)
-
-        batch_size = len(required_input)
-        assert all(
-            len(v) == batch_size for v in encoded_inputs.values()
-        ), "Some items in the output dictionary have a different batch size than others."
-
-        if padding_strategy == PaddingStrategy.LONGEST:
-            max_length = max(len(inputs) for inputs in required_input)
-            padding_strategy = PaddingStrategy.MAX_LENGTH
-
-        batch_outputs = {}
-        for i in range(batch_size):
-            inputs = dict((k, v[i]) for k, v in encoded_inputs.items())
-            outputs = self._pad(
-                inputs,
-                max_length=max_length,
-                padding_strategy=padding_strategy,
-                pad_to_multiple_of=pad_to_multiple_of,
-                return_attention_mask=return_attention_mask,
-            )
-
-            for key, value in outputs.items():
-                if key not in batch_outputs:
-                    batch_outputs[key] = []
-                batch_outputs[key].append(value)
-
-        return BatchEncoding(batch_outputs, tensor_type=return_tensors)
-
-    def _pad(
-        self,
-        encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
-        max_length: Optional[int] = None,
-        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
-        pad_to_multiple_of: Optional[int] = None,
-        return_attention_mask: Optional[bool] = None,
-    ) -> dict:
-        """
-        Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
-
-        Args:
-            encoded_inputs: Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
-            max_length: maximum length of the returned list and optionally padding length (see below).
-                Will truncate by taking into account the special tokens.
-            padding_strategy: PaddingStrategy to use for padding.
-
-                - PaddingStrategy.LONGEST Pad to the longest sequence in the batch
-                - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)
-                - PaddingStrategy.DO_NOT_PAD: Do not pad
-                The tokenizer padding sides are defined in self.padding_side:
-
-                    - 'left': pads on the left of the sequences
-                    - 'right': pads on the right of the sequences
-            pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
-                This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
-                >= 7.5 (Volta).
-            return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics)
-        """
-        # Load from model defaults
-        if return_attention_mask is None:
-            return_attention_mask = "attention_mask" in self.model_input_names
-
-        required_input = encoded_inputs[self.model_input_names[0]]
-
-        if padding_strategy == PaddingStrategy.LONGEST:
-            max_length = len(required_input)
-
-        if (
-            max_length is not None
-            and pad_to_multiple_of is not None
-            and (max_length % pad_to_multiple_of != 0)
-        ):
-            max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of
-
-        needs_to_be_padded = (
-            padding_strategy != PaddingStrategy.DO_NOT_PAD
-            and len(required_input) != max_length
-        )
-
-        if needs_to_be_padded:
-            difference = max_length - len(required_input)
-            if self.padding_side == "right":
-                if return_attention_mask:
-                    encoded_inputs["attention_mask"] = [1] * len(required_input) + [
-                        0
-                    ] * difference
-                if "token_type_ids" in encoded_inputs:
-                    encoded_inputs["token_type_ids"] = (
-                        encoded_inputs["token_type_ids"]
-                        + [self.pad_token_type_id] * difference
-                    )
-                if "special_tokens_mask" in encoded_inputs:
-                    encoded_inputs["special_tokens_mask"] = (
-                        encoded_inputs["special_tokens_mask"] + [1] * difference
-                    )
-                encoded_inputs[self.model_input_names[0]] = (
-                    required_input + [self.pad_token_id] * difference
-                )
-            elif self.padding_side == "left":
-                if return_attention_mask:
-                    encoded_inputs["attention_mask"] = [0] * difference + [1] * len(
-                        required_input
-                    )
-                if "token_type_ids" in encoded_inputs:
-                    encoded_inputs["token_type_ids"] = [
-                        self.pad_token_type_id
-                    ] * difference + encoded_inputs["token_type_ids"]
-                if "special_tokens_mask" in encoded_inputs:
-                    encoded_inputs["special_tokens_mask"] = [
-                        1
-                    ] * difference + encoded_inputs["special_tokens_mask"]
-                encoded_inputs[self.model_input_names[0]] = [
-                    self.pad_token_id
-                ] * difference + required_input
-            else:
-                raise ValueError("Invalid padding strategy:" + str(self.padding_side))
-        elif return_attention_mask and "attention_mask" not in encoded_inputs:
-            encoded_inputs["attention_mask"] = [1] * len(required_input)
-
-        return encoded_inputs
-
-    def get_special_tokens_mask(
-        self,
-        token_ids_0: List[int],
-        token_ids_1: Optional[List[int]] = None,
-        already_has_special_tokens: bool = False,
-    ) -> List[int]:
-        """
-        Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
-        special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods.
-        Args:
-            token_ids_0 (:obj:`List[int]`):
-                List of ids of the first sequence.
-            token_ids_1 (:obj:`List[int]`, `optional`):
-                List of ids of the second sequence.
-            already_has_special_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`):
-                Whether or not the token list is already formatted with special tokens for the model.
-        Returns:
-            A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
-        """
-        assert already_has_special_tokens and token_ids_1 is None, (
-            "You cannot use ``already_has_special_tokens=False`` with this tokenizer. "
-            "Please use a slow (full python) tokenizer to activate this argument."
-            "Or set `return_special_tokens_mask=True` when calling the encoding method "
-            "to get the special tokens mask in any tokenizer. "
-        )
-
-        all_special_ids = self.all_special_ids  # cache the property
-
-        special_tokens_mask = [
-            1 if token in all_special_ids else 0 for token in token_ids_0
-        ]
-
-        return special_tokens_mask
-
-    def convert_tokens_to_ids(
-        self, tokens: Union[str, List[str]]
-    ) -> Union[int, List[int]]:
-        """
-        Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the
-        vocabulary.
-        Args:
-            tokens (:obj:`str` or :obj:`List[str]`): One or several token(s) to convert to token id(s).
-        Returns:
-            :obj:`int` or :obj:`List[int]`: The token id or list of token ids.
-        """
-        if tokens is None:
-            return None
-
-        if isinstance(tokens, str):
-            return self._convert_token_to_id_with_added_voc(tokens)
-
-        ids = []
-        for token in tokens:
-            ids.append(self._convert_token_to_id_with_added_voc(token))
-        return ids
-
-    def _convert_token_to_id_with_added_voc(self, token):
-        if token is None:
-            return None
-
-        return self.token_dictionary.get(token)
-
-    def __len__(self):
-        return len(self.token_dictionary)
-
-
-class GeneformerPretrainer(Trainer):
-    def __init__(self, *args, **kwargs):
-        data_collator = kwargs.get("data_collator", None)
-        token_dictionary = kwargs.pop("token_dictionary")
-        mlm = kwargs.pop("mlm", True)
-        mlm_probability = kwargs.pop("mlm_probability", 0.15)
-
-        if data_collator is None:
-            precollator = GeneformerPreCollator(token_dictionary=token_dictionary)
-
-            # # Data Collator Functions
-            data_collator = DataCollatorForLanguageModeling(
-                tokenizer=precollator, mlm=mlm, mlm_probability=mlm_probability
-            )
-            kwargs["data_collator"] = data_collator
-
-        # load previously saved length vector for dataset to speed up LengthGroupedSampler
-        # pre-obtained with [dataset[i]["length"] for i in range(len(dataset))]
-        example_lengths_file = kwargs.pop("example_lengths_file")
-        if example_lengths_file:
-            with open(example_lengths_file, "rb") as f:
-                self.example_lengths = pickle.load(f)
-        else:
-            raise Exception(
-                "example_lengths_file is required; e.g. https://huggingface.co/datasets/ctheodoris/Genecorpus-30M/tree/main/genecorpus_30M_2048_sorted_lengths.pkl"
-            )
-        super().__init__(*args, **kwargs)
-
-    # updated to not use distributed sampler since Trainer now distributes with accelerate
-    def _get_train_sampler(self) -> Optional[torch.utils.data.sampler.Sampler]:
-        if not isinstance(self.train_dataset, collections.abc.Sized):
-            return None
-
-        generator = None
-        if self.args.world_size <= 1 and _is_torch_generator_available:
-            generator = torch.Generator()
-            generator.manual_seed(
-                int(torch.empty((), dtype=torch.int64).random_().item())
-            )
-
-        # Build the sampler.
-        if self.args.group_by_length:
-            if is_datasets_available() and isinstance(self.train_dataset, Dataset):
-                lengths = self.example_lengths
-            else:
-                lengths = None
-            model_input_name = (
-                self.tokenizer.model_input_names[0]
-                if self.tokenizer is not None
-                else None
-            )
-            return LengthGroupedSampler(
-                    dataset=self.train_dataset,
-                    batch_size=self.args.train_batch_size,
-                    lengths=lengths,
-                    model_input_name=model_input_name,
-                    generator=generator,
-            )
-
-        else:
-            if _is_torch_generator_available:
-                return RandomSampler(self.train_dataset, generator=generator)
-            return RandomSampler(self.train_dataset)

geneformer/tokenizer.py

@@ -1,685 +0,0 @@
-"""
-Geneformer tokenizer.
-
-**Input data:**
-
-| *Required format:* raw counts scRNAseq data without feature selection as .loom or anndata file.
-| *Required row (gene) attribute:* "ensembl_id"; Ensembl ID for each gene.
-| *Required col (cell) attribute:* "n_counts"; total read counts in that cell.
-
-| *Optional col (cell) attribute:* "filter_pass"; binary indicator of whether cell should be tokenized based on user-defined filtering criteria.
-| *Optional col (cell) attributes:* any other cell metadata can be passed on to the tokenized dataset as a custom attribute dictionary as shown below.
-
-**Usage:**
-
-.. code-block :: python
-
-    >>> from geneformer import TranscriptomeTokenizer
-    >>> tk = TranscriptomeTokenizer({"cell_type": "cell_type", "organ_major": "organ"}, nproc=4)
-    >>> tk.tokenize_data("data_directory", "output_directory", "output_prefix")
-    
-**Description:**
-
-| Input data is a directory with .loom or .h5ad files containing raw counts from single cell RNAseq data, including all genes detected in the transcriptome without feature selection. The input file type is specified by the argument file_format in the tokenize_data function.
-
-| The discussion below references the .loom file format, but the analagous labels are required for .h5ad files, just that they will be column instead of row attributes and vice versa due to the transposed format of the two file types.
-
-| Genes should be labeled with Ensembl IDs (loom row attribute "ensembl_id"), which provide a unique identifer for conversion to tokens. Other forms of gene annotations (e.g. gene names) can be converted to Ensembl IDs via Ensembl Biomart. Cells should be labeled with the total read count in the cell (loom column attribute "n_counts") to be used for normalization.
-
-| No cell metadata is required, but custom cell attributes may be passed onto the tokenized dataset by providing a dictionary of custom attributes to be added, which is formatted as loom_col_attr_name : desired_dataset_col_attr_name. For example, if the original .loom dataset has column attributes "cell_type" and "organ_major" and one would like to retain these attributes as labels in the tokenized dataset with the new names "cell_type" and "organ", respectively, the following custom attribute dictionary should be provided: {"cell_type": "cell_type", "organ_major": "organ"}.
-
-| Additionally, if the original .loom file contains a cell column attribute called "filter_pass", this column will be used as a binary indicator of whether to include these cells in the tokenized data. All cells with "1" in this attribute will be tokenized, whereas the others will be excluded. One may use this column to indicate QC filtering or other criteria for selection for inclusion in the final tokenized dataset.
-
-| If one's data is in other formats besides .loom or .h5ad, one can use the relevant tools (such as Anndata tools) to convert the file to a .loom or .h5ad format prior to running the transcriptome tokenizer.
-
-| OF NOTE: Take care that the correct token dictionary and gene median file is used for the correct model. 
-
-| OF NOTE: For 95M model series, special_token should be True and model_input_size should be 4096. For 30M model series, special_token should be False and model_input_size should be 2048.
-
-"""
-
-from __future__ import annotations
-
-import logging
-import os
-import pickle
-import warnings
-from collections import Counter
-from pathlib import Path
-from typing import Literal
-
-import loompy as lp
-import numpy as np
-import pandas as pd
-import scanpy as sc
-import scipy.sparse as sp
-from datasets import Dataset
-from tqdm import tqdm
-
-warnings.filterwarnings("ignore", message=".*The 'nopython' keyword.*")  # noqa
-import loompy as lp  # noqa
-
-logger = logging.getLogger(__name__)
-
-from . import ENSEMBL_MAPPING_FILE, GENE_MEDIAN_FILE, TOKEN_DICTIONARY_FILE
-
-def rank_genes(gene_vector, gene_tokens):
-    """
-    Rank gene expression vector.
-    """
-    # sort by median-scaled gene values
-    sorted_indices = np.argsort(-gene_vector)
-    return gene_tokens[sorted_indices]
-
-
-def tokenize_cell(gene_vector, gene_tokens):
-    """
-    Convert normalized gene expression vector to tokenized rank value encoding.
-    """
-    # create array of gene vector with token indices
-    # mask undetected genes
-    nonzero_mask = np.nonzero(gene_vector)[0]
-    # rank by median-scaled gene values
-    return rank_genes(gene_vector[nonzero_mask], gene_tokens[nonzero_mask])
-
-
-def sum_ensembl_ids(
-    data_directory,
-    collapse_gene_ids,
-    gene_mapping_dict,
-    gene_token_dict,
-    custom_attr_name_dict,
-    file_format="loom",
-    chunk_size=512,
-):
-    if file_format == "loom":
-        """
-        Map Ensembl IDs from gene mapping dictionary. If duplicate Ensembl IDs are found, sum counts together.
-        """
-        with lp.connect(data_directory) as data:
-            assert (
-                "ensembl_id" in data.ra.keys()
-            ), "'ensembl_id' column missing from data.ra.keys()"
-
-            assert (
-                "ensembl_id_collapsed" not in data.ra.keys()
-            ), "'ensembl_id_collapsed' column already exists in data.ra.keys()"
-            
-            assert (
-                "n_counts" in data.ca.keys()
-            ), "'n_counts' column missing from data.ca.keys()"
-
-            if custom_attr_name_dict is not None:
-                for label in custom_attr_name_dict:
-                    assert label in data.ca.keys(), f"Attribute `{label}` not present in dataset features"
-
-            # Get the ensembl ids that exist in data
-            ensembl_ids = data.ra.ensembl_id
-            # Check for duplicate Ensembl IDs if collapse_gene_ids is False.
-            # Comparing to gene_token_dict here, would not perform any mapping steps
-            if not collapse_gene_ids:
-                ensembl_id_check = [
-                    gene for gene in ensembl_ids if gene in gene_token_dict.keys()
-                ]
-                if len(ensembl_id_check) == len(set(ensembl_id_check)):
-                    return data_directory
-                else:
-                    raise ValueError("Error: data Ensembl IDs non-unique.")
-    
-            # Get the genes that exist in the mapping dictionary and the value of those genes
-            genes_in_map_dict = [gene for gene in ensembl_ids if gene in gene_mapping_dict.keys()]
-            vals_from_map_dict = [gene_mapping_dict.get(gene) for gene in genes_in_map_dict]
-
-            # if the genes in the mapping dict and the value of those genes are of the same length,
-            # simply return the mapped values
-            if(len(set(genes_in_map_dict)) == len(set(vals_from_map_dict))):
-                mapped_vals = [gene_mapping_dict.get(gene.upper()) for gene in data.ra["ensembl_id"]]
-                data.ra["ensembl_id_collapsed"] = mapped_vals
-                return data_directory
-            # Genes need to be collapsed
-            else:
-                dedup_filename = data_directory.with_name(
-                    data_directory.stem + "__dedup.loom"
-                )
-                mapped_vals = [gene_mapping_dict.get(gene.upper()) for gene in data.ra["ensembl_id"]]
-                data.ra["ensembl_id_collapsed"] = mapped_vals
-                dup_genes = [
-                    idx
-                    for idx, count in Counter(data.ra["ensembl_id_collapsed"]).items()
-                    if count > 1
-                ]
-                num_chunks = int(np.ceil(data.shape[1] / chunk_size))
-                first_chunk = True
-                for _, _, view in tqdm(
-                    data.scan(axis=1, batch_size=chunk_size), total=num_chunks
-                ):
-
-                    def process_chunk(view, duplic_genes):
-                        data_count_view = pd.DataFrame(
-                            view, index=data.ra["ensembl_id_collapsed"]
-                        )
-                        unique_data_df = data_count_view.loc[
-                            ~data_count_view.index.isin(duplic_genes)
-                        ]
-                        dup_data_df = data_count_view.loc[
-                            data_count_view.index.isin(
-                                [i for i in duplic_genes if "None" not in i]
-                            )
-                        ]
-                        summed_data = dup_data_df.groupby(dup_data_df.index).sum()
-                        if not summed_data.index.is_unique:
-                            raise ValueError(
-                                "Error: Ensembl IDs in summed data frame non-unique."
-                            )
-                        data_count_view = pd.concat(
-                            [unique_data_df, summed_data], axis=0
-                        )
-                        if not data_count_view.index.is_unique:
-                            raise ValueError(
-                                "Error: Ensembl IDs in final data frame non-unique."
-                            )
-                        return data_count_view
-
-                    processed_chunk = process_chunk(view[:, :], dup_genes)
-                    processed_array = processed_chunk.to_numpy()
-                    new_row_attrs = {"ensembl_id_collapsed": processed_chunk.index.to_numpy()}
-
-                    if "n_counts" not in view.ca.keys():
-                        total_count_view = np.sum(view[:, :], axis=0).astype(int)
-                        view.ca["n_counts"] = total_count_view
-
-                    if first_chunk:  # Create the Loom file with the first chunk
-                        lp.create(
-                            f"{dedup_filename}",
-                            processed_array,
-                            row_attrs=new_row_attrs,
-                            col_attrs=view.ca,
-                        )
-                        first_chunk = False
-                    else:  # Append subsequent chunks
-                        with lp.connect(dedup_filename, mode="r+") as dsout:
-                            dsout.add_columns(processed_array, col_attrs=view.ca)
-                return dedup_filename
-
-    elif file_format == "h5ad":
-        """
-        Map Ensembl IDs from gene mapping dictionary. If duplicate Ensembl IDs are found, sum counts together.
-        Returns adata object with deduplicated Ensembl IDs.
-        """
-
-        data = sc.read_h5ad(str(data_directory))
-
-        assert (
-            "ensembl_id" in data.var.columns
-        ), "'ensembl_id' column missing from data.var"
-
-        assert (
-            "ensembl_id_collapsed" not in data.var.columns
-        ), "'ensembl_id_collapsed' column already exists in data.var"
-        assert (
-            "n_counts" in data.obs.columns
-        ), "'n_counts' column missing from data.obs"
-
-        if custom_attr_name_dict is not None:
-            for label in custom_attr_name_dict:
-                assert label in data.obs.columns, f"Attribute `{label}` not present in data.obs"
-
-
-        # Get the ensembl ids that exist in data
-        ensembl_ids = data.var.ensembl_id
-        # Check for duplicate Ensembl IDs if collapse_gene_ids is False.
-        # Comparing to gene_token_dict here, would not perform any mapping steps
-        if not collapse_gene_ids:
-            ensembl_id_check = [
-                gene for gene in ensembl_ids if gene in gene_token_dict.keys()
-            ]
-            if len(ensembl_id_check) == len(set(ensembl_id_check)):
-                return data_directory
-            else:
-                raise ValueError("Error: data Ensembl IDs non-unique.")
-
-        # Get the genes that exist in the mapping dictionary and the value of those genes
-        genes_in_map_dict = [gene for gene in ensembl_ids if gene in gene_mapping_dict.keys()]
-        vals_from_map_dict = [gene_mapping_dict.get(gene) for gene in genes_in_map_dict]
-
-        # if the genes in the mapping dict and the value of those genes are of the same length,
-        # simply return the mapped values
-        if(len(set(genes_in_map_dict)) == len(set(vals_from_map_dict))):
-            data.var["ensembl_id_collapsed"] = data.var.ensembl_id.str.upper().map(gene_mapping_dict)
-            return data
-        # Genes need to be collapsed
-        else:
-            data.var["ensembl_id_collapsed"] = data.var.ensembl_id.str.upper().map(gene_mapping_dict)
-            data.var_names = data.var["ensembl_id_collapsed"]
-            data = data[:, ~data.var.index.isna()]
-            dup_genes = [
-                idx for idx, count in Counter(data.var_names).items() if count > 1
-            ]
-
-            num_chunks = int(np.ceil(data.shape[0] / chunk_size))
-
-            processed_genes = []
-            for i in tqdm(range(num_chunks)):
-                start_idx = i * chunk_size
-                end_idx = min((i + 1) * chunk_size, data.shape[0])
-                data_chunk = data[start_idx:end_idx, :]
-
-                processed_chunks = []
-                for dup_gene in dup_genes:
-                    data_dup_gene = data_chunk[:, data_chunk.var_names == dup_gene]
-                    df = pd.DataFrame.sparse.from_spmatrix(
-                        data_dup_gene.X,
-                        index=data_dup_gene.obs_names,
-                        columns=data_dup_gene.var_names,
-                    )
-                    df_sum = pd.DataFrame(df.sum(axis=1))
-                    df_sum.columns = [dup_gene]
-                    df_sum.index = data_dup_gene.obs.index
-                    processed_chunks.append(df_sum)
-
-                processed_chunks = pd.concat(processed_chunks, axis=1)
-                processed_genes.append(processed_chunks)
-            processed_genes = pd.concat(processed_genes, axis=0)
-            var_df = pd.DataFrame({"ensembl_id_collapsed": processed_genes.columns})
-            var_df.index = processed_genes.columns
-            processed_genes = sc.AnnData(X=processed_genes, obs=data.obs, var=var_df)
-
-            data_dedup = data[:, ~data.var.index.isin(dup_genes)]  # Deduplicated data
-            data_dedup = sc.concat([data_dedup, processed_genes], axis=1)
-            data_dedup.obs = data.obs
-            return data_dedup
-
-
-class TranscriptomeTokenizer:
-    def __init__(
-        self,
-        custom_attr_name_dict=None,
-        nproc=1,
-        chunk_size=512,
-        model_input_size=4096,
-        special_token=True,
-        collapse_gene_ids=True,
-        gene_median_file=GENE_MEDIAN_FILE,
-        token_dictionary_file=TOKEN_DICTIONARY_FILE,
-        gene_mapping_file=ENSEMBL_MAPPING_FILE,
-    ):
-        """
-        Initialize tokenizer.
-        
-        **Parameters:**
-        
-        custom_attr_name_dict : None, dict
-            | Dictionary of custom attributes to be added to the dataset.
-            | Keys are the names of the attributes in the loom file.
-            | Values are the names of the attributes in the dataset.
-        nproc : int
-            | Number of processes to use for dataset mapping.
-        chunk_size : int = 512
-            | Chunk size for anndata tokenizer.
-        model_input_size : int = 4096
-            | Max input size of model to truncate input to.
-            | For the 30M model series, should be 2048. For the 95M model series, should be 4096.
-        special_token : bool = True
-            | Adds CLS token before and EOS token after rank value encoding.
-            | For the 30M model series, should be False. For the 95M model series, should be True.
-        collapse_gene_ids : bool = True
-            | Whether to collapse gene IDs based on gene mapping dictionary.
-        gene_median_file : Path
-            | Path to pickle file containing dictionary of non-zero median
-            | gene expression values across Genecorpus-30M.
-        token_dictionary_file : Path
-            | Path to pickle file containing token dictionary (Ensembl IDs:token).
-        gene_mapping_file : None, Path
-            | Path to pickle file containing dictionary for collapsing gene IDs.
-
-        """
-        # dictionary of custom attributes {output dataset column name: input .loom column name}
-        self.custom_attr_name_dict = custom_attr_name_dict
-
-        # number of processes for dataset mapping
-        self.nproc = nproc
-
-        # chunk size for anndata tokenizer
-        self.chunk_size = chunk_size
-
-        # input size for tokenization
-        self.model_input_size = model_input_size
-
-        # add CLS and EOS tokens
-        self.special_token = special_token
-
-        # load dictionary of gene normalization factors
-        # (non-zero median value of expression across Genecorpus-30M)
-        with open(gene_median_file, "rb") as f:
-            self.gene_median_dict = pickle.load(f)
-
-        # load token dictionary (Ensembl IDs:token)
-        with open(token_dictionary_file, "rb") as f:
-            self.gene_token_dict = pickle.load(f)
-
-        # check for special token in gene_token_dict
-        if self.special_token:
-            if ("<cls>" not in self.gene_token_dict.keys()) and (
-                "<eos>" not in self.gene_token_dict.keys()
-            ):
-                logger.error(
-                    "<cls> and <eos> required in gene_token_dict when special_token = True."
-                )
-                raise
-
-        if not self.special_token:
-            if ("<cls>" in self.gene_token_dict.keys()) and (
-                "<eos>" in self.gene_token_dict.keys()
-            ):
-                logger.warning(
-                    "<cls> and <eos> are in gene_token_dict but special_token = False. Please note that for 95M model series, special_token should be True."
-                )
-
-        # if collapsing duplicate gene IDs
-        self.collapse_gene_ids = collapse_gene_ids
-
-        # load gene mappings dictionary (Ensembl IDs:Ensembl ID)
-        if gene_mapping_file is not None:
-            with open(gene_mapping_file, "rb") as f:
-                self.gene_mapping_dict = pickle.load(f)
-        else:
-            self.gene_mapping_dict = {k: k for k, _ in self.gene_token_dict.items()}
-
-        # gene keys for full vocabulary
-        self.gene_keys = list(self.gene_token_dict.keys())
-
-        #  Filter gene mapping dict for items that exist in gene_token_dict
-        gene_keys_set = set(self.gene_token_dict.keys())
-        self.gene_mapping_dict = {
-            k: v for k, v in self.gene_mapping_dict.items() if v in gene_keys_set
-        }
-
-        # protein-coding and miRNA gene list dictionary for selecting .loom rows for tokenization
-        self.genelist_dict = dict(zip(self.gene_keys, [True] * len(self.gene_keys)))
-
-    def tokenize_data(
-        self,
-        data_directory: Path | str,
-        output_directory: Path | str,
-        output_prefix: str,
-        file_format: Literal["loom", "h5ad"] = "loom",
-        use_generator: bool = False,
-    ):
-        """
-        Tokenize .loom files in data_directory and save as tokenized .dataset in output_directory.
-        
-        **Parameters:**
-        
-        data_directory : Path
-            | Path to directory containing loom files or anndata files
-        output_directory : Path
-            | Path to directory where tokenized data will be saved as .dataset
-        output_prefix : str
-            | Prefix for output .dataset
-        file_format : str
-            | Format of input files. Can be "loom" or "h5ad".
-        use_generator : bool
-            | Whether to use generator or dict for tokenization.
-
-        """
-        tokenized_cells, cell_metadata = self.tokenize_files(
-            Path(data_directory), file_format
-        )
-        tokenized_dataset = self.create_dataset(
-            tokenized_cells,
-            cell_metadata,
-            use_generator=use_generator,
-        )
-
-        output_path = (Path(output_directory) / output_prefix).with_suffix(".dataset")
-        tokenized_dataset.save_to_disk(str(output_path))
-
-    def tokenize_files(
-        self, data_directory, file_format: Literal["loom", "h5ad"] = "loom"
-    ):
-        tokenized_cells = []
-        if self.custom_attr_name_dict is not None:
-            cell_attr = [attr_key for attr_key in self.custom_attr_name_dict.keys()]
-            cell_metadata = {
-                attr_key: [] for attr_key in self.custom_attr_name_dict.values()
-            }
-
-        # loops through directories to tokenize .loom files
-        file_found = 0
-        # loops through directories to tokenize .loom or .h5ad files
-        tokenize_file_fn = (
-            self.tokenize_loom if file_format == "loom" else self.tokenize_anndata
-        )
-        for file_path in data_directory.glob(f"*.{file_format}"):
-            file_found = 1
-            print(f"Tokenizing {file_path}")
-            file_tokenized_cells, file_cell_metadata = tokenize_file_fn(file_path)
-            tokenized_cells += file_tokenized_cells
-            if self.custom_attr_name_dict is not None:
-                for k in cell_attr:
-                    cell_metadata[self.custom_attr_name_dict[k]] += file_cell_metadata[
-                        k
-                    ]
-            else:
-                cell_metadata = None
-
-        if file_found == 0:
-            logger.error(
-                f"No .{file_format} files found in directory {data_directory}."
-            )
-            raise
-        return tokenized_cells, cell_metadata
-
-    def tokenize_anndata(self, adata_file_path, target_sum=10_000):
-        adata = sum_ensembl_ids(
-            adata_file_path,
-            self.collapse_gene_ids,
-            self.gene_mapping_dict,
-            self.gene_token_dict,
-            self.custom_attr_name_dict, 
-            file_format="h5ad",
-            chunk_size=self.chunk_size,
-        )
-
-        if self.custom_attr_name_dict is not None:
-            file_cell_metadata = {
-                attr_key: [] for attr_key in self.custom_attr_name_dict.keys()
-            }
-
-        coding_miRNA_loc = np.where(
-            [self.genelist_dict.get(i, False) for i in adata.var["ensembl_id_collapsed"]]
-        )[0]
-        norm_factor_vector = np.array(
-            [
-                self.gene_median_dict[i]
-                for i in adata.var["ensembl_id_collapsed"][coding_miRNA_loc]
-            ]
-        )
-        coding_miRNA_ids = adata.var["ensembl_id_collapsed"][coding_miRNA_loc]
-        coding_miRNA_tokens = np.array(
-            [self.gene_token_dict[i] for i in coding_miRNA_ids]
-        )
-
-        try:
-            _ = adata.obs["filter_pass"]
-        except KeyError:
-            var_exists = False
-        else:
-            var_exists = True
-
-        if var_exists:
-            filter_pass_loc = np.where([i == 1 for i in adata.obs["filter_pass"]])[0]
-        elif not var_exists:
-            print(
-                f"{adata_file_path} has no column attribute 'filter_pass'; tokenizing all cells."
-            )
-            filter_pass_loc = np.array([i for i in range(adata.shape[0])])
-
-        tokenized_cells = []
-
-        for i in range(0, len(filter_pass_loc), self.chunk_size):
-            idx = filter_pass_loc[i : i + self.chunk_size]
-
-            n_counts = adata[idx].obs["n_counts"].values[:, None]
-            X_view0 = adata[idx, :].X
-            X_view = X_view0[:, coding_miRNA_loc]
-            X_norm = X_view / n_counts * target_sum / norm_factor_vector
-            X_norm = sp.csr_matrix(X_norm)
-
-            tokenized_cells += [
-                rank_genes(X_norm[i].data, coding_miRNA_tokens[X_norm[i].indices])
-                for i in range(X_norm.shape[0])
-            ]
-
-            # add custom attributes for subview to dict
-            if self.custom_attr_name_dict is not None:
-                for k in file_cell_metadata.keys():
-                    file_cell_metadata[k] += adata[idx].obs[k].tolist()
-            else:
-                file_cell_metadata = None
-
-        return tokenized_cells, file_cell_metadata
-
-    def tokenize_loom(self, loom_file_path, target_sum=10_000):
-        if self.custom_attr_name_dict is not None:
-            file_cell_metadata = {
-                attr_key: [] for attr_key in self.custom_attr_name_dict.keys()
-            }
-        loom_file_path_original = loom_file_path
-
-        dedup_filename = loom_file_path.with_name(loom_file_path.stem + "__dedup.loom")
-        loom_file_path = sum_ensembl_ids(
-            loom_file_path,
-            self.collapse_gene_ids,
-            self.gene_mapping_dict,
-            self.gene_token_dict,
-            self.custom_attr_name_dict,
-            file_format="loom",
-            chunk_size=self.chunk_size,
-        )
-
-        with lp.connect(str(loom_file_path)) as data:
-            # define coordinates of detected protein-coding or miRNA genes and vector of their normalization factors
-            coding_miRNA_loc = np.where(
-                [self.genelist_dict.get(i, False) for i in data.ra["ensembl_id_collapsed"]]
-            )[0]
-            norm_factor_vector = np.array(
-                [
-                    self.gene_median_dict[i]
-                    for i in data.ra["ensembl_id_collapsed"][coding_miRNA_loc]
-                ]
-            )
-            coding_miRNA_ids = data.ra["ensembl_id_collapsed"][coding_miRNA_loc]
-            coding_miRNA_tokens = np.array(
-                [self.gene_token_dict[i] for i in coding_miRNA_ids]
-            )
-
-            # define coordinates of cells passing filters for inclusion (e.g. QC)
-            try:
-                data.ca["filter_pass"]
-            except AttributeError:
-                var_exists = False
-            else:
-                var_exists = True
-
-            if var_exists:
-                filter_pass_loc = np.where([i == 1 for i in data.ca["filter_pass"]])[0]
-            elif not var_exists:
-                print(
-                    f"{loom_file_path} has no column attribute 'filter_pass'; tokenizing all cells."
-                )
-                filter_pass_loc = np.array([i for i in range(data.shape[1])])
-
-            # scan through .loom files and tokenize cells
-            tokenized_cells = []
-            for _ix, _selection, view in data.scan(
-                items=filter_pass_loc, axis=1, batch_size=self.chunk_size
-            ):
-                # select subview with protein-coding and miRNA genes
-                subview = view.view[coding_miRNA_loc, :]
-
-                # normalize by total counts per cell and multiply by 10,000 to allocate bits to precision
-                # and normalize by gene normalization factors
-                subview_norm_array = (
-                    subview[:, :]
-                    / subview.ca.n_counts
-                    * target_sum
-                    / norm_factor_vector[:, None]
-                )
-                # tokenize subview gene vectors
-                tokenized_cells += [
-                    tokenize_cell(subview_norm_array[:, i], coding_miRNA_tokens)
-                    for i in range(subview_norm_array.shape[1])
-                ]
-
-                # add custom attributes for subview to dict
-                if self.custom_attr_name_dict is not None:
-                    for k in file_cell_metadata.keys():
-                        file_cell_metadata[k] += subview.ca[k].tolist()
-                else:
-                    file_cell_metadata = None
-
-        if str(dedup_filename) == str(loom_file_path):
-            os.remove(str(dedup_filename))
-
-        with lp.connect(str(loom_file_path_original)) as data:
-            if "ensembl_id_collapsed" in data.ra.keys():
-                del data.ra["ensembl_id_collapsed"]
-
-
-        return tokenized_cells, file_cell_metadata
-
-    def create_dataset(
-        self,
-        tokenized_cells,
-        cell_metadata,
-        use_generator=False,
-        keep_uncropped_input_ids=False,
-    ):
-        print("Creating dataset.")
-        # create dict for dataset creation
-        dataset_dict = {"input_ids": tokenized_cells}
-        if self.custom_attr_name_dict is not None:
-            dataset_dict.update(cell_metadata)
-
-        # create dataset
-        if use_generator:
-
-            def dict_generator():
-                for i in range(len(tokenized_cells)):
-                    yield {k: dataset_dict[k][i] for k in dataset_dict.keys()}
-
-            output_dataset = Dataset.from_generator(dict_generator, num_proc=self.nproc)
-        else:
-            output_dataset = Dataset.from_dict(dataset_dict)
-
-        def format_cell_features(example):
-            # Store original uncropped input_ids in separate feature
-            if keep_uncropped_input_ids:
-                example["input_ids_uncropped"] = example["input_ids"]
-                example["length_uncropped"] = len(example["input_ids"])
-
-            # Truncate/Crop input_ids to input size
-            if self.special_token:
-                example["input_ids"] = example["input_ids"][
-                    0 : self.model_input_size - 2
-                ]  # truncate to leave space for CLS and EOS token
-                example["input_ids"] = np.insert(
-                    example["input_ids"], 0, self.gene_token_dict.get("<cls>")
-                )
-                example["input_ids"] = np.insert(
-                    example["input_ids"],
-                    len(example["input_ids"]),
-                    self.gene_token_dict.get("<eos>"),
-                )
-            else:
-                # Truncate/Crop input_ids to input size
-                example["input_ids"] = example["input_ids"][0 : self.model_input_size]
-            example["length"] = len(example["input_ids"])
-
-            return example
-
-        output_dataset_truncated = output_dataset.map(
-            format_cell_features, num_proc=self.nproc
-        )
-        return output_dataset_truncated

gf-12L-30M-i2048/config.json

@@ -1,23 +0,0 @@
-{
-  "architectures": [
-    "BertForMaskedLM"
-  ],
-  "attention_probs_dropout_prob": 0.02,
-  "gradient_checkpointing": false,
-  "hidden_act": "relu",
-  "hidden_dropout_prob": 0.02,
-  "hidden_size": 512,
-  "initializer_range": 0.02,
-  "intermediate_size": 1024,
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 2048,
-  "model_type": "bert",
-  "num_attention_heads": 8,
-  "num_hidden_layers": 12,
-  "pad_token_id": 0,
-  "position_embedding_type": "absolute",
-  "transformers_version": "4.6.0",
-  "type_vocab_size": 2,
-  "use_cache": true,
-  "vocab_size": 25426
-}

gf-12L-30M-i2048/pytorch_model.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:812f8d85e5ecf9d64c268f052f6ece2c1906bc4f1aecf70d5144b2598386b615
-size 158467410

gf-12L-30M-i2048/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:259cf6067211e24e198690d00f0a222ee5550ad57e23d04ced0d0ca2e1b3738e
-size 2607

gf-12L-95M-i4096/config.json

@@ -1,24 +0,0 @@
-{
-  "architectures": [
-    "BertForMaskedLM"
-  ],
-  "attention_probs_dropout_prob": 0.02,
-  "classifier_dropout": null,
-  "hidden_act": "relu",
-  "hidden_dropout_prob": 0.02,
-  "hidden_size": 512,
-  "initializer_range": 0.02,
-  "intermediate_size": 1024,
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 4096,
-  "model_type": "bert",
-  "num_attention_heads": 8,
-  "num_hidden_layers": 12,
-  "pad_token_id": 0,
-  "position_embedding_type": "absolute",
-  "torch_dtype": "float32",
-  "transformers_version": "4.37.1",
-  "type_vocab_size": 2,
-  "use_cache": true,
-  "vocab_size": 20275
-}

gf-12L-95M-i4096/generation_config.json

@@ -1,5 +0,0 @@
-{
-  "_from_model_config": true,
-  "pad_token_id": 0,
-  "transformers_version": "4.37.1"
-}

gf-12L-95M-i4096/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4365ba23e393fcfa0e65a94ac64a0983cd788bd23a8d4914f4ab66f85cfe043c
-size 152012980

gf-12L-95M-i4096/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:21a45980734b138029422e95a5601def858821a9ec02cd473938b9f525ac108d
-size 4920

gf-20L-95M-i4096/config.json

@@ -1,24 +0,0 @@
-{
-  "architectures": [
-    "BertForMaskedLM"
-  ],
-  "attention_probs_dropout_prob": 0.02,
-  "classifier_dropout": null,
-  "hidden_act": "relu",
-  "hidden_dropout_prob": 0.02,
-  "hidden_size": 896,
-  "initializer_range": 0.02,
-  "intermediate_size": 1792,
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 4096,
-  "model_type": "bert",
-  "num_attention_heads": 14,
-  "num_hidden_layers": 20,
-  "pad_token_id": 0,
-  "position_embedding_type": "absolute",
-  "torch_dtype": "float32",
-  "transformers_version": "4.37.1",
-  "type_vocab_size": 2,
-  "use_cache": true,
-  "vocab_size": 20275
-}

gf-20L-95M-i4096/generation_config.json

@@ -1,5 +0,0 @@
-{
-  "_from_model_config": true,
-  "pad_token_id": 0,
-  "transformers_version": "4.37.1"
-}

gf-20L-95M-i4096/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:db85c081a6d392448955c7d0185e26aba74507518df991ca8c69ee9108ce8bbf
-size 605292732

gf-20L-95M-i4096/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5afed602918d6f0c4916c1b9335bcdb619bca2c6fd6c7e0dd2a86d195264b8cc
-size 5048

gf-20L-95M-i4096_config.json

@@ -1,24 +0,0 @@
-{
-  "architectures": [
-    "BertForMaskedLM"
-  ],
-  "attention_probs_dropout_prob": 0.02,
-  "classifier_dropout": null,
-  "hidden_act": "relu",
-  "hidden_dropout_prob": 0.02,
-  "hidden_size": 896,
-  "initializer_range": 0.02,
-  "intermediate_size": 1792,
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 4096,
-  "model_type": "bert",
-  "num_attention_heads": 14,
-  "num_hidden_layers": 20,
-  "pad_token_id": 0,
-  "position_embedding_type": "absolute",
-  "torch_dtype": "float32",
-  "transformers_version": "4.37.1",
-  "type_vocab_size": 2,
-  "use_cache": true,
-  "vocab_size": 20275
-}

gf-20L-95M-i4096_generation_config.json

@@ -1,5 +0,0 @@
-{
-  "_from_model_config": true,
-  "pad_token_id": 0,
-  "transformers_version": "4.37.1"
-}

gf-6L-30M-i2048/config.json

@@ -1,23 +0,0 @@
-{
-  "architectures": [
-    "BertForMaskedLM"
-  ],
-  "attention_probs_dropout_prob": 0.02,
-  "gradient_checkpointing": false,
-  "hidden_act": "relu",
-  "hidden_dropout_prob": 0.02,
-  "hidden_size": 256,
-  "initializer_range": 0.02,
-  "intermediate_size": 512,
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 2048,
-  "model_type": "bert",
-  "num_attention_heads": 4,
-  "num_hidden_layers": 6,
-  "pad_token_id": 0,
-  "position_embedding_type": "absolute",
-  "transformers_version": "4.6.0",
-  "type_vocab_size": 2,
-  "use_cache": true,
-  "vocab_size": 25426
-}

gf-6L-30M-i2048/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a5e33a757431643b3697de7ef6127950cdc49e06e58d4266b3a3ab191b683f14
-size 41183536

gf-6L-30M-i2048/pytorch_model.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8d860e2125884475dd42bc2cd9a0e60c60808a7351241e08f2154931ffc142da
-size 41216562

gf-6L-30M-i2048/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f0ec3459454205174c9d2e4d6c6930f6b0fbf3364fc03a6f4d99c4d3add2012b
-size 2607