Duplicate from simonduerr/diffdock

Co-authored-by: Simon Duerr <[email protected]>

.gitattributes

@@ -0,0 +1,23 @@
+*.ipynb linguist-vendored=false
+*.ipynb linguist-detectable=false
+/jupyter_notebooks linguist-vendored=false
+jupyter_notebooks/** linguist-vendored
+jupyter_notebooks/** linguist-vendored=false
+jupyter_notebooks/* linguist-vendored
+jupyter_notebooks/* linguist-vendored=false
+data/esm2_output_back/1a46_protein_processed.pdb_chain_1.pt filter=lfs diff=lfs merge=lfs -text
+data/esm2_output/1a46_protein_processed0t6cxvmshf54nq5e.pdb_chain_0.pt filter=lfs diff=lfs merge=lfs -text
+data/esm2_output/1a46_protein_processedhoiape3r6cbddini.pdb_chain_0.pt filter=lfs diff=lfs merge=lfs -text
+data/esm2_output/1a46_protein_processedhoiape3r6cbddini.pdb_chain_1.pt filter=lfs diff=lfs merge=lfs -text
+data/esm2_output/1a46_protein_processeds5c7cs3aj6rvbhy_.pdb_chain_0.pt filter=lfs diff=lfs merge=lfs -text
+data/esm2_output/1a46_protein_processedtyfuvx9z8y4_86ft.pdb_chain_0.pt filter=lfs diff=lfs merge=lfs -text
+data/esm2_output_back/1a46_protein_processed.pdb_chain_0.pt filter=lfs diff=lfs merge=lfs -text
+data/esm2_output/1a46_protein_processed.pdb_chain_0.pt filter=lfs diff=lfs merge=lfs -text
+data/esm2_output_back/1cbr_protein.pdb_chain_1.pt filter=lfs diff=lfs merge=lfs -text
+workdir/paper_confidence_model/best_model_epoch75.pt filter=lfs diff=lfs merge=lfs -text
+data/esm2_output/1a46_protein_processeds5c7cs3aj6rvbhy_.pdb_chain_1.pt filter=lfs diff=lfs merge=lfs -text
+data/esm2_output_back/1cbr_protein.pdb_chain_0.pt filter=lfs diff=lfs merge=lfs -text
+data/esm2_output/1a46_protein_processed0t6cxvmshf54nq5e.pdb_chain_1.pt filter=lfs diff=lfs merge=lfs -text
+data/esm2_output/1a46_protein_processed.pdb_chain_1.pt filter=lfs diff=lfs merge=lfs -text
+data/esm2_output/1a46_protein_processedtyfuvx9z8y4_86ft.pdb_chain_1.pt filter=lfs diff=lfs merge=lfs -text
+workdir/paper_score_model/best_ema_inference_epoch_model.pt filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,164 @@
+homework
+inference_out_dir_not_specified
+.plotly_cache
+.DS_store
+renew.sh
+tmux_renew.sh
+images
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+.so3_*
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+local_config_inference2.yml
+.vscode/
+
+
+*.zip
+
+.idea/
+
+
+#################### Project specific
+.p.npy
+.score.npy
+# this ignores everything in data except for the file
+!/data
+/data/*
+!/data/splits
+!/data/protein_ligand_example_csv.csv
+!/data/testset_csv.csv
+!/data/INDEX_general_PL_data.2020
+test_run
+
+cache
+wandb
+logs
+
+# temporary files
+.openbabel_cache
+temp/
+bsub*
+stderr*
+stdout*
+!/workdir
+/workdir/*
+!/workdir/paper_confidence_model
+!/workdir/paper_score_model
+runs2
+results
+# this excludes everything in the runs directory except for that specific run
+!/runs
+/runs/*
+!/runs/rigid_redocking
+!/runs/flexible_self_docking
+local_config.yml
+local_config_inference.yml
+local_config_confidence.yml
+temp1.py
+temp5.py
+temp3.py
+temp4.py
+temp5.py
+temp6.py
+temp7.py

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Gabriele Corso, Hannes Stärk, Bowen Jing
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,158 @@
+---
+title: Diffdock
+emoji: 🐠
+colorFrom: indigo
+colorTo: pink
+sdk: gradio
+sdk_version: 3.4.1
+app_file: app.py
+pinned: false
+license: mit
+duplicated_from: simonduerr/diffdock
+---
+
+# DiffDock: Diffusion Steps, Twists, and Turns for Molecular Docking
+
+### [Paper on arXiv](https://arxiv.org/abs/2210.01776)
+
+Implementation of DiffDock, state-of-the-art method for molecular docking, by Gabriele Corso*, Hannes Stark*, Bowen Jing*, Regina Barzilay and Tommi Jaakkola.
+This repository contains all code, instructions and model weights necessary to run the method or to retrain a model. 
+If you have any question, feel free to open an issue or reach out to us: [[email protected]]([email protected]), [[email protected]]([email protected]), [[email protected]]([email protected]).
+
+![Alt Text](visualizations/overview.png)
+
+The repository also contains all the scripts to run the baselines and generate the figures.
+Additionally, there are visualization videos in `visualizations`.
+
+
+
+# Dataset
+
+The files in `data` contain the names for the time-based data split.
+
+If you want to train one of our models with the data then: 
+1. download it from [zenodo](https://zenodo.org/record/6034088) 
+2. unzip the directory and place it into `data` such that you have the path `data/PDBBind_processed`
+
+
+
+## Setup Environment
+
+We will set up the environment using [Anaconda](https://docs.anaconda.com/anaconda/install/index.html). Clone the
+current repo
+
+    git clone https://github.com/gcorso/DiffDock.git
+
+Create a new environment with all required packages using `environment.yml`. While in the project directory run:
+
+    conda env create
+
+Activate the environment
+
+    conda activate diffdock
+
+If you want to install the packages yourself in case something does not work, these are the required ones:
+    
+    pytorch
+    pyg
+    pyyaml
+    scipy
+    networkx
+    biopython
+    rdkit-pypi
+    e3nn
+    spyrmsd
+    pandas
+    biopandas
+
+# Running DiffDock on your own complexes
+We support multiple input formats depending on whether you only want to make predictions for a single complex or for many at once.\
+The protein inputs need to be .pdb files. The ligand input can either be a SMILES string or a filetype that RDKit can read like `.sdf` or `.mol2`.
+
+For a single complex: specify the protein with, e.g., `--protein_path protein.pdb` and the ligand with `--ligand_path ligand.sdf` or `--ligand_smiles COc(cc1)ccc1C#N`
+
+For many complexes: create a csv file with paths to proteins and ligand files or SMILES. The first column of the .csv has to be called `protein_path` and the second one `ligand`.
+An example .csv is at `data/protein_ligand_example_csv.csv` and you would use it with `--protein_ligand_csv protein_ligand_example_csv.csv`.
+
+### Generate the ESM2 embeddings for the proteins
+We will soon also provide weights of a trained model without ESM2 embeddings such that this step is not necessary. Luckily, it is rather easy. First prepare a fasta for ESM2 (for a single protein use `--protein_path protein.pdb` instead):
+
+    python datasets/esm_embedding_preparation.py --protein_ligand_csv data/protein_ligand_example_csv.csv --out_file data/prepared_for_esm.fasta 
+    
+Generate the embeddings with ESM2 (assuming that you are in the DiffDock directory):
+
+    git clone https://github.com/facebookresearch/esm
+    cd esm
+    pip install -e .
+    cd ..
+    HOME=esm/model_weights python esm/scripts/extract.py esm2_t33_650M_UR50D data/prepared_for_esm.fasta data/esm2_output --repr_layers 33 --include per_tok
+    
+And done, that is it!
+
+### Run inference
+
+    python -m inference --protein_ligand_csv data/protein_ligand_example_csv.csv --out_dir results/user_predictions_small --inference_steps 20 --samples_per_complex 40 --batch_size 10
+
+
+
+# Running DiffDock to reproduce paper numbers
+Download the data and place it as described in the "Dataset" section above.
+
+### Generate the ESM2 embeddings for the proteins
+First run:
+
+    python datasets/pdbbind_lm_embedding_preparation.py
+
+Use the generated file `data/pdbbind_sequences.fasta` to generate the ESM2 language model embeddings using the library https://github.com/facebookresearch/esm by installing their repository and executing the following in their repository:
+
+    python scripts/extract.py esm2_t33_650M_UR50D pdbbind_sequences.fasta embeddings_output --repr_layers 33 --include per_tok
+
+This generates the `embeddings_output` directory which you have to copy into the `data` folder of our repository to have `data/embeddings_output`.
+Then run the command:
+
+    python datasets/esm_embeddings_to_pt.py
+
+### Using the provided model weights for evaluation
+To predict binding structures using the provided model weights run: 
+
+    python -m evaluate --model_dir workdir/paper_score_model --ckpt best_ema_inference_epoch_model.pt --confidence_ckpt best_model_epoch75.pt --confidence_model_dir workdir/paper_confidence_model --run_name DiffDockInference --inference_steps 20 --split_path data/splits/timesplit_test --samples_per_complex 40 --batch_size 10
+
+To additionally save the .sdf files of the generated molecules, add the flag `--save_visualisation`
+
+### Training a model yourself and using those weights
+Train the large score model:
+
+    python -m train --run_name big_score_model --test_sigma_intervals --esm_embeddings_path data/esm2_3billion_embeddings.pt --log_dir workdir --lr 1e-3 --tr_sigma_min 0.1 --tr_sigma_max 19 --rot_sigma_min 0.03 --rot_sigma_max 1.55 --batch_size 16 --ns 48 --nv 10 --num_conv_layers 6 --dynamic_max_cross --scheduler plateau --scale_by_sigma --dropout 0.1 --sampling_alpha 1 --sampling_beta 1 --remove_hs --c_alpha_max_neighbors 24 --receptor_radius 15 --num_dataloader_workers 1 --cudnn_benchmark --rot_alpha 1 --rot_beta 1 --tor_alpha 1 --tor_beta 1 --val_inference_freq 5 --num_inference_complexes 500 --use_ema --distance_embed_dim 64 --cross_distance_embed_dim 64 --sigma_embed_dim 64 --scheduler_patience 30 --n_epochs 850
+
+The model weights are saved in the `workdir` directory.
+
+Train a small score model with higher maximum translation sigma that will be used to generate the samples for training the confidence model:
+
+    python -m train --run_name small_score_model --test_sigma_intervals --esm_embeddings_path data/esm2_3billion_embeddings.pt --log_dir workdir --lr 1e-3 --tr_sigma_min 0.1 --tr_sigma_max 34 --rot_sigma_min 0.03 --rot_sigma_max 1.55 --batch_size 16 --ns 24 --nv 6 --num_conv_layers 5 --dynamic_max_cross --scheduler plateau --scale_by_sigma --dropout 0.1 --sampling_alpha 1 --sampling_beta 1 --remove_hs --c_alpha_max_neighbors 24 --receptor_radius 15 --num_dataloader_workers 1 --cudnn_benchmark --rot_alpha 1 --rot_beta 1 --tor_alpha 1 --tor_beta 1 --val_inference_freq 5 --num_inference_complexes 500 --use_ema --scheduler_patience 30 --n_epochs 300
+
+In practice, you could also likely achieve the same or better results by using the first score model for creating the samples to train the confidence model, but this is what we did in the paper.  
+The score model used to generate the samples to train the confidence model does not have to be the same as the score model that is used with that confidence model during inference.
+
+Train the confidence model by running the following:
+
+    python -m confidence.confidence_train --original_model_dir workdir/small_score_model --run_name confidence_model --inference_steps 20 --samples_per_complex 7 --inf_sched_alpha 1 --inf_sched_beta 1 --batch_size 16 --n_epochs 100 --lr 3e-4 --scheduler_patience 50 --tr_sigma_min 0.1 --tr_sigma_max 34 --rot_sigma_min 0.03 --rot_sigma_max 1.55 --ns 24 --nv 6 --num_conv_layers 5 --dynamic_max_cross --scale_by_sigma --dropout 0.1 --all_atoms --remove_hs --c_alpha_max_neighbors 24 --receptor_radius 15 --esm_embeddings_path data/esm2_3billion_embeddings.pt --main_metric loss --main_metric_goal min --best_model_save_frequency 5 --rmsd_classification_cutoff 2 --cache_creation_id 1 --cache_ids_to_combine 1 2 3 4
+
+first with `--cache_creation_id 1` then `--cache_creation_id 2` etc. up to 4
+
+Now everything is trained and you can run inference with:
+
+    python -m evaluate --model_dir workdir/big_score_model --ckpt best_ema_inference_epoch_model.pt --confidence_ckpt best_model_epoch75.pt --confidence_model_dir workdir/confidence_model --run_name DiffDockInference --inference_steps 20 --split_path data/splits/timesplit_test --samples_per_complex 40 --batch_size 10
+
+
+## Citation
+    @article{corso2022diffdock,
+          title={DiffDock: Diffusion Steps, Twists, and Turns for Molecular Docking}, 
+          author = {Corso, Gabriele and Stärk, Hannes and Jing, Bowen and Barzilay, Regina and Jaakkola, Tommi},
+          journal={arXiv preprint arXiv:2210.01776},
+          year={2022}
+    }
+
+## License
+MIT
+
+![Alt Text](visualizations/example_6agt_symmetric.gif)

app.py

@@ -0,0 +1,610 @@
+import gradio as gr
+import os
+
+import copy
+import os
+import torch
+
+import subprocess
+
+
+import time
+from argparse import ArgumentParser, Namespace, FileType
+from rdkit.Chem import RemoveHs
+from functools import partial
+import numpy as np
+import pandas as pd
+from rdkit import RDLogger
+from rdkit.Chem import MolFromSmiles, AddHs
+from torch_geometric.loader import DataLoader
+import yaml
+import sys
+import csv
+
+csv.field_size_limit(sys.maxsize)
+
+print(torch.__version__)
+os.makedirs("data/esm2_output", exist_ok=True)
+os.makedirs("results", exist_ok=True)
+from datasets.process_mols import (
+    read_molecule,
+    generate_conformer,
+    write_mol_with_coords,
+)
+from datasets.pdbbind import PDBBind
+from utils.diffusion_utils import t_to_sigma as t_to_sigma_compl, get_t_schedule
+from utils.sampling import randomize_position, sampling
+from utils.utils import get_model
+from utils.visualise import PDBFile
+from tqdm import tqdm
+from datasets.esm_embedding_preparation import esm_embedding_prep
+import subprocess
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+with open(f"workdir/paper_score_model/model_parameters.yml") as f:
+    score_model_args = Namespace(**yaml.full_load(f))
+
+with open(f"workdir/paper_confidence_model/model_parameters.yml") as f:
+    confidence_args = Namespace(**yaml.full_load(f))
+
+import shutil
+
+t_to_sigma = partial(t_to_sigma_compl, args=score_model_args)
+
+model = get_model(score_model_args, device, t_to_sigma=t_to_sigma, no_parallel=True)
+state_dict = torch.load(
+    f"workdir/paper_score_model/best_ema_inference_epoch_model.pt",
+    map_location=torch.device("cpu"),
+)
+model.load_state_dict(state_dict, strict=True)
+model = model.to(device)
+model.eval()
+
+confidence_model = get_model(
+    confidence_args,
+    device,
+    t_to_sigma=t_to_sigma,
+    no_parallel=True,
+    confidence_mode=True,
+)
+state_dict = torch.load(
+    f"workdir/paper_confidence_model/best_model_epoch75.pt",
+    map_location=torch.device("cpu"),
+)
+confidence_model.load_state_dict(state_dict, strict=True)
+confidence_model = confidence_model.to(device)
+confidence_model.eval()
+
+
+def get_pdb(pdb_code="", filepath=""):
+    try:
+        return filepath.name
+    except AttributeError as e:
+        if pdb_code is None or pdb_code == "":
+            return None
+        else:
+            os.system(f"wget -qnc https://files.rcsb.org/view/{pdb_code}.pdb")
+            return f"{pdb_code}.pdb"
+
+
+def get_ligand(smiles="", filepath=""):
+    if smiles is None or smiles == "":
+        try:
+            return filepath.name
+        except AttributeError as e:
+            return None
+    else:
+        return smiles
+
+
+def read_mol(molpath):
+    with open(molpath, "r") as fp:
+        lines = fp.readlines()
+    mol = ""
+    for l in lines:
+        mol += l
+    return mol
+
+
+def molecule(input_pdb, ligand_pdb, original_ligand):
+
+    structure = read_mol(input_pdb)
+    mol = read_mol(ligand_pdb)
+
+    try:
+        ligand = read_mol(original_ligand.name)
+        _, ext = os.path.splitext(original_ligand.name)
+        lig_str_1 = """let original_ligand = `""" + ligand + """`"""
+        lig_str_2 = f"""
+        viewer.addModel( original_ligand, "{ext[1:]}" );
+        viewer.getModel(2).setStyle({{stick:{{colorscheme:"greenCarbon"}}}});"""
+    except AttributeError as e:
+        ligand = None
+        lig_str_1 = ""
+        lig_str_2 = ""
+
+    x = (
+        """<!DOCTYPE html>
+        <html>
+        <head>    
+    <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+    <style>
+    body{
+        font-family:sans-serif
+    }
+    .mol-container {
+    width: 600px;
+    height: 600px;
+    position: relative;
+    mx-auto:0
+    }
+    .mol-container select{
+        background-image:None;
+    }
+    .green{
+        width:20px;
+        height:20px;
+        background-color:#33ff45;
+        display:inline-block;
+    }
+    .magenta{
+        width:20px;
+        height:20px;
+        background-color:magenta;
+        display:inline-block;
+    }
+    </style>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.3/jquery.min.js" integrity="sha512-STof4xm1wgkfm7heWqFJVn58Hm3EtS31XFaagaa8VMReCXAkQnJZ+jEy8PCC/iT18dFy95WcExNHFTqLyp72eQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+    <script src="https://3Dmol.csb.pitt.edu/build/3Dmol-min.js"></script>
+    </head>
+    <body>  
+     <button id="startanimation">Replay diffusion process</button> 
+     <button id="togglesurface">Toggle surface representation</button> 
+     <div>
+     <span class="green"></span> Uploaded ligand position
+     <span class="magenta"></span> Predicted ligand position
+     </div>
+    <div id="container" class="mol-container"></div>
+  
+            <script>
+               let ligand = `"""
+        + mol
+        + """`  
+        let structure = `"""
+        + structure
+        + """`
+        """
+        + lig_str_1
+        + """
+      
+             let viewer = null;
+             let surface = false;
+             let surf = null;
+             $(document).ready(function () {
+                let element = $("#container");
+                let config = { backgroundColor: "white" };
+                viewer = $3Dmol.createViewer(element, config);
+                viewer.addModel( structure, "pdb" );
+                viewer.setStyle({}, {cartoon: {color: "gray"}});
+                viewer.zoomTo();
+                viewer.zoom(0.7);
+                viewer.addModelsAsFrames(ligand, "pdb");
+                viewer.animate({loop: "forward",reps: 1});
+                
+                viewer.getModel(1).setStyle({stick:{colorscheme:"magentaCarbon"}});
+                """
+        + lig_str_2
+        + """
+                viewer.render();
+                
+              })
+
+              $("#startanimation").click(function() {
+                viewer.animate({loop: "forward",reps: 1});
+              });
+              $("#togglesurface").click(function() {
+               if (surface != true) {
+                    surf = viewer.addSurface($3Dmol.SurfaceType.VDW, { "opacity": 0.9, "color": "white" }, { model: 0 });  
+                    surface = true;
+                } else {
+                    viewer.removeAllSurfaces()
+                    surface = false;
+                }
+              });
+        </script>
+        </body></html>"""
+    )
+
+    return f"""<iframe style="width: 100%; height: 700px" name="result" allow="midi; geolocation; microphone; camera; 
+    display-capture; encrypted-media;" sandbox="allow-modals allow-forms 
+    allow-scripts allow-same-origin allow-popups 
+    allow-top-navigation-by-user-activation allow-downloads" allowfullscreen="" 
+    allowpaymentrequest="" frameborder="0" srcdoc='{x}'></iframe>"""
+
+
+import sys
+
+
+def esm(protein_path, out_file):
+    print("running esm")
+    esm_embedding_prep(out_file, protein_path)
+    # create args object with defaults
+    os.environ["HOME"] = "esm/model_weights"
+    subprocess.call(
+        f"python esm/scripts/extract.py esm2_t33_650M_UR50D {out_file} data/esm2_output --repr_layers 33 --include per_tok",
+        shell=True,
+        env=os.environ,
+    )
+
+
+def update(inp, file, ligand_inp, ligand_file, n_it, n_samples, actual_steps, no_final_step_noise):
+    pdb_path = get_pdb(inp, file)
+    ligand_path = get_ligand(ligand_inp, ligand_file)
+
+    esm(
+        pdb_path,
+        f"data/{os.path.basename(pdb_path)}_prepared_for_esm.fasta",
+    )
+    tr_schedule = get_t_schedule(inference_steps=n_it)
+    rot_schedule = tr_schedule
+    tor_schedule = tr_schedule
+    print("common t schedule", tr_schedule)
+    (
+        failures,
+        skipped,
+        confidences_list,
+        names_list,
+        run_times,
+        min_self_distances_list,
+    ) = (
+        0,
+        0,
+        [],
+        [],
+        [],
+        [],
+    )
+    N = n_samples  # number of samples to generate
+    protein_path_list = [pdb_path]
+    ligand_descriptions = [ligand_path]
+    no_random = False
+    ode = False
+    no_final_step_noise = no_final_step_noise
+    out_dir = "results/"
+    test_dataset = PDBBind(
+        transform=None,
+        root="",
+        protein_path_list=protein_path_list,
+        ligand_descriptions=ligand_descriptions,
+        receptor_radius=score_model_args.receptor_radius,
+        cache_path="data/cache",
+        remove_hs=score_model_args.remove_hs,
+        max_lig_size=None,
+        c_alpha_max_neighbors=score_model_args.c_alpha_max_neighbors,
+        matching=False,
+        keep_original=False,
+        popsize=score_model_args.matching_popsize,
+        maxiter=score_model_args.matching_maxiter,
+        all_atoms=score_model_args.all_atoms,
+        atom_radius=score_model_args.atom_radius,
+        atom_max_neighbors=score_model_args.atom_max_neighbors,
+        esm_embeddings_path="data/esm2_output",
+        require_ligand=True,
+        num_workers=1,
+        keep_local_structures=False,
+    )
+    test_loader = DataLoader(dataset=test_dataset, batch_size=1, shuffle=False)
+    confidence_test_dataset = PDBBind(
+        transform=None,
+        root="",
+        protein_path_list=protein_path_list,
+        ligand_descriptions=ligand_descriptions,
+        receptor_radius=confidence_args.receptor_radius,
+        cache_path="data/cache",
+        remove_hs=confidence_args.remove_hs,
+        max_lig_size=None,
+        c_alpha_max_neighbors=confidence_args.c_alpha_max_neighbors,
+        matching=False,
+        keep_original=False,
+        popsize=confidence_args.matching_popsize,
+        maxiter=confidence_args.matching_maxiter,
+        all_atoms=confidence_args.all_atoms,
+        atom_radius=confidence_args.atom_radius,
+        atom_max_neighbors=confidence_args.atom_max_neighbors,
+        esm_embeddings_path="data/esm2_output",
+        require_ligand=True,
+        num_workers=1,
+    )
+    confidence_complex_dict = {d.name: d for d in confidence_test_dataset}
+    for idx, orig_complex_graph in tqdm(enumerate(test_loader)):
+        if (
+            confidence_model is not None
+            and not (
+                confidence_args.use_original_model_cache
+                or confidence_args.transfer_weights
+            )
+            and orig_complex_graph.name[0] not in confidence_complex_dict.keys()
+        ):
+            skipped += 1
+            print(
+                f"HAPPENING | The confidence dataset did not contain {orig_complex_graph.name[0]}. We are skipping this complex."
+            )
+            continue
+        try:
+            data_list = [copy.deepcopy(orig_complex_graph) for _ in range(N)]
+            randomize_position(
+                data_list,
+                score_model_args.no_torsion,
+                no_random,
+                score_model_args.tr_sigma_max,
+            )
+            pdb = None
+            lig = orig_complex_graph.mol[0]
+            visualization_list = []
+            for graph in data_list:
+                pdb = PDBFile(lig)
+                pdb.add(lig, 0, 0)
+                pdb.add(
+                    (
+                        orig_complex_graph["ligand"].pos
+                        + orig_complex_graph.original_center
+                    )
+                    .detach()
+                    .cpu(),
+                    1,
+                    0,
+                )
+                pdb.add(
+                    (graph["ligand"].pos + graph.original_center).detach().cpu(),
+                    part=1,
+                    order=1,
+                )
+                visualization_list.append(pdb)
+
+            start_time = time.time()
+            if confidence_model is not None and not (
+                confidence_args.use_original_model_cache
+                or confidence_args.transfer_weights
+            ):
+                confidence_data_list = [
+                    copy.deepcopy(confidence_complex_dict[orig_complex_graph.name[0]])
+                    for _ in range(N)
+                ]
+            else:
+                confidence_data_list = None
+
+            data_list, confidence = sampling(
+                data_list=data_list,
+                model=model,
+                inference_steps=actual_steps,
+                tr_schedule=tr_schedule,
+                rot_schedule=rot_schedule,
+                tor_schedule=tor_schedule,
+                device=device,
+                t_to_sigma=t_to_sigma,
+                model_args=score_model_args,
+                no_random=no_random,
+                ode=ode,
+                visualization_list=visualization_list,
+                confidence_model=confidence_model,
+                confidence_data_list=confidence_data_list,
+                confidence_model_args=confidence_args,
+                batch_size=1,
+                no_final_step_noise=no_final_step_noise,
+            )
+            ligand_pos = np.asarray(
+                [
+                    complex_graph["ligand"].pos.cpu().numpy()
+                    + orig_complex_graph.original_center.cpu().numpy()
+                    for complex_graph in data_list
+                ]
+            )
+            run_times.append(time.time() - start_time)
+
+            if confidence is not None and isinstance(
+                confidence_args.rmsd_classification_cutoff, list
+            ):
+                confidence = confidence[:, 0]
+            if confidence is not None:
+                confidence = confidence.cpu().numpy()
+                re_order = np.argsort(confidence)[::-1]
+                confidence = confidence[re_order]
+                confidences_list.append(confidence)
+                ligand_pos = ligand_pos[re_order]
+            write_dir = (
+                f'{out_dir}/index{idx}_{data_list[0]["name"][0].replace("/","-")}'
+            )
+            os.makedirs(write_dir, exist_ok=True)
+            confidences = []
+            for rank, pos in enumerate(ligand_pos):
+                mol_pred = copy.deepcopy(lig)
+                if score_model_args.remove_hs:
+                    mol_pred = RemoveHs(mol_pred)
+                if rank == 0:
+                    write_mol_with_coords(
+                        mol_pred, pos, os.path.join(write_dir, f"rank{rank+1}.sdf")
+                    )
+                confidences.append(confidence[rank])
+                write_mol_with_coords(
+                    mol_pred,
+                    pos,
+                    os.path.join(
+                        write_dir, f"rank{rank+1}_confidence{confidence[rank]:.2f}.sdf"
+                    ),
+                )
+            self_distances = np.linalg.norm(
+                ligand_pos[:, :, None, :] - ligand_pos[:, None, :, :], axis=-1
+            )
+            self_distances = np.where(
+                np.eye(self_distances.shape[2]), np.inf, self_distances
+            )
+            min_self_distances_list.append(np.min(self_distances, axis=(1, 2)))
+
+            filenames = []
+            if confidence is not None:
+                for rank, batch_idx in enumerate(re_order):
+                    visualization_list[batch_idx].write(
+                        os.path.join(write_dir, f"rank{rank+1}_reverseprocess.pdb")
+                    )
+                    filenames.append(
+                        os.path.join(write_dir, f"rank{rank+1}_reverseprocess.pdb")
+                    )
+            else:
+                for rank, batch_idx in enumerate(ligand_pos):
+                    visualization_list[batch_idx].write(
+                        os.path.join(write_dir, f"rank{rank+1}_reverseprocess.pdb")
+                    )
+                    filenames.append(
+                        os.path.join(write_dir, f"rank{rank+1}_reverseprocess.pdb")
+                    )
+            names_list.append(orig_complex_graph.name[0])
+        except Exception as e:
+            print("Failed on", orig_complex_graph["name"], e)
+            failures += 1
+            return None
+    # zip outputs
+    zippath = shutil.make_archive(
+        os.path.join("results", os.path.basename(pdb_path)), "zip", write_dir
+    )
+    print("Zipped outputs to", zippath)
+    labels = [
+        f"rank {i+1}, confidence {confidences[i]:.2f}" for i in range(len(filenames))
+    ]
+
+    torch.cuda.empty_cache()
+    return (
+        molecule(pdb_path, filenames[0], ligand_file),
+        gr.Dropdown.update(choices=labels, value=labels[0]),
+        filenames,
+        pdb_path,
+        zippath,
+    )
+
+
+def updateView(out, filenames, pdb, ligand_file):
+    print("updating view")
+    i = out  # int(out.replace("rank", ""))
+    print(i)
+    i = int(i.split(",")[0].replace("rank", "")) - 1
+    return molecule(pdb, filenames[i], ligand_file)
+
+
+demo = gr.Blocks()
+
+with demo:
+    gr.Markdown("# DiffDock")
+    gr.Markdown(
+        ">**DiffDock: Diffusion Steps, Twists, and Turns for Molecular Docking**, Corso, Gabriele and Stärk, Hannes and Jing, Bowen and Barzilay, Regina and Jaakkola, Tommi, arXiv:2210.01776  [GitHub](https://github.com/gcorso/diffdock)"
+    )
+    gr.Markdown("")
+    with gr.Box():
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("## Protein")
+                inp = gr.Textbox(
+                    placeholder="PDB Code or upload file below", label="Input structure"
+                )
+                file = gr.File(file_count="single", label="Input PDB")
+            with gr.Column():
+                gr.Markdown("## Ligand")
+                ligand_inp = gr.Textbox(
+                    placeholder="Provide SMILES input or upload mol2/sdf file below",
+                    label="SMILES string",
+                )
+                ligand_file = gr.File(file_count="single", label="Input Ligand")
+        n_it = gr.Slider(value=20,
+            minimum=10, maximum=40, label="Number of inference steps", step=1
+        )
+        actual_steps = gr.Slider(value=18,
+            minimum=10, maximum=40, label="Number of actual inference steps", step=1
+        )
+        n_samples = gr.Slider(value=40,
+            minimum=10, maximum=40, label="Number of samples", step=1
+        )
+        no_final_step_noise = gr.Checkbox(value=True,label="No final step noise"
+        )
+
+    btn = gr.Button("Run predictions")
+
+    gr.Markdown("## Output")
+    pdb = gr.Variable()
+    filenames = gr.Variable()
+    out = gr.Dropdown(interactive=True, label="Ranked samples")
+    mol = gr.HTML()
+    output_file = gr.File(file_count="single", label="Output files")
+    gr.Examples(
+        [
+            [
+                "6w70",
+                "examples/6w70.pdb",
+                "COc1ccc(cc1)n2c3c(c(n2)C(=O)N)CCN(C3=O)c4ccc(cc4)N5CCCCC5=O",
+                "examples/6w70_ligand.sdf",
+                20,
+                10,
+                18,
+                True
+            ],
+            [
+                "6moa",
+                "examples/6moa_protein_processed.pdb",
+                "",
+                "examples/6moa_ligand.sdf",
+                20,
+                10,
+                18,
+                True
+            ],
+            [
+                "",
+                "examples/6o5u_protein_processed.pdb",
+                "",
+                "examples/6o5u_ligand.sdf",
+                20,
+                10,
+                18,
+                True
+            ],
+            [
+                "",
+                "examples/6o5u_protein_processed.pdb",
+                "[NH3+]C[C@H]1O[C@H](O[C@@H]2[C@@H]([NH3+])C[C@H]([C@@H]([C@H]2O)O[C@H]2O[C@H](CO)[C@H]([C@@H]([C@H]2O)[NH3+])O)[NH3+])[C@@H]([C@H]([C@@H]1O)O)O",
+                "examples/6o5u_ligand.sdf",
+                20,
+                10,
+                18,
+                True
+            ],
+            [
+                "",
+                "examples/6o5u_protein_processed.pdb",
+                "",
+                "examples/6o5u_ligand.sdf",
+                20,
+                10,
+                18,
+                True
+            ],
+            [
+                "",
+                "examples/6ahs_protein_processed.pdb",
+                "",
+                "examples/6ahs_ligand.sdf",
+                20,
+                10,
+                18,
+                True
+            ],
+        ],
+        [inp, file, ligand_inp, ligand_file, n_it, n_samples, actual_steps, no_final_step_noise],
+        [mol, out, filenames, pdb, output_file],
+        # fn=update,
+        # cache_examples=True,
+    )
+    btn.click(
+        fn=update,
+        inputs=[inp, file, ligand_inp, ligand_file, n_it, n_samples, actual_steps, no_final_step_noise],
+        outputs=[mol, out, filenames, pdb, output_file],
+    )
+    out.change(fn=updateView, inputs=[out, filenames, pdb, ligand_file], outputs=mol)
+demo.launch()

baselines/baseline_evaluation.py

@@ -0,0 +1,219 @@
+# small script to extract the ligand and save it in a separate file because GNINA will use the ligand position as initial pose
+import os
+
+import plotly.express as px
+import time
+from argparse import FileType, ArgumentParser
+
+import numpy as np
+import pandas as pd
+import wandb
+from biopandas.pdb import PandasPdb
+from rdkit import Chem
+
+from tqdm import tqdm
+
+from datasets.pdbbind import read_mol
+from datasets.process_mols import read_molecule
+from utils.utils import read_strings_from_txt, get_symmetry_rmsd
+
+parser = ArgumentParser()
+parser.add_argument('--config', type=FileType(mode='r'), default=None)
+parser.add_argument('--run_name', type=str, default='gnina_results', help='')
+parser.add_argument('--data_dir', type=str, default='data/PDBBind_processed', help='')
+parser.add_argument('--results_path', type=str, default='results/user_inference', help='Path to folder with trained model and hyperparameters')
+parser.add_argument('--file_suffix', type=str, default='_baseline_ligand.pdb', help='Path to folder with trained model and hyperparameters')
+parser.add_argument('--project', type=str, default='ligbind_inf', help='')
+parser.add_argument('--wandb', action='store_true', default=False, help='')
+parser.add_argument('--file_to_exclude', type=str, default=None, help='')
+parser.add_argument('--all_dirs_in_results', action='store_true', default=True, help='Evaluate all directories in the results path instead of using directly looking for the names')
+parser.add_argument('--num_predictions', type=int, default=10, help='')
+parser.add_argument('--no_id_in_filename', action='store_true', default=False, help='')
+args = parser.parse_args()
+
+print('Reading paths and names.')
+names = read_strings_from_txt(f'data/splits/timesplit_test')
+names_no_rec_overlap = read_strings_from_txt(f'data/splits/timesplit_test_no_rec_overlap')
+results_path_containments = os.listdir(args.results_path)
+
+if args.wandb:
+    wandb.init(
+        entity='coarse-graining-mit',
+        settings=wandb.Settings(start_method="fork"),
+        project=args.project,
+        name=args.run_name,
+        config=args
+    )
+
+all_times = []
+successful_names_list = []
+rmsds_list = []
+centroid_distances_list = []
+min_cross_distances_list = []
+min_self_distances_list = []
+without_rec_overlap_list = []
+start_time = time.time()
+for i, name in enumerate(tqdm(names)):
+    mol = read_mol(args.data_dir, name, remove_hs=True)
+    mol = Chem.RemoveAllHs(mol)
+    orig_ligand_pos = np.array(mol.GetConformer().GetPositions())
+
+    if args.all_dirs_in_results:
+        directory_with_name = [directory for directory in results_path_containments if name in directory][0]
+        ligand_pos = []
+        for i in range(args.num_predictions):
+            file_paths = os.listdir(os.path.join(args.results_path, directory_with_name))
+            file_path = [path for path in file_paths if f'rank{i+1}' in path][0]
+            if args.file_to_exclude is not None and args.file_to_exclude in file_path: continue
+            mol_pred = read_molecule(os.path.join(args.results_path, directory_with_name, file_path),remove_hs=True, sanitize=True)
+            mol_pred = Chem.RemoveAllHs(mol_pred)
+            ligand_pos.append(mol_pred.GetConformer().GetPositions())
+        ligand_pos = np.asarray(ligand_pos)
+    else:
+        if not os.path.exists(os.path.join(args.results_path, name, f'{"" if args.no_id_in_filename else name}{args.file_suffix}')): raise Exception('path did not exists:', os.path.join(args.results_path, name, f'{"" if args.no_id_in_filename else name}{args.file_suffix}'))
+        mol_pred = read_molecule(os.path.join(args.results_path, name, f'{"" if args.no_id_in_filename else name}{args.file_suffix}'), remove_hs=True, sanitize=True)
+        if mol_pred == None:
+            print("Skipping ", name, ' because RDKIT could not read it.')
+            continue
+        mol_pred = Chem.RemoveAllHs(mol_pred)
+        ligand_pos = np.asarray([np.array(mol_pred.GetConformer(i).GetPositions()) for i in range(args.num_predictions)])
+    try:
+        rmsd = get_symmetry_rmsd(mol, orig_ligand_pos, [l for l in ligand_pos], mol_pred)
+    except Exception as e:
+        print("Using non corrected RMSD because of the error:", e)
+        rmsd = np.sqrt(((ligand_pos - orig_ligand_pos) ** 2).sum(axis=2).mean(axis=1))
+
+    rmsds_list.append(rmsd)
+    centroid_distances_list.append(np.linalg.norm(ligand_pos.mean(axis=1) - orig_ligand_pos[None,:].mean(axis=1), axis=1))
+
+    rec_path = os.path.join(args.data_dir, name, f'{name}_protein_processed.pdb')
+    if not os.path.exists(rec_path):
+        rec_path = os.path.join(args.data_dir, name,f'{name}_protein_obabel_reduce.pdb')
+    rec = PandasPdb().read_pdb(rec_path)
+    rec_df = rec.df['ATOM']
+    receptor_pos = rec_df[['x_coord', 'y_coord', 'z_coord']].to_numpy().squeeze().astype(np.float32)
+    receptor_pos = np.tile(receptor_pos, (args.num_predictions, 1, 1))
+
+    cross_distances = np.linalg.norm(receptor_pos[:, :, None, :] - ligand_pos[:, None, :, :], axis=-1)
+    self_distances = np.linalg.norm(ligand_pos[:, :, None, :] - ligand_pos[:, None, :, :], axis=-1)
+    self_distances =  np.where(np.eye(self_distances.shape[2]), np.inf, self_distances)
+    min_cross_distances_list.append(np.min(cross_distances, axis=(1,2)))
+    min_self_distances_list.append(np.min(self_distances, axis=(1, 2)))
+    successful_names_list.append(name)
+    without_rec_overlap_list.append(1 if name in names_no_rec_overlap else 0)
+performance_metrics = {}
+for overlap in ['', 'no_overlap_']:
+    if 'no_overlap_' == overlap:
+        without_rec_overlap = np.array(without_rec_overlap_list, dtype=bool)
+        rmsds = np.array(rmsds_list)[without_rec_overlap]
+        centroid_distances = np.array(centroid_distances_list)[without_rec_overlap]
+        min_cross_distances = np.array(min_cross_distances_list)[without_rec_overlap]
+        min_self_distances = np.array(min_self_distances_list)[without_rec_overlap]
+        successful_names = np.array(successful_names_list)[without_rec_overlap]
+    else:
+        rmsds = np.array(rmsds_list)
+        centroid_distances = np.array(centroid_distances_list)
+        min_cross_distances = np.array(min_cross_distances_list)
+        min_self_distances = np.array(min_self_distances_list)
+        successful_names = np.array(successful_names_list)
+
+    np.save(os.path.join(args.results_path, f'{overlap}rmsds.npy'), rmsds)
+    np.save(os.path.join(args.results_path, f'{overlap}names.npy'), successful_names)
+    np.save(os.path.join(args.results_path, f'{overlap}min_cross_distances.npy'), np.array(min_cross_distances))
+    np.save(os.path.join(args.results_path, f'{overlap}min_self_distances.npy'), np.array(min_self_distances))
+
+    performance_metrics.update({
+        f'{overlap}steric_clash_fraction': (100 * (min_cross_distances < 0.4).sum() / len(min_cross_distances) / args.num_predictions).__round__(2),
+        f'{overlap}self_intersect_fraction': (100 * (min_self_distances < 0.4).sum() / len(min_self_distances) / args.num_predictions).__round__(2),
+        f'{overlap}mean_rmsd': rmsds[:,0].mean(),
+        f'{overlap}rmsds_below_2': (100 * (rmsds[:,0] < 2).sum() / len(rmsds[:,0])),
+        f'{overlap}rmsds_below_5': (100 * (rmsds[:,0] < 5).sum() / len(rmsds[:,0])),
+        f'{overlap}rmsds_percentile_25': np.percentile(rmsds[:,0], 25).round(2),
+        f'{overlap}rmsds_percentile_50': np.percentile(rmsds[:,0], 50).round(2),
+        f'{overlap}rmsds_percentile_75': np.percentile(rmsds[:,0], 75).round(2),
+
+        f'{overlap}mean_centroid': centroid_distances[:,0].mean().__round__(2),
+        f'{overlap}centroid_below_2': (100 * (centroid_distances[:,0] < 2).sum() / len(centroid_distances[:,0])).__round__(2),
+        f'{overlap}centroid_below_5': (100 * (centroid_distances[:,0] < 5).sum() / len(centroid_distances[:,0])).__round__(2),
+        f'{overlap}centroid_percentile_25': np.percentile(centroid_distances[:,0], 25).round(2),
+        f'{overlap}centroid_percentile_50': np.percentile(centroid_distances[:,0], 50).round(2),
+        f'{overlap}centroid_percentile_75': np.percentile(centroid_distances[:,0], 75).round(2),
+    })
+
+    top5_rmsds = np.min(rmsds[:, :5], axis=1)
+    top5_centroid_distances = centroid_distances[np.arange(rmsds.shape[0])[:,None],np.argsort(rmsds[:, :5], axis=1)][:,0]
+    top5_min_cross_distances = min_cross_distances[np.arange(rmsds.shape[0])[:,None],np.argsort(rmsds[:, :5], axis=1)][:,0]
+    top5_min_self_distances = min_self_distances[np.arange(rmsds.shape[0])[:,None],np.argsort(rmsds[:, :5], axis=1)][:,0]
+    performance_metrics.update({
+        f'{overlap}top5_steric_clash_fraction': (100 * (top5_min_cross_distances < 0.4).sum() / len(top5_min_cross_distances)).__round__(2),
+        f'{overlap}top5_self_intersect_fraction': (100 * (top5_min_self_distances < 0.4).sum() / len(top5_min_self_distances)).__round__(2),
+        f'{overlap}top5_rmsds_below_2': (100 * (top5_rmsds < 2).sum() / len(top5_rmsds)).__round__(2),
+        f'{overlap}top5_rmsds_below_5': (100 * (top5_rmsds < 5).sum() / len(top5_rmsds)).__round__(2),
+        f'{overlap}top5_rmsds_percentile_25': np.percentile(top5_rmsds, 25).round(2),
+        f'{overlap}top5_rmsds_percentile_50': np.percentile(top5_rmsds, 50).round(2),
+        f'{overlap}top5_rmsds_percentile_75': np.percentile(top5_rmsds, 75).round(2),
+
+        f'{overlap}top5_centroid_below_2': (100 * (top5_centroid_distances < 2).sum() / len(top5_centroid_distances)).__round__(2),
+        f'{overlap}top5_centroid_below_5': (100 * (top5_centroid_distances < 5).sum() / len(top5_centroid_distances)).__round__(2),
+        f'{overlap}top5_centroid_percentile_25': np.percentile(top5_centroid_distances, 25).round(2),
+        f'{overlap}top5_centroid_percentile_50': np.percentile(top5_centroid_distances, 50).round(2),
+        f'{overlap}top5_centroid_percentile_75': np.percentile(top5_centroid_distances, 75).round(2),
+    })
+
+
+    top10_rmsds = np.min(rmsds[:, :10], axis=1)
+    top10_centroid_distances = centroid_distances[np.arange(rmsds.shape[0])[:,None],np.argsort(rmsds[:, :10], axis=1)][:,0]
+    top10_min_cross_distances = min_cross_distances[np.arange(rmsds.shape[0])[:,None],np.argsort(rmsds[:, :10], axis=1)][:,0]
+    top10_min_self_distances = min_self_distances[np.arange(rmsds.shape[0])[:,None],np.argsort(rmsds[:, :10], axis=1)][:,0]
+    performance_metrics.update({
+        f'{overlap}top10_self_intersect_fraction': (100 * (top10_min_self_distances < 0.4).sum() / len(top10_min_self_distances)).__round__(2),
+        f'{overlap}top10_steric_clash_fraction': ( 100 * (top10_min_cross_distances < 0.4).sum() / len(top10_min_cross_distances)).__round__(2),
+        f'{overlap}top10_rmsds_below_2': (100 * (top10_rmsds < 2).sum() / len(top10_rmsds)).__round__(2),
+        f'{overlap}top10_rmsds_below_5': (100 * (top10_rmsds < 5).sum() / len(top10_rmsds)).__round__(2),
+        f'{overlap}top10_rmsds_percentile_25': np.percentile(top10_rmsds, 25).round(2),
+        f'{overlap}top10_rmsds_percentile_50': np.percentile(top10_rmsds, 50).round(2),
+        f'{overlap}top10_rmsds_percentile_75': np.percentile(top10_rmsds, 75).round(2),
+
+        f'{overlap}top10_centroid_below_2': (100 * (top10_centroid_distances < 2).sum() / len(top10_centroid_distances)).__round__(2),
+        f'{overlap}top10_centroid_below_5': (100 * (top10_centroid_distances < 5).sum() / len(top10_centroid_distances)).__round__(2),
+        f'{overlap}top10_centroid_percentile_25': np.percentile(top10_centroid_distances, 25).round(2),
+        f'{overlap}top10_centroid_percentile_50': np.percentile(top10_centroid_distances, 50).round(2),
+        f'{overlap}top10_centroid_percentile_75': np.percentile(top10_centroid_distances, 75).round(2),
+    })
+for k in performance_metrics:
+    print(k, performance_metrics[k])
+
+if args.wandb:
+    wandb.log(performance_metrics)
+    histogram_metrics_list = [('rmsd', rmsds[:,0]),
+                              ('centroid_distance', centroid_distances[:,0]),
+                              ('mean_rmsd', rmsds[:,0]),
+                              ('mean_centroid_distance', centroid_distances[:,0])]
+    histogram_metrics_list.append(('top5_rmsds', top5_rmsds))
+    histogram_metrics_list.append(('top5_centroid_distances', top5_centroid_distances))
+    histogram_metrics_list.append(('top10_rmsds', top10_rmsds))
+    histogram_metrics_list.append(('top10_centroid_distances', top10_centroid_distances))
+
+    os.makedirs(f'.plotly_cache/baseline_cache', exist_ok=True)
+    images = []
+    for metric_name, metric in histogram_metrics_list:
+        d = {args.results_path: metric}
+        df = pd.DataFrame(data=d)
+        fig = px.ecdf(df, width=900, height=600, range_x=[0, 40])
+        fig.add_vline(x=2, annotation_text='2 A;', annotation_font_size=20, annotation_position="top right",
+                      line_dash='dash', line_color='firebrick', annotation_font_color='firebrick')
+        fig.add_vline(x=5, annotation_text='5 A;', annotation_font_size=20, annotation_position="top right",
+                      line_dash='dash', line_color='green', annotation_font_color='green')
+        fig.update_xaxes(title=f'{metric_name} in Angstrom', title_font={"size": 20}, tickfont={"size": 20})
+        fig.update_yaxes(title=f'Fraction of predictions with lower error', title_font={"size": 20},
+                         tickfont={"size": 20})
+        fig.update_layout(autosize=False, margin={'l': 0, 'r': 0, 't': 0, 'b': 0}, plot_bgcolor='white',
+                          paper_bgcolor='white', legend_title_text='Method', legend_title_font_size=17,
+                          legend=dict(yanchor="bottom", y=0.1, xanchor="right", x=0.99, font=dict(size=17), ), )
+        fig.update_xaxes(showgrid=True, gridcolor='lightgrey')
+        fig.update_yaxes(showgrid=True, gridcolor='lightgrey')
+
+        fig.write_image(os.path.join(f'.plotly_cache/baseline_cache', f'{metric_name}.png'))
+        wandb.log({metric_name: wandb.Image(os.path.join(f'.plotly_cache/baseline_cache', f'{metric_name}.png'), caption=f"{metric_name}")})
+        images.append(wandb.Image(os.path.join(f'.plotly_cache/baseline_cache', f'{metric_name}.png'), caption=f"{metric_name}"))
+    wandb.log({'images': images})

baselines/baseline_gnina.py

@@ -0,0 +1,175 @@
+# small script to extract the ligand and save it in a separate file because GNINA will use the ligand position as
+# initial pose
+import os
+import shutil
+import subprocess
+import sys
+
+import time
+from argparse import ArgumentParser, FileType
+from datetime import datetime
+
+import numpy as np
+import pandas as pd
+from biopandas.pdb import PandasPdb
+from rdkit import Chem
+from rdkit.Chem import AllChem, MolToPDBFile
+from scipy.spatial.distance import cdist
+
+from datasets.pdbbind import read_mol
+from utils.utils import read_strings_from_txt
+
+parser = ArgumentParser()
+parser.add_argument('--data_dir', type=str, default='data/PDBBind_processed', help='')
+parser.add_argument('--file_suffix', type=str, default='_baseline_ligand', help='Path to folder with trained model and hyperparameters')
+parser.add_argument('--results_path', type=str, default='results/gnina_predictions', help='')
+parser.add_argument('--complex_names_path', type=str, default='data/splits/timesplit_test', help='')
+parser.add_argument('--seed_molecules_path', type=str, default=None, help='Use the molecules at seed molecule path as initialization and only search around them')
+parser.add_argument('--seed_molecule_filename', type=str, default='equibind_corrected.sdf', help='Use the molecules at seed molecule path as initialization and only search around them')
+parser.add_argument('--smina', action='store_true', default=False, help='')
+parser.add_argument('--no_gpu', action='store_true', default=False, help='')
+parser.add_argument('--exhaustiveness', type=int, default=8, help='')
+parser.add_argument('--num_cpu', type=int, default=16, help='')
+parser.add_argument('--pocket_mode', action='store_true', default=False, help='')
+parser.add_argument('--pocket_cutoff', type=int, default=5, help='')
+parser.add_argument('--num_modes', type=int, default=10, help='')
+parser.add_argument('--autobox_add', type=int, default=4, help='')
+parser.add_argument('--use_p2rank_pocket', action='store_true', default=False, help='')
+parser.add_argument('--skip_p2rank', action='store_true', default=False, help='')
+parser.add_argument('--prank_path', type=str, default='/Users/hstark/projects/p2rank_2.3/prank', help='')
+parser.add_argument('--skip_existing', action='store_true', default=False, help='')
+
+
+
+
+
+args = parser.parse_args()
+
+class Logger(object):
+    def __init__(self, logpath, syspart=sys.stdout):
+        self.terminal = syspart
+        self.log = open(logpath, "a")
+
+    def write(self, message):
+        self.terminal.write(message)
+        self.log.write(message)
+        self.log.flush()
+
+    def flush(self):
+        # this flush method is needed for python 3 compatibility.
+        # this handles the flush command by doing nothing.
+        # you might want to specify some extra behavior here.
+        pass
+
+
+def log(*args):
+    print(f'[{datetime.now()}]', *args)
+
+
+# parameters
+names = read_strings_from_txt(args.complex_names_path)
+
+if os.path.exists(args.results_path) and not args.skip_existing:
+    shutil.rmtree(args.results_path)
+os.makedirs(args.results_path, exist_ok=True)
+sys.stdout = Logger(logpath=f'{args.results_path}/gnina.log', syspart=sys.stdout)
+sys.stderr = Logger(logpath=f'{args.results_path}/error.log', syspart=sys.stderr)
+
+p2rank_cache_path = "results/.p2rank_cache"
+if args.use_p2rank_pocket and not args.skip_p2rank:
+    os.makedirs(p2rank_cache_path, exist_ok=True)
+    pdb_files_cache = os.path.join(p2rank_cache_path,'pdb_files')
+    os.makedirs(pdb_files_cache, exist_ok=True)
+    with open(f"{p2rank_cache_path}/pdb_list_p2rank.txt", "w") as out:
+        for name in names:
+            shutil.copy(os.path.join(args.data_dir, name, f'{name}_protein_processed.pdb'), f'{pdb_files_cache}/{name}_protein_processed.pdb')
+            out.write(os.path.join('pdb_files', f'{name}_protein_processed.pdb\n'))
+    cmd = f"bash {args.prank_path} predict {p2rank_cache_path}/pdb_list_p2rank.txt -o {p2rank_cache_path}/p2rank_output -threads 4"
+    os.system(cmd)
+
+
+all_times = []
+start_time = time.time()
+for i, name in enumerate(names):
+    os.makedirs(os.path.join(args.results_path, name), exist_ok=True)
+    log('\n')
+    log(f'complex {i} of {len(names)}')
+    # call gnina to find binding pose
+    rec_path = os.path.join(args.data_dir, name, f'{name}_protein_processed.pdb')
+    prediction_output_name = os.path.join(args.results_path, name, f'{name}{args.file_suffix}.pdb')
+    log_path = os.path.join(args.results_path, name, f'{name}{args.file_suffix}.log')
+    if args.seed_molecules_path is not None: seed_mol_path = os.path.join(args.seed_molecules_path, name, f'{args.seed_molecule_filename}')
+    if args.skip_existing and os.path.exists(prediction_output_name): continue
+
+    if args.pocket_mode:
+        mol = read_mol(args.data_dir, name, remove_hs=False)
+        rec = PandasPdb().read_pdb(rec_path)
+        rec_df = rec.get(s='c-alpha')
+        rec_pos = rec_df[['x_coord', 'y_coord', 'z_coord']].to_numpy().squeeze().astype(np.float32)
+        lig_pos = mol.GetConformer().GetPositions()
+        d = cdist(rec_pos, lig_pos)
+        label = np.any(d < args.pocket_cutoff, axis=1)
+
+        if np.any(label):
+            center_pocket = rec_pos[label].mean(axis=0)
+        else:
+            print("No pocket residue below minimum distance ", args.pocket_cutoff, "taking closest at", np.min(d))
+            center_pocket = rec_pos[np.argmin(np.min(d, axis=1)[0])]
+        radius_pocket = np.max(np.linalg.norm(lig_pos - center_pocket[None, :], axis=1))
+        diameter_pocket = radius_pocket * 2
+        center_x = center_pocket[0]
+        size_x = diameter_pocket + 8
+        center_y = center_pocket[1]
+        size_y = diameter_pocket + 8
+        center_z = center_pocket[2]
+        size_z = diameter_pocket + 8
+
+
+    mol_rdkit = read_mol(args.data_dir, name, remove_hs=False)
+    single_time = time.time()
+
+    mol_rdkit.RemoveAllConformers()
+    ps = AllChem.ETKDGv2()
+    id = AllChem.EmbedMolecule(mol_rdkit, ps)
+    if id == -1:
+        print('rdkit pos could not be generated without using random pos. using random pos now.')
+        ps.useRandomCoords = True
+        AllChem.EmbedMolecule(mol_rdkit, ps)
+        AllChem.MMFFOptimizeMolecule(mol_rdkit, confId=0)
+    rdkit_mol_path = os.path.join(args.data_dir, name, f'{name}_rdkit_ligand.pdb')
+    MolToPDBFile(mol_rdkit, rdkit_mol_path)
+
+    fallback_without_p2rank = False
+    if args.use_p2rank_pocket:
+        df = pd.read_csv(f'{p2rank_cache_path}/p2rank_output/{name}_protein_processed.pdb_predictions.csv')
+        rdkit_lig_pos = mol_rdkit.GetConformer().GetPositions()
+        diameter_pocket = np.max(cdist(rdkit_lig_pos, rdkit_lig_pos))
+        size_x = diameter_pocket + args.autobox_add * 2
+        size_y = diameter_pocket + args.autobox_add * 2
+        size_z = diameter_pocket + args.autobox_add * 2
+        if df.empty:
+            fallback_without_p2rank = True
+        else:
+            center_x = df.iloc[0]['   center_x']
+            center_y = df.iloc[0]['   center_y']
+            center_z = df.iloc[0]['   center_z']
+
+
+
+    log(f'processing {rec_path}')
+    if not args.pocket_mode and not args.use_p2rank_pocket or fallback_without_p2rank:
+        return_code = subprocess.run(
+            f"gnina --receptor {rec_path} --ligand {rdkit_mol_path} --num_modes {args.num_modes} -o {prediction_output_name} {'--no_gpu' if args.no_gpu else ''} --autobox_ligand {rec_path if args.seed_molecules_path is None else seed_mol_path} --autobox_add {args.autobox_add} --log {log_path} --exhaustiveness {args.exhaustiveness} --cpu {args.num_cpu} {'--cnn_scoring none' if args.smina else ''}",
+            shell=True)
+    else:
+        return_code = subprocess.run(
+            f"gnina --receptor {rec_path} --ligand {rdkit_mol_path} --num_modes {args.num_modes} -o {prediction_output_name} {'--no_gpu' if args.no_gpu else ''} --log {log_path} --exhaustiveness {args.exhaustiveness} --cpu {args.num_cpu} {'--cnn_scoring none' if args.smina else ''} --center_x {center_x} --center_y {center_y} --center_z {center_z} --size_x {size_x} --size_y {size_y} --size_z {size_z}",
+            shell=True)
+    log(return_code)
+    all_times.append(time.time() - single_time)
+
+    log("single time: --- %s seconds ---" % (time.time() - single_time))
+    log("time so far: --- %s seconds ---" % (time.time() - start_time))
+    log('\n')
+log(all_times)
+log("--- %s seconds ---" % (time.time() - start_time))

baselines/baseline_run_tankbind_parallel.sh

@@ -0,0 +1,5 @@
+for i in $(seq 0 15); do
+ 	python baseline_tankbind_runtime.py --parallel_id $i --parallel_tot 16 --prank_path /data/rsg/nlp/hstark/TankBind/packages/p2rank_2.3/prank --data_dir /data/rsg/nlp/hstark/ligbind/data/PDBBind_processed --split_path /data/rsg/nlp/hstark/ligbind/data/splits/timesplit_test --results_path /data/rsg/nlp/hstark/ligbind/results/tankbind_16_worker_runtime --device cpu --skip_p2rank --num_workers 1 --skip_multiple_pocket_outputs &
+done
+wait
+

baselines/baseline_tankbind_evaluation.py

@@ -0,0 +1,239 @@
+
+import copy
+import os
+
+import plotly.express as px
+import time
+from argparse import FileType, ArgumentParser
+
+import numpy as np
+import pandas as pd
+import wandb
+from biopandas.pdb import PandasPdb
+from rdkit import Chem
+from rdkit.Chem import RemoveHs
+
+from tqdm import tqdm
+
+from datasets.pdbbind import read_mol
+from datasets.process_mols import read_molecule, read_sdf_or_mol2
+from utils.utils import read_strings_from_txt, get_symmetry_rmsd, remove_all_hs
+
+parser = ArgumentParser()
+parser.add_argument('--config', type=FileType(mode='r'), default=None)
+parser.add_argument('--run_name', type=str, default='tankbind', help='')
+parser.add_argument('--data_dir', type=str, default='data/PDBBind_processed', help='')
+parser.add_argument('--renumbered_atoms_dir', type=str, default='../TankBind/examples/tankbind_pdb/renumber_atom_index_same_as_smiles', help='')
+parser.add_argument('--results_path', type=str, default='results/tankbind_top5', help='Path to folder with trained model and hyperparameters')
+parser.add_argument('--project', type=str, default='ligbind_inf', help='')
+parser.add_argument('--wandb', action='store_true', default=True, help='')
+parser.add_argument('--num_predictions', type=int, default=5, help='')
+args = parser.parse_args()
+
+names = read_strings_from_txt(f'data/splits/timesplit_test')
+names_no_rec_overlap = read_strings_from_txt(f'data/splits/timesplit_test_no_rec_overlap')
+
+if args.wandb:
+    wandb.init(
+        entity='coarse-graining-mit',
+        settings=wandb.Settings(start_method="fork"),
+        project=args.project,
+        name=args.run_name,
+        config=args
+    )
+
+all_times = []
+rmsds_list = []
+unsym_rmsds_list = []
+centroid_distances_list = []
+min_cross_distances_list = []
+min_self_distances_list = []
+made_prediction_list = []
+steric_clash_list = []
+without_rec_overlap_list = []
+
+start_time = time.time()
+successful_names_list = []
+for i, name in enumerate(tqdm(names)):
+    mol, _ = read_sdf_or_mol2(f"{args.renumbered_atoms_dir}/{name}.sdf", None)
+    sm = Chem.MolToSmiles(mol)
+    m_order = list(mol.GetPropsAsDict(includePrivate=True, includeComputed=True)['_smilesAtomOutputOrder'])
+    mol = Chem.RenumberAtoms(mol, m_order)
+    mol = Chem.RemoveHs(mol)
+    orig_ligand_pos = np.array(mol.GetConformer().GetPositions())
+
+    assert(os.path.exists(os.path.join(args.results_path, name, f'{name}_tankbind_0.sdf')))
+    ligand_pos = []
+    for i in range(args.num_predictions):
+        if not os.path.exists(os.path.join(args.results_path, name, f'{name}_tankbind_{i}.sdf')): break
+        mol_pred, _ = read_sdf_or_mol2(os.path.join(args.results_path, name, f'{name}_tankbind_{i}.sdf'),None)
+        sm = Chem.MolToSmiles(mol_pred)
+        m_order = list(mol_pred.GetPropsAsDict(includePrivate=True, includeComputed=True)['_smilesAtomOutputOrder'])
+        mol_pred = Chem.RenumberAtoms(mol_pred, m_order)
+        mol_pred = RemoveHs(mol_pred)
+        ligand_pos.append(np.array(mol_pred.GetConformer().GetPositions()))
+    ligand_pos = np.asarray(ligand_pos)
+
+    try:
+        unsym_rmsd = np.sqrt(((ligand_pos - orig_ligand_pos) ** 2).sum(axis=2).mean(axis=1))
+        rmsd = np.array(get_symmetry_rmsd(mol, orig_ligand_pos, [l for l in ligand_pos], mol_pred))
+    except Exception as e:
+        print("Using non corrected RMSD because of the error:", e)
+        rmsd = np.sqrt(((ligand_pos - orig_ligand_pos) ** 2).sum(axis=2).mean(axis=1))
+
+    num_pockets = len(ligand_pos)
+    unsym_rmsds_list.append(np.lib.pad(unsym_rmsd, (0,10-len(unsym_rmsd)), 'constant', constant_values=(0)) )
+    rmsds_list.append(np.lib.pad(rmsd, (0,10-len(rmsd)), 'constant', constant_values=(0)) )
+    centroid_distance = np.linalg.norm(ligand_pos.mean(axis=1) - orig_ligand_pos[None,:].mean(axis=1), axis=1)
+    centroid_distances_list.append(np.lib.pad(centroid_distance, (0,10-len(rmsd)), 'constant', constant_values=(0)) )
+
+    rec_path = os.path.join(args.data_dir, name, f'{name}_protein_processed.pdb')
+    if not os.path.exists(rec_path):
+        rec_path = os.path.join(args.data_dir, name,f'{name}_protein_obabel_reduce.pdb')
+    rec = PandasPdb().read_pdb(rec_path)
+    rec_df = rec.df['ATOM']
+    receptor_pos = rec_df[['x_coord', 'y_coord', 'z_coord']].to_numpy().squeeze().astype(np.float32)
+    receptor_pos = np.tile(receptor_pos, (10, 1, 1))
+
+    ligand_pos_padded = np.lib.pad(ligand_pos, ((0,10-len(ligand_pos)), (0,0), (0,0)), 'constant', constant_values=(np.inf))
+    ligand_pos_padded_zero = np.lib.pad(ligand_pos, ((0, 10 - len(ligand_pos)), (0, 0), (0, 0)), 'constant',constant_values=0)
+    cross_distances = np.linalg.norm(receptor_pos[:, :, None, :] - ligand_pos_padded[:, None, :, :], axis=-1)
+    self_distances = np.linalg.norm(ligand_pos_padded_zero[:, :, None, :] - ligand_pos_padded_zero[:, None, :, :], axis=-1)
+    self_distances =  np.where(np.eye(self_distances.shape[2]), np.inf, self_distances)
+    min_self_distances_list.append(np.min(self_distances, axis=(1, 2)))
+    min_cross_distance = np.min(cross_distances, axis=(1, 2))
+    individual_made_prediction = np.lib.pad(np.ones(num_pockets), (0,10-len(rmsd)), 'constant', constant_values=(0))
+    made_prediction_list.append(individual_made_prediction)
+    min_cross_distances_list.append(min_cross_distance)
+    successful_names_list.append(name)
+    without_rec_overlap_list.append(1 if name in names_no_rec_overlap else 0)
+
+performance_metrics = {}
+for overlap in ['', 'no_overlap_']:
+    if 'no_overlap_' == overlap:
+        without_rec_overlap = np.array(without_rec_overlap_list, dtype=bool)
+        unsym_rmsds = np.array(unsym_rmsds_list)[without_rec_overlap]
+        rmsds  = np.array(rmsds_list)[without_rec_overlap]
+        centroid_distances = np.array(centroid_distances_list)[without_rec_overlap]
+        min_cross_distances = np.array(min_cross_distances_list)[without_rec_overlap]
+        min_self_distances = np.array(min_self_distances_list)[without_rec_overlap]
+        made_prediction = np.array(made_prediction_list)[without_rec_overlap]
+        successful_names = np.array(successful_names_list)[without_rec_overlap]
+    else:
+        unsym_rmsds = np.array(unsym_rmsds_list)
+        rmsds = np.array(rmsds_list)
+        centroid_distances = np.array(centroid_distances_list)
+        min_cross_distances = np.array(min_cross_distances_list)
+        min_self_distances = np.array(min_self_distances_list)
+        made_prediction = np.array(made_prediction_list)
+        successful_names = np.array(successful_names_list)
+
+    inf_rmsds = copy.deepcopy(rmsds)
+    inf_rmsds[~made_prediction.astype(bool)] = np.inf
+    inf_centroid_distances = copy.deepcopy(centroid_distances)
+    inf_centroid_distances[~made_prediction.astype(bool)] = np.inf
+
+    np.save(os.path.join(args.results_path, f'{overlap}rmsds.npy'), rmsds)
+    np.save(os.path.join(args.results_path, f'{overlap}names.npy'), np.array(successful_names))
+    np.save(os.path.join(args.results_path, f'{overlap}centroid_distances.npy'), centroid_distances)
+    np.save(os.path.join(args.results_path, f'{overlap}min_cross_distances.npy'), min_cross_distances)
+    np.save(os.path.join(args.results_path, f'{overlap}min_self_distances.npy'), min_self_distances)
+
+    performance_metrics.update({
+        f'{overlap}self_intersect_fraction': (100 * (min_self_distances[:, 0] < 0.4).sum() / len(min_self_distances[:, 0])),
+        f'{overlap}steric_clash_fraction': (100 * (min_cross_distances[:,0] < 0.4).sum() / len(min_cross_distances[:,0])),
+        f'{overlap}mean_rmsd': rmsds[:,0].mean(),
+        f'{overlap}unsym_rmsds_below_2': (100 * (unsym_rmsds[:,0] < 2).sum() / len(unsym_rmsds[:,0])),
+        f'{overlap}rmsds_below_2': (100 * (rmsds[:,0] < 2).sum() / len(rmsds[:,0])),
+        f'{overlap}rmsds_below_5': (100 * (rmsds[:,0] < 5).sum() / len(rmsds[:,0])),
+        f'{overlap}rmsds_percentile_25': np.percentile(rmsds[:,0], 25).round(2),
+        f'{overlap}rmsds_percentile_50': np.percentile(rmsds[:,0], 50).round(2),
+        f'{overlap}rmsds_percentile_75': np.percentile(rmsds[:,0], 75).round(2),
+
+        f'{overlap}mean_centroid': centroid_distances[:,0].mean().__round__(2),
+        f'{overlap}centroid_below_2': (100 * (centroid_distances[:,0] < 2).sum() / len(centroid_distances[:,0])).__round__(2),
+        f'{overlap}centroid_below_5': (100 * (centroid_distances[:,0] < 5).sum() / len(centroid_distances[:,0])).__round__(2),
+        f'{overlap}centroid_percentile_25': np.percentile(centroid_distances[:,0], 25).round(2),
+        f'{overlap}centroid_percentile_50': np.percentile(centroid_distances[:,0], 50).round(2),
+        f'{overlap}centroid_percentile_75': np.percentile(centroid_distances[:,0], 75).round(2),
+    })
+
+    top5_rmsds = np.min(inf_rmsds[:, :5], axis=1)
+    top5_centroid_distances = centroid_distances[np.arange(rmsds.shape[0])[:,None],np.argsort(inf_rmsds[:, :5], axis=1)][:,0]
+    top5_min_cross_distances = min_cross_distances[np.arange(rmsds.shape[0])[:,None],np.argsort(inf_rmsds[:, :5], axis=1)][:,0]
+    top5_min_self_distances = min_self_distances[np.arange(rmsds.shape[0])[:,None],np.argsort(inf_rmsds[:, :5], axis=1)][:,0]
+    performance_metrics.update({
+        f'{overlap}top5_steric_clash_fraction': (100 * (top5_min_cross_distances < 0.4).sum() / len(top5_min_cross_distances)).__round__(2),
+        f'{overlap}top5_self_intersect_fraction': (100 * (top5_min_self_distances < 0.4).sum() / len(top5_min_self_distances)).__round__(2),
+        f'{overlap}top5_rmsds_below_2': (100 * (top5_rmsds < 2).sum() / len(top5_rmsds)).__round__(2),
+        f'{overlap}top5_rmsds_below_5': (100 * (top5_rmsds < 5).sum() / len(top5_rmsds)).__round__(2),
+        f'{overlap}top5_rmsds_percentile_25': np.percentile(top5_rmsds, 25).round(2),
+        f'{overlap}top5_rmsds_percentile_50': np.percentile(top5_rmsds, 50).round(2),
+        f'{overlap}top5_rmsds_percentile_75': np.percentile(top5_rmsds, 75).round(2),
+
+        f'{overlap}top5_centroid_below_2': (100 * (top5_centroid_distances < 2).sum() / len(top5_centroid_distances)).__round__(2),
+        f'{overlap}top5_centroid_below_5': (100 * (top5_centroid_distances < 5).sum() / len(top5_centroid_distances)).__round__(2),
+        f'{overlap}top5_centroid_percentile_25': np.percentile(top5_centroid_distances, 25).round(2),
+        f'{overlap}top5_centroid_percentile_50': np.percentile(top5_centroid_distances, 50).round(2),
+        f'{overlap}top5_centroid_percentile_75': np.percentile(top5_centroid_distances, 75).round(2),
+    })
+
+
+
+
+    top10_rmsds = np.min(inf_rmsds[:, :10], axis=1)
+    top10_centroid_distances = centroid_distances[np.arange(rmsds.shape[0])[:,None],np.argsort(inf_rmsds[:, :10], axis=1)][:,0]
+    top10_min_cross_distances = min_cross_distances[np.arange(rmsds.shape[0])[:,None],np.argsort(inf_rmsds[:, :10], axis=1)][:,0]
+    top10_min_self_distances = min_self_distances[np.arange(rmsds.shape[0])[:,None],np.argsort(inf_rmsds[:, :10], axis=1)][:,0]
+    performance_metrics.update({
+        f'{overlap}top10_steric_clash_fraction': (100 * (top10_min_cross_distances < 0.4).sum() / len(top10_min_cross_distances)).__round__(2),
+        f'{overlap}top10_self_intersect_fraction': (100 * (top10_min_self_distances < 0.4).sum() / len(top10_min_self_distances)).__round__(2),
+        f'{overlap}top10_rmsds_below_2': (100 * (top10_rmsds < 2).sum() / len(top10_rmsds)).__round__(2),
+        f'{overlap}top10_rmsds_below_5': (100 * (top10_rmsds < 5).sum() / len(top10_rmsds)).__round__(2),
+        f'{overlap}top10_rmsds_percentile_25': np.percentile(top10_rmsds, 25).round(2),
+        f'{overlap}top10_rmsds_percentile_50': np.percentile(top10_rmsds, 50).round(2),
+        f'{overlap}top10_rmsds_percentile_75': np.percentile(top10_rmsds, 75).round(2),
+
+        f'{overlap}top10_centroid_below_2': (100 * (top10_centroid_distances < 2).sum() / len(top10_centroid_distances)).__round__(2),
+        f'{overlap}top10_centroid_below_5': (100 * (top10_centroid_distances < 5).sum() / len(top10_centroid_distances)).__round__(2),
+        f'{overlap}top10_centroid_percentile_25': np.percentile(top10_centroid_distances, 25).round(2),
+        f'{overlap}top10_centroid_percentile_50': np.percentile(top10_centroid_distances, 50).round(2),
+        f'{overlap}top10_centroid_percentile_75': np.percentile(top10_centroid_distances, 75).round(2),
+    })
+for k in performance_metrics:
+    print(k, performance_metrics[k])
+
+if args.wandb:
+    wandb.log(performance_metrics)
+    histogram_metrics_list = [('rmsd', rmsds[:,0]),
+                              ('centroid_distance', centroid_distances[:,0]),
+                              ('mean_rmsd', rmsds[:,0]),
+                              ('mean_centroid_distance', centroid_distances[:,0])]
+    histogram_metrics_list.append(('top5_rmsds', top5_rmsds))
+    histogram_metrics_list.append(('top5_centroid_distances', top5_centroid_distances))
+    histogram_metrics_list.append(('top10_rmsds', top10_rmsds))
+    histogram_metrics_list.append(('top10_centroid_distances', top10_centroid_distances))
+
+    os.makedirs(f'.plotly_cache/baseline_cache', exist_ok=True)
+    images = []
+    for metric_name, metric in histogram_metrics_list:
+        d = {args.results_path: metric}
+        df = pd.DataFrame(data=d)
+        fig = px.ecdf(df, width=900, height=600, range_x=[0, 40])
+        fig.add_vline(x=2, annotation_text='2 A;', annotation_font_size=20, annotation_position="top right",
+                      line_dash='dash', line_color='firebrick', annotation_font_color='firebrick')
+        fig.add_vline(x=5, annotation_text='5 A;', annotation_font_size=20, annotation_position="top right",
+                      line_dash='dash', line_color='green', annotation_font_color='green')
+        fig.update_xaxes(title=f'{metric_name} in Angstrom', title_font={"size": 20}, tickfont={"size": 20})
+        fig.update_yaxes(title=f'Fraction of predictions with lower error', title_font={"size": 20},
+                         tickfont={"size": 20})
+        fig.update_layout(autosize=False, margin={'l': 0, 'r': 0, 't': 0, 'b': 0}, plot_bgcolor='white',
+                          paper_bgcolor='white', legend_title_text='Method', legend_title_font_size=17,
+                          legend=dict(yanchor="bottom", y=0.1, xanchor="right", x=0.99, font=dict(size=17), ), )
+        fig.update_xaxes(showgrid=True, gridcolor='lightgrey')
+        fig.update_yaxes(showgrid=True, gridcolor='lightgrey')
+
+        fig.write_image(os.path.join(f'.plotly_cache/baseline_cache', f'{metric_name}.png'))
+        wandb.log({metric_name: wandb.Image(os.path.join(f'.plotly_cache/baseline_cache', f'{metric_name}.png'), caption=f"{metric_name}")})
+        images.append(wandb.Image(os.path.join(f'.plotly_cache/baseline_cache', f'{metric_name}.png'), caption=f"{metric_name}"))
+    wandb.log({'images': images})

baselines/baseline_tankbind_runtime.py

@@ -0,0 +1,342 @@
+# This file needs to be ran in the TANKBind repository together with baseline_run_tankbind_parallel.sh
+
+import sys
+import time
+from multiprocessing import Pool
+
+
+import copy
+import warnings
+from argparse import ArgumentParser
+
+from rdkit.Chem import AllChem, RemoveHs
+
+from feature_utils import save_cleaned_protein, read_mol
+from generation_utils import get_LAS_distance_constraint_mask, get_info_pred_distance, write_with_new_coords
+import logging
+from torch_geometric.loader import DataLoader
+from tqdm import tqdm  # pip install tqdm if fails.
+from model import get_model
+# from utils import *
+import torch
+
+
+from data import TankBind_prediction
+
+import os
+import numpy as np
+import pandas as pd
+import rdkit.Chem as Chem
+from feature_utils import generate_sdf_from_smiles_using_rdkit
+from feature_utils import get_protein_feature
+from Bio.PDB import PDBParser
+from feature_utils import extract_torchdrug_feature_from_mol
+
+
+def read_strings_from_txt(path):
+    # every line will be one element of the returned list
+    with open(path) as file:
+        lines = file.readlines()
+        return [line.rstrip() for line in lines]
+
+
+def read_molecule(molecule_file, sanitize=False, calc_charges=False, remove_hs=False):
+    if molecule_file.endswith('.mol2'):
+        mol = Chem.MolFromMol2File(molecule_file, sanitize=False, removeHs=False)
+    elif molecule_file.endswith('.sdf'):
+        supplier = Chem.SDMolSupplier(molecule_file, sanitize=False, removeHs=False)
+        mol = supplier[0]
+    elif molecule_file.endswith('.pdbqt'):
+        with open(molecule_file) as file:
+            pdbqt_data = file.readlines()
+        pdb_block = ''
+        for line in pdbqt_data:
+            pdb_block += '{}\n'.format(line[:66])
+        mol = Chem.MolFromPDBBlock(pdb_block, sanitize=False, removeHs=False)
+    elif molecule_file.endswith('.pdb'):
+        mol = Chem.MolFromPDBFile(molecule_file, sanitize=False, removeHs=False)
+    else:
+        return ValueError('Expect the format of the molecule_file to be '
+                          'one of .mol2, .sdf, .pdbqt and .pdb, got {}'.format(molecule_file))
+    try:
+        if sanitize or calc_charges:
+            Chem.SanitizeMol(mol)
+
+        if calc_charges:
+            # Compute Gasteiger charges on the molecule.
+            try:
+                AllChem.ComputeGasteigerCharges(mol)
+            except:
+                warnings.warn('Unable to compute charges for the molecule.')
+
+        if remove_hs:
+            mol = Chem.RemoveHs(mol, sanitize=sanitize)
+    except:
+        return None
+
+    return mol
+
+
+def parallel_save_prediction(arguments):
+    dataset, y_pred_list, chosen,rdkit_mol_path, result_folder, name = arguments
+    for idx, line in chosen.iterrows():
+        pocket_name = line['pocket_name']
+        compound_name = line['compound_name']
+        ligandName = compound_name.split("_")[1]
+        dataset_index = line['dataset_index']
+        coords = dataset[dataset_index].coords.to('cpu')
+        protein_nodes_xyz = dataset[dataset_index].node_xyz.to('cpu')
+        n_compound = coords.shape[0]
+        n_protein = protein_nodes_xyz.shape[0]
+        y_pred = y_pred_list[dataset_index].reshape(n_protein, n_compound).to('cpu')
+        compound_pair_dis_constraint = torch.cdist(coords, coords)
+        mol = Chem.MolFromMolFile(rdkit_mol_path)
+        LAS_distance_constraint_mask = get_LAS_distance_constraint_mask(mol).bool()
+        pred_dist_info = get_info_pred_distance(coords, y_pred, protein_nodes_xyz, compound_pair_dis_constraint,
+                                                LAS_distance_constraint_mask=LAS_distance_constraint_mask,
+                                                n_repeat=1, show_progress=False)
+
+        toFile = f'{result_folder}/{name}_tankbind_chosen.sdf'
+        new_coords = pred_dist_info.sort_values("loss")['coords'].iloc[0].astype(np.double)
+        write_with_new_coords(mol, new_coords, toFile)
+
+if __name__ == '__main__':
+    tankbind_src_folder = "../tankbind"
+    sys.path.insert(0, tankbind_src_folder)
+    torch.set_num_threads(16)
+    parser = ArgumentParser()
+    parser.add_argument('--data_dir', type=str, default='/Users/hstark/projects/ligbind/data/PDBBind_processed', help='')
+    parser.add_argument('--split_path', type=str, default='/Users/hstark/projects/ligbind/data/splits/timesplit_test', help='')
+    parser.add_argument('--prank_path', type=str, default='/Users/hstark/projects/p2rank_2.3/prank', help='')
+    parser.add_argument('--results_path', type=str, default='results/tankbind_results', help='')
+    parser.add_argument('--skip_existing', action='store_true', default=False, help='')
+    parser.add_argument('--skip_p2rank', action='store_true', default=False, help='')
+    parser.add_argument('--skip_multiple_pocket_outputs', action='store_true', default=False, help='')
+    parser.add_argument('--device', type=str, default='cpu', help='')
+    parser.add_argument('--num_workers', type=int, default=1, help='')
+    parser.add_argument('--parallel_id', type=int, default=0, help='')
+    parser.add_argument('--parallel_tot', type=int, default=1, help='')
+    args = parser.parse_args()
+
+    device = args.device
+    cache_path = "tankbind_cache"
+    os.makedirs(cache_path, exist_ok=True)
+    os.makedirs(args.results_path, exist_ok=True)
+
+
+
+    logging.basicConfig(level=logging.INFO)
+    model = get_model(0, logging, device)
+    # re-dock model
+    # modelFile = "../saved_models/re_dock.pt"
+    # self-dock model
+    modelFile = f"{tankbind_src_folder}/../saved_models/self_dock.pt"
+
+    model.load_state_dict(torch.load(modelFile, map_location=device))
+    _ = model.eval()
+    batch_size = 5
+    names = read_strings_from_txt(args.split_path)
+    if args.parallel_tot > 1:
+        size = len(names) // args.parallel_tot + 1
+        names = names[args.parallel_id*size:(args.parallel_id+1)*size]
+    rmsds = []
+
+    forward_pass_time = []
+    times_preprocess = []
+    times_inference = []
+    top_10_generation_time = []
+    top_1_generation_time = []
+    start_time = time.time()
+    if not args.skip_p2rank:
+        for name in names:
+            if args.skip_existing and os.path.exists(f'{args.results_path}/{name}/{name}_tankbind_1.sdf'): continue
+            print("Now processing: ", name)
+            protein_path = f'{args.data_dir}/{name}/{name}_protein_processed.pdb'
+            cleaned_protein_path = f"{cache_path}/{name}_protein_tankbind_cleaned.pdb"  # if you change this you also need to change below
+            parser = PDBParser(QUIET=True)
+            s = parser.get_structure(name, protein_path)
+            c = s[0]
+            clean_res_list, ligand_list = save_cleaned_protein(c, cleaned_protein_path)
+
+        with open(f"{cache_path}/pdb_list_p2rank.txt", "w") as out:
+            for name in names:
+                out.write(f"{name}_protein_tankbind_cleaned.pdb\n")
+        cmd = f"bash {args.prank_path} predict {cache_path}/pdb_list_p2rank.txt -o {cache_path}/p2rank -threads 4"
+        os.system(cmd)
+    times_preprocess.append(time.time() - start_time)
+    p2_rank_time = time.time() - start_time
+
+
+
+
+    list_to_parallelize = []
+    for name in tqdm(names):
+        single_preprocess_time = time.time()
+        if args.skip_existing and os.path.exists(f'{args.results_path}/{name}/{name}_tankbind_1.sdf'): continue
+        print("Now processing: ", name)
+        protein_path = f'{args.data_dir}/{name}/{name}_protein_processed.pdb'
+        ligand_path = f"{args.data_dir}/{name}/{name}_ligand.sdf"
+        cleaned_protein_path = f"{cache_path}/{name}_protein_tankbind_cleaned.pdb"  # if you change this you also need to change below
+        rdkit_mol_path = f"{cache_path}/{name}_rdkit_ligand.sdf"
+
+        parser = PDBParser(QUIET=True)
+        s = parser.get_structure(name, protein_path)
+        c = s[0]
+        clean_res_list, ligand_list = save_cleaned_protein(c, cleaned_protein_path)
+        lig, _ = read_mol(f"{args.data_dir}/{name}/{name}_ligand.sdf", f"{args.data_dir}/{name}/{name}_ligand.mol2")
+
+        lig = RemoveHs(lig)
+        smiles = Chem.MolToSmiles(lig)
+        generate_sdf_from_smiles_using_rdkit(smiles, rdkit_mol_path, shift_dis=0)
+
+        parser = PDBParser(QUIET=True)
+        s = parser.get_structure("x", cleaned_protein_path)
+        res_list = list(s.get_residues())
+
+        protein_dict = {}
+        protein_dict[name] = get_protein_feature(res_list)
+        compound_dict = {}
+
+        mol = Chem.MolFromMolFile(rdkit_mol_path)
+        compound_dict[name + f"_{name}" + "_rdkit"] = extract_torchdrug_feature_from_mol(mol, has_LAS_mask=True)
+
+        info = []
+        for compound_name in list(compound_dict.keys()):
+            # use protein center as the block center.
+            com = ",".join([str(a.round(3)) for a in protein_dict[name][0].mean(axis=0).numpy()])
+            info.append([name, compound_name, "protein_center", com])
+
+            p2rankFile = f"{cache_path}/p2rank/{name}_protein_tankbind_cleaned.pdb_predictions.csv"
+            pocket = pd.read_csv(p2rankFile)
+            pocket.columns = pocket.columns.str.strip()
+            pocket_coms = pocket[['center_x', 'center_y', 'center_z']].values
+            for ith_pocket, com in enumerate(pocket_coms):
+                com = ",".join([str(a.round(3)) for a in com])
+                info.append([name, compound_name, f"pocket_{ith_pocket + 1}", com])
+        info = pd.DataFrame(info, columns=['protein_name', 'compound_name', 'pocket_name', 'pocket_com'])
+
+        dataset_path = f"{cache_path}/{name}_dataset/"
+        os.system(f"rm -r {dataset_path}")
+        os.system(f"mkdir -p {dataset_path}")
+        dataset = TankBind_prediction(dataset_path, data=info, protein_dict=protein_dict, compound_dict=compound_dict)
+
+        # dataset = TankBind_prediction(dataset_path)
+        times_preprocess.append(time.time() - single_preprocess_time)
+        single_forward_pass_time = time.time()
+        data_loader = DataLoader(dataset, batch_size=batch_size, follow_batch=['x', 'y', 'compound_pair'], shuffle=False,
+                                 num_workers=0)
+        affinity_pred_list = []
+        y_pred_list = []
+        for data in tqdm(data_loader):
+            data = data.to(device)
+            y_pred, affinity_pred = model(data)
+            affinity_pred_list.append(affinity_pred.detach().cpu())
+            for i in range(data.y_batch.max() + 1):
+                y_pred_list.append((y_pred[data['y_batch'] == i]).detach().cpu())
+
+        affinity_pred_list = torch.cat(affinity_pred_list)
+        forward_pass_time.append(time.time() - single_forward_pass_time)
+        output_info = copy.deepcopy(dataset.data)
+        output_info['affinity'] = affinity_pred_list
+        output_info['dataset_index'] = range(len(output_info))
+        output_info_sorted = output_info.sort_values('affinity', ascending=False)
+
+
+        result_folder = f'{args.results_path}/{name}'
+        os.makedirs(result_folder, exist_ok=True)
+        output_info_sorted.to_csv(f"{result_folder}/output_info_sorted_by_affinity.csv")
+
+        if not args.skip_multiple_pocket_outputs:
+            for idx, (dataframe_idx, line) in enumerate(copy.deepcopy(output_info_sorted).iterrows()):
+                single_top10_generation_time = time.time()
+                pocket_name = line['pocket_name']
+                compound_name = line['compound_name']
+                ligandName = compound_name.split("_")[1]
+                coords = dataset[dataframe_idx].coords.to('cpu')
+                protein_nodes_xyz = dataset[dataframe_idx].node_xyz.to('cpu')
+                n_compound = coords.shape[0]
+                n_protein = protein_nodes_xyz.shape[0]
+                y_pred = y_pred_list[dataframe_idx].reshape(n_protein, n_compound).to('cpu')
+                y = dataset[dataframe_idx].dis_map.reshape(n_protein, n_compound).to('cpu')
+                compound_pair_dis_constraint = torch.cdist(coords, coords)
+                mol = Chem.MolFromMolFile(rdkit_mol_path)
+                LAS_distance_constraint_mask = get_LAS_distance_constraint_mask(mol).bool()
+                pred_dist_info = get_info_pred_distance(coords, y_pred, protein_nodes_xyz, compound_pair_dis_constraint,
+                                              LAS_distance_constraint_mask=LAS_distance_constraint_mask,
+                                              n_repeat=1, show_progress=False)
+
+                toFile = f'{result_folder}/{name}_tankbind_{idx}.sdf'
+                new_coords = pred_dist_info.sort_values("loss")['coords'].iloc[0].astype(np.double)
+                write_with_new_coords(mol, new_coords, toFile)
+                if idx < 10:
+                    top_10_generation_time.append(time.time() - single_top10_generation_time)
+                if idx == 0:
+                    top_1_generation_time.append(time.time() - single_top10_generation_time)
+
+        output_info_chosen = copy.deepcopy(dataset.data)
+        output_info_chosen['affinity'] = affinity_pred_list
+        output_info_chosen['dataset_index'] = range(len(output_info_chosen))
+        chosen = output_info_chosen.loc[
+            output_info_chosen.groupby(['protein_name', 'compound_name'], sort=False)['affinity'].agg(
+                'idxmax')].reset_index()
+
+        list_to_parallelize.append((dataset, y_pred_list, chosen, rdkit_mol_path, result_folder, name))
+
+    chosen_generation_start_time = time.time()
+    if args.num_workers > 1:
+        p = Pool(args.num_workers, maxtasksperchild=1)
+        p.__enter__()
+    with tqdm(total=len(list_to_parallelize), desc=f'running optimization {i}/{len(list_to_parallelize)}') as pbar:
+        map_fn = p.imap_unordered if args.num_workers > 1 else map
+        for t in map_fn(parallel_save_prediction, list_to_parallelize):
+            pbar.update()
+    if args.num_workers > 1: p.__exit__(None, None, None)
+    chosen_generation_time = time.time() - chosen_generation_start_time
+    """
+        lig, _ = read_mol(f"{args.data_dir}/{name}/{name}_ligand.sdf", f"{args.data_dir}/{name}/{name}_ligand.mol2")
+        sm = Chem.MolToSmiles(lig)
+        m_order = list(lig.GetPropsAsDict(includePrivate=True, includeComputed=True)['_smilesAtomOutputOrder'])
+        lig = Chem.RenumberAtoms(lig, m_order)
+        lig = Chem.RemoveAllHs(lig)
+        lig = RemoveHs(lig)
+        true_ligand_pos = np.array(lig.GetConformer().GetPositions())
+    
+        toFile = f'{result_folder}/{name}_tankbind_chosen.sdf'
+        mol_pred, _ = read_mol(toFile, None)
+        sm = Chem.MolToSmiles(mol_pred)
+        m_order = list(mol_pred.GetPropsAsDict(includePrivate=True, includeComputed=True)['_smilesAtomOutputOrder'])
+        mol_pred = Chem.RenumberAtoms(mol_pred, m_order)
+        mol_pred = RemoveHs(mol_pred)
+        mol_pred_pos = np.array(mol_pred.GetConformer().GetPositions())
+        rmsds.append(np.sqrt(((true_ligand_pos - mol_pred_pos) ** 2).sum(axis=1).mean(axis=0)))
+        print(np.sqrt(((true_ligand_pos - mol_pred_pos) ** 2).sum(axis=1).mean(axis=0)))
+    """
+    forward_pass_time  = np.array(forward_pass_time).sum()
+    times_preprocess  = np.array(times_preprocess).sum()
+    times_inference  = np.array(times_inference).sum()
+    top_10_generation_time  = np.array(top_10_generation_time).sum()
+    top_1_generation_time  = np.array(top_1_generation_time).sum()
+
+    rmsds = np.array(rmsds)
+
+    print(f'forward_pass_time: {forward_pass_time}')
+    print(f'times_preprocess: {times_preprocess}')
+    print(f'times_inference: {times_inference}')
+    print(f'top_10_generation_time: {top_10_generation_time}')
+    print(f'top_1_generation_time: {top_1_generation_time}')
+    print(f'chosen_generation_time: {chosen_generation_time}')
+    print(f'rmsds_below_2: {(100 * (rmsds < 2).sum() / len(rmsds))}')
+    print(f'p2rank Time: {p2_rank_time}')
+    print(
+        f'total_time: '
+        f'{forward_pass_time + times_preprocess + times_inference + top_10_generation_time + top_1_generation_time + p2_rank_time}')
+
+    with open(os.path.join(args.results_path, 'tankbind_log.log'), 'w') as file:
+        file.write(f'forward_pass_time: {forward_pass_time}')
+        file.write(f'times_preprocess: {times_preprocess}')
+        file.write(f'times_inference: {times_inference}')
+        file.write(f'top_10_generation_time: {top_10_generation_time}')
+        file.write(f'top_1_generation_time: {top_1_generation_time}')
+        file.write(f'rmsds_below_2: {(100 * (rmsds < 2).sum() / len(rmsds))}')
+        file.write(f'p2rank Time: {p2_rank_time}')
+        file.write(f'total_time: {forward_pass_time + times_preprocess + times_inference + top_10_generation_time + top_1_generation_time + p2_rank_time}')

confidence/confidence_train.py

@@ -0,0 +1,319 @@
+import gc
+import math
+import os
+
+import shutil
+
+from argparse import Namespace, ArgumentParser, FileType
+import torch.nn.functional as F
+
+import wandb
+import torch
+from sklearn.metrics import roc_auc_score
+from torch_geometric.loader import DataListLoader, DataLoader
+from tqdm import tqdm
+
+from confidence.dataset import ConfidenceDataset
+from utils.training import AverageMeter
+
+torch.multiprocessing.set_sharing_strategy('file_system')
+
+import yaml
+from utils.utils import save_yaml_file, get_optimizer_and_scheduler, get_model
+
+
+parser = ArgumentParser()
+parser.add_argument('--config', type=FileType(mode='r'), default=None)
+parser.add_argument('--original_model_dir', type=str, default='workdir', help='Path to folder with trained model and hyperparameters')
+parser.add_argument('--restart_dir', type=str, default=None, help='')
+parser.add_argument('--use_original_model_cache', action='store_true', default=False, help='If this is true, the same dataset as in the original model will be used. Otherwise, the dataset parameters are used.')
+parser.add_argument('--data_dir', type=str, default='data/PDBBind_processed/', help='Folder containing original structures')
+parser.add_argument('--ckpt', type=str, default='best_model.pt', help='Checkpoint to use inside the folder')
+parser.add_argument('--model_save_frequency', type=int, default=0, help='Frequency with which to save the last model. If 0, then only the early stopping criterion best model is saved and overwritten.')
+parser.add_argument('--best_model_save_frequency', type=int, default=0, help='Frequency with which to save the best model. If 0, then only the early stopping criterion best model is saved and overwritten.')
+parser.add_argument('--run_name', type=str, default='test_confidence', help='')
+parser.add_argument('--project', type=str, default='diffdock_confidence', help='')
+parser.add_argument('--split_train', type=str, default='data/splits/timesplit_no_lig_overlap_train', help='Path of file defining the split')
+parser.add_argument('--split_val', type=str, default='data/splits/timesplit_no_lig_overlap_val', help='Path of file defining the split')
+parser.add_argument('--split_test', type=str, default='data/splits/timesplit_test', help='Path of file defining the split')
+
+# Inference parameters for creating the positions and rmsds that the confidence predictor will be trained on.
+parser.add_argument('--cache_path', type=str, default='data/cacheNew', help='Folder from where to load/restore cached dataset')
+parser.add_argument('--cache_ids_to_combine', nargs='+', type=str, default=None, help='RMSD value below which a prediction is considered a postitive. This can also be multiple cutoffs.')
+parser.add_argument('--cache_creation_id', type=int, default=None, help='number of times that inference is run on the full dataset before concatenating it and coming up with the full confidence dataset')
+parser.add_argument('--wandb', action='store_true', default=False, help='')
+parser.add_argument('--inference_steps', type=int, default=2, help='Number of denoising steps')
+parser.add_argument('--samples_per_complex', type=int, default=3, help='')
+parser.add_argument('--balance', action='store_true', default=False, help='If this is true than we do not force the samples seen during training to be the same amount of negatives as positives')
+parser.add_argument('--rmsd_prediction', action='store_true', default=False, help='')
+parser.add_argument('--rmsd_classification_cutoff', nargs='+', type=float, default=2, help='RMSD value below which a prediction is considered a postitive. This can also be multiple cutoffs.')
+
+parser.add_argument('--log_dir', type=str, default='workdir', help='')
+parser.add_argument('--main_metric', type=str, default='accuracy', help='Metric to track for early stopping. Mostly [loss, accuracy, ROC AUC]')
+parser.add_argument('--main_metric_goal', type=str, default='max', help='Can be [min, max]')
+parser.add_argument('--transfer_weights', action='store_true', default=False, help='')
+parser.add_argument('--batch_size', type=int, default=5, help='')
+parser.add_argument('--lr', type=float, default=1e-3, help='')
+parser.add_argument('--w_decay', type=float, default=0.0, help='')
+parser.add_argument('--scheduler', type=str, default='plateau', help='')
+parser.add_argument('--scheduler_patience', type=int, default=20, help='')
+parser.add_argument('--n_epochs', type=int, default=5, help='')
+
+# Dataset
+parser.add_argument('--limit_complexes', type=int, default=0, help='')
+parser.add_argument('--all_atoms', action='store_true', default=True, help='')
+parser.add_argument('--multiplicity', type=int, default=1, help='')
+parser.add_argument('--chain_cutoff', type=float, default=10, help='')
+parser.add_argument('--receptor_radius', type=float, default=30, help='')
+parser.add_argument('--c_alpha_max_neighbors', type=int, default=10, help='')
+parser.add_argument('--atom_radius', type=float, default=5, help='')
+parser.add_argument('--atom_max_neighbors', type=int, default=8, help='')
+parser.add_argument('--matching_popsize', type=int, default=20, help='')
+parser.add_argument('--matching_maxiter', type=int, default=20, help='')
+parser.add_argument('--max_lig_size', type=int, default=None, help='Maximum number of heavy atoms')
+parser.add_argument('--remove_hs', action='store_true', default=False, help='remove Hs')
+parser.add_argument('--num_conformers', type=int, default=1, help='')
+parser.add_argument('--esm_embeddings_path', type=str, default=None,help='If this is set then the LM embeddings at that path will be used for the receptor features')
+parser.add_argument('--no_torsion', action='store_true', default=False, help='')
+
+# Model
+parser.add_argument('--num_conv_layers', type=int, default=2, help='Number of interaction layers')
+parser.add_argument('--max_radius', type=float, default=5.0, help='Radius cutoff for geometric graph')
+parser.add_argument('--scale_by_sigma', action='store_true', default=True, help='Whether to normalise the score')
+parser.add_argument('--ns', type=int, default=16, help='Number of hidden features per node of order 0')
+parser.add_argument('--nv', type=int, default=4, help='Number of hidden features per node of order >0')
+parser.add_argument('--distance_embed_dim', type=int, default=32, help='')
+parser.add_argument('--cross_distance_embed_dim', type=int, default=32, help='')
+parser.add_argument('--no_batch_norm', action='store_true', default=False, help='If set, it removes the batch norm')
+parser.add_argument('--use_second_order_repr', action='store_true', default=False, help='Whether to use only up to first order representations or also second')
+parser.add_argument('--cross_max_distance', type=float, default=80, help='')
+parser.add_argument('--dynamic_max_cross', action='store_true', default=False, help='')
+parser.add_argument('--dropout', type=float, default=0.0, help='MLP dropout')
+parser.add_argument('--embedding_type', type=str, default="sinusoidal", help='')
+parser.add_argument('--sigma_embed_dim', type=int, default=32, help='')
+parser.add_argument('--embedding_scale', type=int, default=10000, help='')
+parser.add_argument('--confidence_no_batchnorm', action='store_true', default=False, help='')
+parser.add_argument('--confidence_dropout', type=float, default=0.0, help='MLP dropout in confidence readout')
+
+args = parser.parse_args()
+if args.config:
+    config_dict = yaml.load(args.config, Loader=yaml.FullLoader)
+    arg_dict = args.__dict__
+    for key, value in config_dict.items():
+        if isinstance(value, list):
+            for v in value:
+                arg_dict[key].append(v)
+        else:
+            arg_dict[key] = value
+    args.config = args.config.name
+assert(args.main_metric_goal == 'max' or args.main_metric_goal == 'min')
+
+def train_epoch(model, loader, optimizer, rmsd_prediction):
+    model.train()
+    meter = AverageMeter(['confidence_loss'])
+
+    for data in tqdm(loader, total=len(loader)):
+        if device.type == 'cuda' and len(data) % torch.cuda.device_count() == 1 or device.type == 'cpu' and data.num_graphs == 1:
+            print("Skipping batch of size 1 since otherwise batchnorm would not work.")
+        optimizer.zero_grad()
+        try:
+            pred = model(data)
+            if rmsd_prediction:
+                labels = torch.cat([graph.rmsd for graph in data]).to(device) if isinstance(data, list) else data.rmsd
+                confidence_loss = F.mse_loss(pred, labels)
+            else:
+                if isinstance(args.rmsd_classification_cutoff, list):
+                    labels = torch.cat([graph.y_binned for graph in data]).to(device) if isinstance(data, list) else data.y_binned
+                    confidence_loss = F.cross_entropy(pred, labels)
+                else:
+                    labels = torch.cat([graph.y for graph in data]).to(device) if isinstance(data, list) else data.y
+                    confidence_loss = F.binary_cross_entropy_with_logits(pred, labels)
+            confidence_loss.backward()
+            optimizer.step()
+            meter.add([confidence_loss.cpu().detach()])
+        except RuntimeError as e:
+            if 'out of memory' in str(e):
+                print('| WARNING: ran out of memory, skipping batch')
+                for p in model.parameters():
+                    if p.grad is not None:
+                        del p.grad  # free some memory
+                torch.cuda.empty_cache()
+                gc.collect()
+                continue
+            else:
+                raise e
+
+    return meter.summary()
+
+def test_epoch(model, loader, rmsd_prediction):
+    model.eval()
+    meter = AverageMeter(['loss'], unpooled_metrics=True) if rmsd_prediction else AverageMeter(['confidence_loss', 'accuracy', 'ROC AUC'], unpooled_metrics=True)
+    all_labels = []
+    all_affinities = []
+    for data in tqdm(loader, total=len(loader)):
+        try:
+            with torch.no_grad():
+                pred = model(data)
+            affinity_loss = torch.tensor(0.0, dtype=torch.float, device=pred[0].device)
+            accuracy = torch.tensor(0.0, dtype=torch.float, device=pred[0].device)
+            if rmsd_prediction:
+                labels = torch.cat([graph.rmsd for graph in data]).to(device) if isinstance(data, list) else data.rmsd
+                confidence_loss = F.mse_loss(pred, labels)
+                meter.add([confidence_loss.cpu().detach()])
+            else:
+                if isinstance(args.rmsd_classification_cutoff, list):
+                    labels = torch.cat([graph.y_binned for graph in data]).to(device) if isinstance(data,list) else data.y_binned
+                    confidence_loss = F.cross_entropy(pred, labels)
+                else:
+                    labels = torch.cat([graph.y for graph in data]).to(device) if isinstance(data, list) else data.y
+                    confidence_loss = F.binary_cross_entropy_with_logits(pred, labels)
+                    accuracy = torch.mean((labels == (pred > 0).float()).float())
+                try:
+                    roc_auc = roc_auc_score(labels.detach().cpu().numpy(), pred.detach().cpu().numpy())
+                except ValueError as e:
+                    if 'Only one class present in y_true. ROC AUC score is not defined in that case.' in str(e):
+                        roc_auc = 0
+                    else:
+                        raise e
+            meter.add([confidence_loss.cpu().detach(), accuracy.cpu().detach(), torch.tensor(roc_auc)])
+            all_labels.append(labels)
+
+        except RuntimeError as e:
+            if 'out of memory' in str(e):
+                print('| WARNING: ran out of memory, skipping batch')
+                for p in model.parameters():
+                    if p.grad is not None:
+                        del p.grad  # free some memory
+                torch.cuda.empty_cache()
+                continue
+            else:
+                raise e
+
+    all_labels = torch.cat(all_labels)
+
+    if rmsd_prediction:
+        baseline_metric = ((all_labels - all_labels.mean()).abs()).mean()
+    else:
+        baseline_metric = all_labels.sum() / len(all_labels)
+    results = meter.summary()
+    results.update({'baseline_metric': baseline_metric})
+    return meter.summary(), baseline_metric
+
+
+def train(args, model, optimizer, scheduler, train_loader, val_loader, run_dir):
+    best_val_metric = math.inf if args.main_metric_goal == 'min' else 0
+    best_epoch = 0
+
+    print("Starting training...")
+    for epoch in range(args.n_epochs):
+        logs = {}
+        train_metrics = train_epoch(model, train_loader, optimizer, args.rmsd_prediction)
+        print("Epoch {}: Training loss {:.4f}".format(epoch, train_metrics['confidence_loss']))
+
+        val_metrics, baseline_metric = test_epoch(model, val_loader, args.rmsd_prediction)
+        if args.rmsd_prediction:
+            print("Epoch {}: Validation loss {:.4f}".format(epoch, val_metrics['confidence_loss']))
+        else:
+            print("Epoch {}: Validation loss {:.4f}  accuracy {:.4f}".format(epoch, val_metrics['confidence_loss'], val_metrics['accuracy']))
+
+        if args.wandb:
+            logs.update({'valinf_' + k: v for k, v in val_metrics.items()}, step=epoch + 1)
+            logs.update({'train_' + k: v for k, v in train_metrics.items()}, step=epoch + 1)
+            logs.update({'mean_rmsd' if args.rmsd_prediction else 'fraction_positives': baseline_metric,
+                         'current_lr': optimizer.param_groups[0]['lr']})
+            wandb.log(logs, step=epoch + 1)
+
+        if scheduler:
+            scheduler.step(val_metrics[args.main_metric])
+
+        state_dict = model.module.state_dict() if device.type == 'cuda' else model.state_dict()
+
+        if args.main_metric_goal == 'min' and val_metrics[args.main_metric] < best_val_metric or \
+                args.main_metric_goal == 'max' and val_metrics[args.main_metric] > best_val_metric:
+            best_val_metric = val_metrics[args.main_metric]
+            best_epoch = epoch
+            torch.save(state_dict, os.path.join(run_dir, 'best_model.pt'))
+        if args.model_save_frequency > 0 and (epoch + 1) % args.model_save_frequency == 0:
+            torch.save(state_dict, os.path.join(run_dir, f'model_epoch{epoch+1}.pt'))
+        if args.best_model_save_frequency > 0 and (epoch + 1) % args.best_model_save_frequency == 0:
+            shutil.copyfile(os.path.join(run_dir, 'best_model.pt'), os.path.join(run_dir, f'best_model_epoch{epoch+1}.pt'))
+
+        torch.save({
+            'epoch': epoch,
+            'model': state_dict,
+            'optimizer': optimizer.state_dict(),
+        }, os.path.join(run_dir, 'last_model.pt'))
+
+    print("Best Validation accuracy {} on Epoch {}".format(best_val_metric, best_epoch))
+
+
+def construct_loader_confidence(args, device):
+    common_args = {'cache_path': args.cache_path, 'original_model_dir': args.original_model_dir, 'device': device,
+                   'inference_steps': args.inference_steps, 'samples_per_complex': args.samples_per_complex,
+                   'limit_complexes': args.limit_complexes, 'all_atoms': args.all_atoms, 'balance': args.balance, 'rmsd_classification_cutoff': args.rmsd_classification_cutoff,
+                   'use_original_model_cache': args.use_original_model_cache, 'cache_creation_id': args.cache_creation_id, "cache_ids_to_combine": args.cache_ids_to_combine}
+    loader_class = DataListLoader if torch.cuda.is_available() else DataLoader
+
+    exception_flag = False
+    try:
+        train_dataset = ConfidenceDataset(split="train", args=args, **common_args)
+        train_loader = loader_class(dataset=train_dataset, batch_size=args.batch_size, shuffle=True)
+    except Exception as e:
+        if 'The generated ligand positions with cache_id do not exist:' in str(e):
+            print("HAPPENING | Encountered the following exception when loading the confidence train dataset:")
+            print(str(e))
+            print("HAPPENING | We are still continuing because we want to try to generate the validation dataset if it has not been created yet:")
+            exception_flag = True
+        else: raise e
+
+    val_dataset = ConfidenceDataset(split="val", args=args, **common_args)
+    val_loader = loader_class(dataset=val_dataset, batch_size=args.batch_size, shuffle=True)
+
+    if exception_flag: raise Exception('We encountered the exception during train dataset loading: ', e)
+    return train_loader, val_loader
+
+
+if __name__ == '__main__':
+    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
+    with open(f'{args.original_model_dir}/model_parameters.yml') as f:
+        score_model_args = Namespace(**yaml.full_load(f))
+
+    # construct loader
+    train_loader, val_loader = construct_loader_confidence(args, device)
+    model = get_model(score_model_args if args.transfer_weights else args, device, t_to_sigma=None, confidence_mode=True)
+    optimizer, scheduler = get_optimizer_and_scheduler(args, model, scheduler_mode=args.main_metric_goal)
+
+    if args.transfer_weights:
+        print("HAPPENING | Transferring weights from original_model_dir to the new model after using original_model_dir's arguments to construct the new model.")
+        checkpoint = torch.load(os.path.join(args.original_model_dir,args.ckpt), map_location=device)
+        model_state_dict = model.state_dict()
+        transfer_weights_dict = {k: v for k, v in checkpoint.items() if k in list(model_state_dict.keys())}
+        model_state_dict.update(transfer_weights_dict)  # update the layers with the pretrained weights
+        model.load_state_dict(model_state_dict)
+
+    elif args.restart_dir:
+        dict = torch.load(f'{args.restart_dir}/last_model.pt', map_location=torch.device('cpu'))
+        model.module.load_state_dict(dict['model'], strict=True)
+        optimizer.load_state_dict(dict['optimizer'])
+        print("Restarting from epoch", dict['epoch'])
+
+    numel = sum([p.numel() for p in model.parameters()])
+    print('Model with', numel, 'parameters')
+
+    if args.wandb:
+        wandb.init(
+            entity='entity',
+            settings=wandb.Settings(start_method="fork"),
+            project=args.project,
+            name=args.run_name,
+            config=args
+        )
+        wandb.log({'numel': numel})
+
+    # record parameters
+    run_dir = os.path.join(args.log_dir, args.run_name)
+    yaml_file_name = os.path.join(run_dir, 'model_parameters.yml')
+    save_yaml_file(yaml_file_name, args.__dict__)
+    args.device = device
+
+    train(args, model, optimizer, scheduler, train_loader, val_loader, run_dir)

confidence/dataset.py

@@ -0,0 +1,272 @@
+import itertools
+import math
+import os
+import pickle
+import random
+from argparse import Namespace
+from functools import partial
+import copy
+
+import numpy as np
+import pandas as pd
+import torch
+import yaml
+from torch_geometric.data import Dataset, Data
+from torch_geometric.loader import DataLoader
+from tqdm import tqdm
+
+from datasets.pdbbind import PDBBind
+from utils.diffusion_utils import get_t_schedule
+from utils.sampling import randomize_position, sampling
+from utils.utils import get_model
+from utils.diffusion_utils import t_to_sigma as t_to_sigma_compl
+
+
+class ListDataset(Dataset):
+    def __init__(self, list):
+        super().__init__()
+        self.data_list = list
+
+    def len(self) -> int:
+        return len(self.data_list)
+
+    def get(self, idx: int) -> Data:
+        return self.data_list[idx]
+
+def get_cache_path(args, split):
+    cache_path = args.cache_path
+    if not args.no_torsion:
+        cache_path += '_torsion'
+    if args.all_atoms:
+        cache_path += '_allatoms'
+    split_path = args.split_train if split == 'train' else args.split_val
+    cache_path = os.path.join(cache_path, f'limit{args.limit_complexes}_INDEX{os.path.splitext(os.path.basename(split_path))[0]}_maxLigSize{args.max_lig_size}_H{int(not args.remove_hs)}_recRad{args.receptor_radius}_recMax{args.c_alpha_max_neighbors}'
+                                       + ('' if not args.all_atoms else f'_atomRad{args.atom_radius}_atomMax{args.atom_max_neighbors}')
+                                       + ('' if args.no_torsion or args.num_conformers == 1 else
+                                           f'_confs{args.num_conformers}')
+                              + ('' if args.esm_embeddings_path is None else f'_esmEmbeddings'))
+    return cache_path
+
+def get_args_and_cache_path(original_model_dir, split):
+    with open(f'{original_model_dir}/model_parameters.yml') as f:
+        model_args = Namespace(**yaml.full_load(f))
+    return model_args, get_cache_path(model_args,split)
+
+
+
+class ConfidenceDataset(Dataset):
+    def __init__(self, cache_path, original_model_dir, split, device, limit_complexes,
+                 inference_steps, samples_per_complex, all_atoms,
+                 args, balance=False, use_original_model_cache=True, rmsd_classification_cutoff=2,
+                 cache_ids_to_combine= None, cache_creation_id=None):
+
+        super(ConfidenceDataset, self).__init__()
+
+        self.device = device
+        self.inference_steps = inference_steps
+        self.limit_complexes = limit_complexes
+        self.all_atoms = all_atoms
+        self.original_model_dir = original_model_dir
+        self.balance = balance
+        self.use_original_model_cache = use_original_model_cache
+        self.rmsd_classification_cutoff = rmsd_classification_cutoff
+        self.cache_ids_to_combine = cache_ids_to_combine
+        self.cache_creation_id = cache_creation_id
+        self.samples_per_complex = samples_per_complex
+
+        self.original_model_args, original_model_cache = get_args_and_cache_path(original_model_dir, split)
+        self.complex_graphs_cache = original_model_cache if self.use_original_model_cache else get_cache_path(args, split)
+        print('Using the cached complex graphs of the original model args' if self.use_original_model_cache else 'Not using the cached complex graphs of the original model args. Instead the complex graphs are used that are at the location given by the dataset parameters given to confidence_train.py')
+        print(self.complex_graphs_cache)
+        if not os.path.exists(os.path.join(self.complex_graphs_cache, "heterographs.pkl")):
+            print(f'HAPPENING | Complex graphs path does not exist yet: {os.path.join(self.complex_graphs_cache, "heterographs.pkl")}. For that reason, we are now creating the dataset.')
+            PDBBind(transform=None, root=args.data_dir, limit_complexes=args.limit_complexes,
+                    receptor_radius=args.receptor_radius,
+                    cache_path=args.cache_path, split_path=args.split_val if split == 'val' else args.split_train,
+                    remove_hs=args.remove_hs, max_lig_size=None,
+                    c_alpha_max_neighbors=args.c_alpha_max_neighbors,
+                    matching=not args.no_torsion, keep_original=True,
+                    popsize=args.matching_popsize,
+                    maxiter=args.matching_maxiter,
+                    all_atoms=args.all_atoms,
+                    atom_radius=args.atom_radius,
+                    atom_max_neighbors=args.atom_max_neighbors,
+                    esm_embeddings_path=args.esm_embeddings_path,
+                    require_ligand=True)
+
+        print(f'HAPPENING | Loading complex graphs from: {os.path.join(self.complex_graphs_cache, "heterographs.pkl")}')
+        with open(os.path.join(self.complex_graphs_cache, "heterographs.pkl"), 'rb') as f:
+            complex_graphs = pickle.load(f)
+        self.complex_graph_dict = {d.name: d for d in complex_graphs}
+
+        self.full_cache_path = os.path.join(cache_path, f'model_{os.path.splitext(os.path.basename(original_model_dir))[0]}'
+                                            f'_split_{split}_limit_{limit_complexes}')
+
+        if (not os.path.exists(os.path.join(self.full_cache_path, "ligand_positions.pkl")) and self.cache_creation_id is None) or \
+                (not os.path.exists(os.path.join(self.full_cache_path, f"ligand_positions_id{self.cache_creation_id}.pkl")) and self.cache_creation_id is not None):
+            os.makedirs(self.full_cache_path, exist_ok=True)
+            self.preprocessing(original_model_cache)
+
+        if self.cache_ids_to_combine is None:
+            print(f'HAPPENING | Loading positions and rmsds from: {os.path.join(self.full_cache_path, "ligand_positions.pkl")}')
+            with open(os.path.join(self.full_cache_path, "ligand_positions.pkl"), 'rb') as f:
+                self.full_ligand_positions, self.rmsds = pickle.load(f)
+            if os.path.exists(os.path.join(self.full_cache_path, "complex_names_in_same_order.pkl")):
+                with open(os.path.join(self.full_cache_path, "complex_names_in_same_order.pkl"), 'rb') as f:
+                    generated_rmsd_complex_names = pickle.load(f)
+            else:
+                print('HAPPENING | The path, ', os.path.join(self.full_cache_path, "complex_names_in_same_order.pkl"),
+                      ' does not exist. \n => We assume that means that we are using a ligand_positions.pkl where the '
+                      'code was not saving the complex names for them yet. We now instead use the complex names of '
+                      'the dataset that the original model used to create the ligand positions and RMSDs.')
+                with open(os.path.join(original_model_cache, "heterographs.pkl"), 'rb') as f:
+                    original_model_complex_graphs = pickle.load(f)
+                    generated_rmsd_complex_names = [d.name for d in original_model_complex_graphs]
+            assert (len(self.rmsds) == len(generated_rmsd_complex_names))
+        else:
+            all_rmsds_unsorted, all_full_ligand_positions_unsorted, all_names_unsorted = [], [], []
+            for idx, cache_id in enumerate(self.cache_ids_to_combine):
+                print(f'HAPPENING | Loading positions and rmsds from cache_id from the path: {os.path.join(self.full_cache_path, "ligand_positions_"+ str(cache_id)+ ".pkl")}')
+                if not os.path.exists(os.path.join(self.full_cache_path, f"ligand_positions_id{cache_id}.pkl")): raise Exception(f'The generated ligand positions with cache_id do not exist: {cache_id}') # be careful with changing this error message since it is sometimes cought in a try catch
+                with open(os.path.join(self.full_cache_path, f"ligand_positions_id{cache_id}.pkl"), 'rb') as f:
+                    full_ligand_positions, rmsds = pickle.load(f)
+                with open(os.path.join(self.full_cache_path, f"complex_names_in_same_order_id{cache_id}.pkl"), 'rb') as f:
+                    names_unsorted = pickle.load(f)
+                all_names_unsorted.append(names_unsorted)
+                all_rmsds_unsorted.append(rmsds)
+                all_full_ligand_positions_unsorted.append(full_ligand_positions)
+            names_order = list(set(sum(all_names_unsorted, [])))
+            all_rmsds, all_full_ligand_positions, all_names = [], [], []
+            for idx, (rmsds_unsorted, full_ligand_positions_unsorted, names_unsorted) in enumerate(zip(all_rmsds_unsorted,all_full_ligand_positions_unsorted, all_names_unsorted)):
+                name_to_pos_dict = {name: (rmsd, pos) for name, rmsd, pos in zip(names_unsorted, full_ligand_positions_unsorted, rmsds_unsorted) }
+                intermediate_rmsds = [name_to_pos_dict[name][1] for name in names_order]
+                all_rmsds.append((intermediate_rmsds))
+                intermediate_pos = [name_to_pos_dict[name][0] for name in names_order]
+                all_full_ligand_positions.append((intermediate_pos))
+            self.full_ligand_positions, self.rmsds = [], []
+            for positions_tuple in list(zip(*all_full_ligand_positions)):
+                self.full_ligand_positions.append(np.concatenate(positions_tuple, axis=0))
+            for positions_tuple in list(zip(*all_rmsds)):
+                self.rmsds.append(np.concatenate(positions_tuple, axis=0))
+            generated_rmsd_complex_names = names_order
+        print('Number of complex graphs: ', len(self.complex_graph_dict))
+        print('Number of RMSDs and positions for the complex graphs: ', len(self.full_ligand_positions))
+
+        self.all_samples_per_complex = samples_per_complex * (1 if self.cache_ids_to_combine is None else len(self.cache_ids_to_combine))
+
+        self.positions_rmsds_dict = {name: (pos, rmsd) for name, pos, rmsd in zip (generated_rmsd_complex_names, self.full_ligand_positions, self.rmsds)}
+        self.dataset_names = list(set(self.positions_rmsds_dict.keys()) & set(self.complex_graph_dict.keys()))
+        if limit_complexes > 0:
+            self.dataset_names = self.dataset_names[:limit_complexes]
+
+    def len(self):
+        return len(self.dataset_names)
+
+    def get(self, idx):
+        complex_graph = copy.deepcopy(self.complex_graph_dict[self.dataset_names[idx]])
+        positions, rmsds = self.positions_rmsds_dict[self.dataset_names[idx]]
+
+        if self.balance:
+            if isinstance(self.rmsd_classification_cutoff, list): raise ValueError("a list for --rmsd_classification_cutoff can only be used without --balance")
+            label = random.randint(0, 1)
+            success = rmsds < self.rmsd_classification_cutoff
+            n_success = np.count_nonzero(success)
+            if label == 0 and n_success != self.all_samples_per_complex:
+                # sample negative complex
+                sample = random.randint(0, self.all_samples_per_complex - n_success - 1)
+                lig_pos = positions[~success][sample]
+                complex_graph['ligand'].pos = torch.from_numpy(lig_pos)
+            else:
+                # sample positive complex
+                if n_success > 0: # if no successfull sample returns the matched complex
+                    sample = random.randint(0, n_success - 1)
+                    lig_pos = positions[success][sample]
+                    complex_graph['ligand'].pos = torch.from_numpy(lig_pos)
+            complex_graph.y = torch.tensor(label).float()
+        else:
+            sample = random.randint(0, self.all_samples_per_complex - 1)
+            complex_graph['ligand'].pos = torch.from_numpy(positions[sample])
+            complex_graph.y = torch.tensor(rmsds[sample] < self.rmsd_classification_cutoff).float().unsqueeze(0)
+            if isinstance(self.rmsd_classification_cutoff, list):
+                complex_graph.y_binned = torch.tensor(np.logical_and(rmsds[sample] < self.rmsd_classification_cutoff + [math.inf],rmsds[sample] >= [0] + self.rmsd_classification_cutoff), dtype=torch.float).unsqueeze(0)
+                complex_graph.y = torch.tensor(rmsds[sample] < self.rmsd_classification_cutoff[0]).unsqueeze(0).float()
+            complex_graph.rmsd = torch.tensor(rmsds[sample]).unsqueeze(0).float()
+
+        complex_graph['ligand'].node_t = {'tr': 0 * torch.ones(complex_graph['ligand'].num_nodes),
+                                          'rot': 0 * torch.ones(complex_graph['ligand'].num_nodes),
+                                          'tor': 0 * torch.ones(complex_graph['ligand'].num_nodes)}
+        complex_graph['receptor'].node_t = {'tr': 0 * torch.ones(complex_graph['receptor'].num_nodes),
+                                            'rot': 0 * torch.ones(complex_graph['receptor'].num_nodes),
+                                            'tor': 0 * torch.ones(complex_graph['receptor'].num_nodes)}
+        if self.all_atoms:
+            complex_graph['atom'].node_t = {'tr': 0 * torch.ones(complex_graph['atom'].num_nodes),
+                                            'rot': 0 * torch.ones(complex_graph['atom'].num_nodes),
+                                            'tor': 0 * torch.ones(complex_graph['atom'].num_nodes)}
+        complex_graph.complex_t = {'tr': 0 * torch.ones(1), 'rot': 0 * torch.ones(1), 'tor': 0 * torch.ones(1)}
+        return complex_graph
+
+    def preprocessing(self, original_model_cache):
+        t_to_sigma = partial(t_to_sigma_compl, args=self.original_model_args)
+
+        model = get_model(self.original_model_args, self.device, t_to_sigma=t_to_sigma, no_parallel=True)
+        state_dict = torch.load(f'{self.original_model_dir}/best_model.pt', map_location=torch.device('cpu'))
+        model.load_state_dict(state_dict, strict=True)
+        model = model.to(self.device)
+        model.eval()
+
+        tr_schedule = get_t_schedule(inference_steps=self.inference_steps)
+        rot_schedule = tr_schedule
+        tor_schedule = tr_schedule
+        print('common t schedule', tr_schedule)
+
+        print('HAPPENING | loading cached complexes of the original model to create the confidence dataset RMSDs and predicted positions. Doing that from: ', os.path.join(self.complex_graphs_cache, "heterographs.pkl"))
+        with open(os.path.join(original_model_cache, "heterographs.pkl"), 'rb') as f:
+            complex_graphs = pickle.load(f)
+        dataset = ListDataset(complex_graphs)
+        loader = DataLoader(dataset=dataset, batch_size=1, shuffle=False)
+
+        rmsds, full_ligand_positions, names = [], [], []
+        for idx, orig_complex_graph in tqdm(enumerate(loader)):
+            data_list = [copy.deepcopy(orig_complex_graph) for _ in range(self.samples_per_complex)]
+            randomize_position(data_list, self.original_model_args.no_torsion, False, self.original_model_args.tr_sigma_max)
+
+            predictions_list = None
+            failed_convergence_counter = 0
+            while predictions_list is None:
+                try:
+                    predictions_list, confidences = sampling(data_list=data_list, model=model, inference_steps=self.inference_steps,
+                                                             tr_schedule=tr_schedule, rot_schedule=rot_schedule, tor_schedule=tor_schedule,
+                                                             device=self.device, t_to_sigma=t_to_sigma, model_args=self.original_model_args)
+                except Exception as e:
+                    if 'failed to converge' in str(e):
+                        failed_convergence_counter += 1
+                        if failed_convergence_counter > 5:
+                            print('| WARNING: SVD failed to converge 5 times - skipping the complex')
+                            break
+                        print('| WARNING: SVD failed to converge - trying again with a new sample')
+                    else:
+                        raise e
+            if failed_convergence_counter > 5: predictions_list = data_list
+            if self.original_model_args.no_torsion:
+                orig_complex_graph['ligand'].orig_pos = (orig_complex_graph['ligand'].pos.cpu().numpy() + orig_complex_graph.original_center.cpu().numpy())
+
+            filterHs = torch.not_equal(predictions_list[0]['ligand'].x[:, 0], 0).cpu().numpy()
+
+            if isinstance(orig_complex_graph['ligand'].orig_pos, list):
+                orig_complex_graph['ligand'].orig_pos = orig_complex_graph['ligand'].orig_pos[0]
+
+            ligand_pos = np.asarray([complex_graph['ligand'].pos.cpu().numpy()[filterHs] for complex_graph in predictions_list])
+            orig_ligand_pos = np.expand_dims(orig_complex_graph['ligand'].orig_pos[filterHs] - orig_complex_graph.original_center.cpu().numpy(), axis=0)
+            rmsd = np.sqrt(((ligand_pos - orig_ligand_pos) ** 2).sum(axis=2).mean(axis=1))
+
+            rmsds.append(rmsd)
+            full_ligand_positions.append(np.asarray([complex_graph['ligand'].pos.cpu().numpy() for complex_graph in predictions_list]))
+            names.append(orig_complex_graph.name[0])
+            assert(len(orig_complex_graph.name) == 1) # I just put this assert here because of the above line where I assumed that the list is always only lenght 1. Just in case it isn't maybe check what the names in there are.
+        with open(os.path.join(self.full_cache_path, f"ligand_positions{'' if self.cache_creation_id is None else '_id' + str(self.cache_creation_id)}.pkl"), 'wb') as f:
+            pickle.dump((full_ligand_positions, rmsds), f)
+        with open(os.path.join(self.full_cache_path, f"complex_names_in_same_order{'' if self.cache_creation_id is None else '_id' + str(self.cache_creation_id)}.pkl"), 'wb') as f:
+            pickle.dump((names), f)
+
+
+

data/protein_ligand_example_csv.csv

@@ -0,0 +1,2 @@
+protein_path,ligand
+examples/1cbr_protein.pdb,CCCC

data/splits/timesplit_no_lig_overlap_train

@@ -0,0 +1,16379 @@
+3dpf
+2zy1
+6h77
+5c0m
+4o2c
+3a2c
+3b2t
+1v2q
+4q87
+4j28
+3efw
+4mi6
+3k8c
+3s54
+2fda
+2q55
+5ylv
+4ozo
+3fzn
+3eoc
+4tn2
+3hfv
+5gg4
+5svi
+2r58
+5d1n
+4yqv
+1hs6
+3zxh
+1ft7
+4hww
+6b1j
+3di6
+5jaz
+5ema
+6gip
+5am5
+4bnt
+1fq6
+3pqz
+6eil
+2q11
+4nw7
+4ahr
+1p1o
+3le8
+3nzk
+1eub
+1eoc
+3std
+3qw8
+4za0
+5l8a
+5teg
+5u6j
+3g2n
+2wp1
+1zyr
+1xs7
+4m6q
+2jq9
+4m5h
+4d63
+3shb
+3i6m
+4hw3
+6eq5
+3hqy
+3zos
+6h96
+2gv6
+5yjb
+4lm4
+4lwv
+3g30
+5f32
+4dkr
+6b4l
+4j9a
+5w86
+4mpn
+5m51
+1ro6
+5q0m
+3bea
+1mu6
+2wtd
+3sr4
+6dqb
+5hh5
+5q0e
+2r03
+4xv9
+4y0a
+5ota
+3u5l
+3m3o
+4idz
+2w47
+2g5u
+6cec
+5wzs
+3iu7
+5mky
+3ibc
+2qwf
+3p76
+4np3
+4kb8
+1uyi
+4tkf
+2fai
+2xvd
+5ar8
+2lzg
+3e62
+1p6e
+5wqa
+1p02
+6gjr
+1nhu
+4yyl
+3jzq
+4pce
+4gvu
+4n7g
+3s9e
+3k3b
+1pmn
+1ogd
+2j4i
+1oss
+1v0k
+6cn6
+3k3e
+2btr
+1ppk
+6egw
+4zx1
+4zfi
+5yso
+2rfn
+5cep
+2xyu
+3eou
+4mme
+4fnn
+4og7
+4gkh
+6d1l
+6dlj
+5i94
+1eb2
+4qzw
+3fu0
+4aif
+5eec
+2qlj
+5cfb
+5sy3
+1mhw
+2flu
+3e6y
+4wiv
+5oot
+4bae
+6bfd
+3po6
+2wo9
+1rp7
+1l2s
+5ab1
+3pce
+2w6z
+1iew
+3e85
+3e12
+5arf
+4jgv
+1ikw
+6goo
+4zdu
+3wcb
+2w1h
+4x2i
+6ggv
+3h0b
+5nwe
+1o46
+5f2u
+2feq
+2jnw
+3gpj
+5uex
+1ndw
+1zhk
+4mcd
+3r4m
+4cps
+3gcp
+2g9v
+5try
+4c2v
+4og3
+3otq
+2yns
+4m0e
+5tkj
+2m3z
+3pa5
+3iww
+5o0b
+3m58
+5lz2
+5u6k
+4yw6
+3ewh
+3wc7
+1rhk
+3gi4
+2wd7
+3jq8
+5yr5
+6fnf
+3hx3
+4mz6
+1ol2
+4aa5
+1rry
+1ozv
+5jf6
+4mnq
+5wa5
+3nlb
+3f6e
+2xu5
+3wk7
+4p6w
+2pmc
+3byo
+5ehe
+6chq
+6c8x
+3pp1
+3uev
+3ad7
+5aqv
+3qzq
+2z97
+5d1r
+5ajy
+4k67
+2w1e
+5j3v
+3upk
+4gj8
+1td7
+5lz9
+6erw
+4rhx
+1hxw
+6cbf
+5qal
+1q8w
+4mf0
+5t1s
+1fzk
+4tju
+5t2y
+4el5
+2v87
+1epo
+2r5p
+4pis
+3ehy
+5erg
+2r1x
+2f8i
+2wi7
+5wre
+3dek
+3hp2
+4bcw
+4uau
+5ap3
+3udy
+2ksa
+5ekg
+5lh8
+2uw7
+3uvq
+2glp
+3m3r
+6bsx
+2ddf
+6eqm
+3ufl
+2mwp
+1h4n
+5f4r
+4ybj
+4kz8
+2z60
+4ogi
+5ufp
+4uiw
+3igv
+5b56
+1bn3
+2xg9
+1u59
+4m5j
+5oua
+5qaa
+6h7j
+2o7v
+5v7t
+2r02
+2j94
+5o87
+1o3k
+4mr4
+5am7
+6g35
+1ljt
+3srv
+4bi6
+2yoh
+4zed
+5tln
+3rkb
+5jiq
+2ll6
+1tkz
+2vj6
+3qrk
+3c45
+4pks
+3nf3
+2oaz
+3pwk
+3ryy
+2y7k
+6dm8
+5ism
+5i7u
+2qtn
+5cuu
+5n1y
+4kot
+5wa1
+2wyi
+188l
+3h2o
+1sw1
+1zuc
+2nnq
+4l8m
+5j75
+4afe
+5nwi
+2n06
+3l4w
+3ibl
+4l31
+4ruy
+4ufm
+5sxk
+4hnn
+5q0x
+4zij
+3fdn
+5fe6
+3nuj
+5h8x
+5ets
+4x5z
+1w3k
+3hs9
+3ro0
+3zi8
+1nl4
+5cao
+4gjd
+2p3d
+2z8e
+3w2p
+1f28
+2iw6
+1h9l
+4o78
+1soj
+3krw
+5g4n
+3t4v
+5drr
+5o0a
+4c4i
+2vwx
+6miq
+6guk
+5c3k
+5fhn
+3o75
+5kap
+4zo5
+2jko
+3iss
+4eg6
+4fcd
+3v7s
+4bcf
+2qyn
+1ikx
+6f6i
+5os5
+1om2
+6axb
+3s3n
+1fl6
+4luo
+1qiw
+5f2w
+3udj
+5tbj
+5k0c
+5ndf
+1ek2
+4zb8
+3r9o
+5qah
+3avb
+4c52
+4ybi
+5mi6
+6fu5
+1yc4
+4ymq
+1xn3
+3dpo
+4z7q
+5ehn
+5mri
+2vcw
+2byp
+5du8
+1i7z
+1ec3
+5jmx
+4yxi
+3iw7
+3l4v
+3wcf
+5igk
+1sz0
+4x6i
+1wum
+1d3d
+3rv6
+1x7r
+2c57
+4u82
+6cwh
+2vrj
+2xag
+5nw7
+5qau
+4ocz
+3chg
+5oh1
+1hkm
+3ayd
+3mn8
+5ngf
+4bo6
+1b57
+5wzt
+4uzd
+1cyn
+3eys
+4umc
+6guh
+2e99
+2xb8
+5tus
+6ap7
+4ajn
+4od7
+6g2l
+5v40
+5wjj
+2bow
+3kc0
+3zs1
+4igq
+3b24
+5e8r
+5a4l
+4ci1
+3qd0
+2ohv
+3uw5
+3lp4
+3ftw
+6c5q
+4e0x
+4hbw
+2yoj
+4b2i
+3mkn
+1i5r
+5tig
+5bqs
+5j2x
+1oz0
+5a2s
+3fkv
+3qnj
+5w5o
+2cht
+5dj5
+5oje
+5o9y
+4hxs
+5t36
+5gmh
+4eh9
+3kr2
+3rk5
+2b54
+4ufh
+2zxa
+4mji
+4e70
+4huo
+3zrk
+3wb5
+2iw8
+4ge4
+6ccs
+5nwc
+5lmb
+4i6h
+4dhu
+5y6e
+5bvn
+4lxd
+3b3s
+5j9f
+3jzp
+2no3
+1ouk
+5epk
+1fo2
+1g7g
+4dve
+6au2
+4wt6
+3w2q
+5tdr
+4xx9
+1jg0
+2n27
+5lyn
+1p1q
+1hi3
+2wih
+4hym
+4ew3
+5ixq
+1h60
+3ppk
+4x8s
+5nzn
+3ddf
+1yvm
+1kv2
+4djq
+3sym
+4fz3
+1ze8
+5ia1
+2qcm
+5m9d
+2erz
+4g0l
+5tqg
+2i5j
+3ow6
+1met
+3r42
+5tdi
+3i97
+1alw
+1doj
+3f5j
+2r5q
+6eir
+2ie4
+5cav
+1n4m
+5wj6
+2o7n
+2r4f
+5nkn
+2g6p
+1bnu
+4uc5
+3kze
+5dg6
+1nym
+6cki
+4kmz
+6mjf
+4ch8
+6hsh
+1txr
+4zj8
+6ggn
+4qzx
+4qsu
+5ety
+1o5a
+4otf
+1pgp
+5wex
+4p4b
+4h4m
+3k54
+4j53
+5dva
+1e3v
+4wop
+1c3i
+4jls
+5mxv
+4qrc
+3fvn
+5ivn
+4xu1
+2o4n
+1m5b
+3t1a
+1mqi
+5n84
+5ifu
+5eie
+5lgr
+5unf
+5w5u
+5tq1
+3dkj
+5ye8
+1br8
+3tsd
+6dvn
+1oxg
+4kz5
+5v3h
+5in9
+4avi
+220l
+4b7z
+2xbv
+3twx
+5zo8
+2w4i
+5opc
+4q8y
+5acy
+5ncz
+3in4
+3arp
+4rt1
+4mq6
+6ey7
+3uzd
+2b9a
+5o4z
+2ow0
+6f5w
+1x7q
+5agv
+5pzm
+6afh
+4z1e
+5flp
+5xsr
+1b4z
+5f1r
+1o2g
+1uwh
+3bmo
+1p19
+2ans
+6fbv
+4lrr
+1gi9
+4xuh
+2weo
+5xpi
+4btm
+5h0e
+4gye
+6bgu
+5wlt
+5oul
+1t49
+5k1v
+5lz4
+1fsw
+6cvx
+6c3e
+6cj5
+5fqc
+1cze
+3uwl
+5l3g
+5jnc
+3ps6
+6csr
+6bw4
+3hau
+3ud9
+6csp
+5uyu
+5lqf
+4bfz
+2xp6
+3sjo
+5os3
+4z8d
+5un1
+6esy
+3kjq
+3sl4
+5mgl
+5dp5
+3n0h
+5fao
+4dmy
+6dxl
+4qw0
+3q92
+3rwf
+2xke
+4gfo
+3t03
+3pd8
+3f18
+4utr
+4cu7
+5n1s
+3wkb
+2zdt
+2yit
+4cmt
+5dpa
+4rt0
+2y5h
+3cde
+5lud
+3vva
+2wev
+5mrm
+4ypf
+5ea4
+3fr5
+4tkh
+5tyi
+1db1
+4hxj
+3hu1
+5ftg
+2eh8
+1jiz
+1gww
+5nih
+1rr6
+4fp1
+4y24
+1fki
+3lea
+4lch
+2xk1
+4qjr
+4os1
+4pft
+5eb3
+5qad
+3cpc
+3c94
+5jhb
+6b22
+3vfj
+6hmx
+1kuk
+2gh9
+1xh6
+3eyl
+5hng
+3nes
+3i1y
+2anl
+4xit
+3n2e
+2n0u
+5hdx
+4zt6
+4f6v
+5n7g
+4mc1
+4yc9
+3q6w
+2cmb
+5g4m
+3pww
+5d7x
+4cpr
+3muf
+5f95
+5o9o
+5e2w
+2chx
+2pk6
+3bet
+6cje
+2rd6
+5khk
+4eh5
+1i9l
+1swg
+4q4o
+1kf0
+1llb
+5aep
+2zp0
+3zze
+5csp
+5eyk
+3u7n
+4z0e
+5fh6
+5kj2
+2o4k
+5llg
+4b5t
+3li2
+3of8
+3mvh
+1jut
+1auj
+5awj
+5x73
+1eol
+1f90
+4crd
+1d2e
+2oht
+1ua4
+2h9m
+5byz
+3rse
+4mp2
+3hhk
+5tur
+2xxr
+2r3w
+4u6e
+4qlt
+3db8
+2bub
+1ths
+3s8o
+3x1k
+3eg6
+1jaq
+3anr
+3ua8
+2kfg
+4m3g
+3d3x
+5a5v
+3nuu
+1jh1
+1iy7
+5dlz
+4wpf
+4pvy
+1sve
+4qmy
+1pyg
+5drq
+4ivt
+2gcd
+1azg
+3wjw
+2qiq
+4leq
+2qfu
+1thz
+1y3a
+5yjp
+5cgv
+1oi9
+3cd5
+3w1f
+2v12
+5f37
+1e03
+3b67
+4q8x
+5uk8
+4v25
+3u8k
+5l8o
+5hlb
+4qfn
+4li7
+1dth
+4gkm
+6f3e
+4rn4
+3vtb
+5l6i
+4lrh
+4jvq
+5est
+3gtc
+2jbu
+2yfx
+4yx9
+5bs0
+5ea3
+1vyf
+5a4e
+3kku
+3cyw
+3tvx
+5mkj
+6b4d
+4bny
+5j8x
+5tza
+1uk1
+5yg2
+1d6v
+5t4e
+2j4g
+5tci
+1z9y
+2ca8
+4hyf
+5mg2
+4bcq
+1lqf
+6b3v
+5htc
+2qh6
+4pul
+1gj7
+4mmf
+3sfg
+5ap2
+2oi9
+4h81
+3s77
+1awh
+3s74
+3oyp
+4u5u
+6euz
+6eq3
+4obq
+4u4x
+2r23
+4tjw
+4anq
+2a4z
+2mwy
+4pzh
+5m7m
+4mzj
+1o3l
+3d62
+2xow
+5hog
+3rwq
+2kmx
+4fpf
+5ajx
+1f5k
+4de7
+5c7c
+3q77
+4pnr
+5tkb
+5hz8
+2vle
+5ho7
+5xmx
+2xpk
+6mdq
+3tge
+5fls
+3oku
+5jr2
+3e0q
+2l98
+2uzj
+5csz
+5yr6
+3f7i
+1ex8
+6ft7
+2bdj
+5l7g
+1cnx
+1mmp
+5j8m
+2ay2
+6fnq
+5eng
+1pg2
+4a6v
+3ivv
+2zq2
+4wag
+3s7a
+4is6
+1lhw
+5tq8
+3v51
+4w9x
+5lto
+1o86
+5a69
+4zg7
+4unq
+2q88
+5c1m
+6bgy
+4fil
+2qi1
+2x7o
+6exj
+3nkk
+1q65
+2xxw
+3l7d
+1om9
+4jq7
+1lf2
+4xg4
+3sgx
+1sdt
+3ow4
+4kp8
+2cem
+5nzo
+3oaw
+2wwj
+2v6n
+4g11
+1lf3
+3ljz
+1amn
+6hm4
+3jq9
+1qx1
+2wi5
+4old
+1g5s
+3tic
+3cs8
+4er4
+5kr0
+2pe0
+5abf
+3v04
+4e6q
+2yc3
+3bys
+5jeo
+4b00
+4bek
+4fci
+4qna
+5eh5
+4r02
+5ef7
+4hu1
+5v5n
+5ehp
+5yr4
+3dt1
+4cd0
+6gl8
+3uvl
+3v0l
+5ne5
+4nvp
+4qok
+4ipj
+4zyi
+1o3j
+1lrt
+3iog
+5zwe
+4hbn
+2gz2
+5tco
+3gc5
+4l6t
+5mnx
+5m23
+2etr
+5gwy
+3b5j
+1oai
+2ath
+2v11
+5wuk
+2ym6
+4mwu
+5u69
+4i8w
+6g86
+4f7l
+2yi5
+4nan
+1j37
+2xup
+2byi
+4f63
+3m3x
+2bpm
+2wb5
+6fel
+3tdc
+4z1s
+4fgy
+3cyz
+6bh1
+3m8p
+2df6
+3jrs
+2wmv
+6fau
+5oah
+3d50
+3aza
+1uwt
+4nct
+1jqe
+4nb3
+4y3b
+1okw
+1db4
+4n98
+3inh
+6hzp
+1hdt
+6aoy
+4o3u
+5es1
+5i9i
+6ccm
+5nev
+5wp5
+4bo7
+5byy
+3su2
+4hlf
+4c1g
+3sww
+2os9
+4men
+2wxd
+4qq4
+5eei
+4hnf
+3b26
+3eht
+3oqf
+3dd0
+3si4
+4v1f
+4e6d
+2c8w
+1jyi
+4ob1
+5x4n
+5alc
+2pbw
+5eym
+4e28
+2xui
+4qtl
+5jz9
+3chr
+2p2a
+1v1j
+4o3b
+5v83
+5v5e
+3byu
+7abp
+5jga
+3sw2
+5jgb
+4dtk
+1q91
+3cqu
+1nje
+3vhd
+4knm
+4yrt
+1bgo
+5ih5
+6afe
+4res
+1akt
+1kyv
+1sqn
+4oas
+5u5l
+1v79
+3hp5
+5ivc
+1q84
+3avg
+3tws
+1rmz
+4rse
+5zae
+5w2q
+5glu
+5b4w
+5f62
+4y6r
+4mha
+5etk
+5nzm
+4o2b
+4cwq
+2pj8
+3lvw
+2fw3
+1dwc
+3zln
+3ejq
+2f6y
+6bic
+4p0x
+2cct
+3dxh
+2xhx
+1d3q
+3qbh
+5tkt
+4acd
+1yt7
+5aki
+3zy2
+4tw7
+3b3w
+1nkm
+5uoo
+2l75
+4l7u
+4iut
+1pbq
+4dow
+3hab
+1owe
+6ft3
+4y2q
+2hs1
+6gu6
+4dpi
+5aac
+4j2c
+1s50
+6nao
+5fox
+3jvs
+4ozl
+1zub
+5wxp
+5iuh
+5oh7
+2c90
+5owl
+3oof
+4ona
+4ydg
+1h08
+1uk0
+2fxs
+3f37
+4xtz
+4mww
+6gg4
+4zz3
+5cf4
+3blr
+4twc
+3zsz
+4er2
+4uuh
+1i8j
+6faw
+5opv
+3voz
+3d1g
+5q0n
+3kfc
+3qqs
+1oay
+1uvs
+5os4
+4abe
+1r0x
+4jwr
+1l6m
+3nf6
+3qip
+1nlt
+3zdg
+3o84
+3ehn
+3k16
+5szb
+2j2u
+5aiv
+2weg
+3dz4
+4jdf
+1ttm
+3wyy
+3ifo
+5j41
+3fl5
+1q1m
+2y80
+3fr4
+3o56
+1yrs
+2gvv
+3uxk
+2zq0
+5uac
+3sfi
+2wk6
+184l
+6gvz
+6dik
+3u8w
+3rxb
+2qpq
+3nwe
+13gs
+4gpl
+4rxz
+4qbm
+5y9l
+4nh8
+4rra
+5ey8
+4j03
+2ou7
+5tt3
+1uwf
+4dgr
+5xzr
+4mw4
+3n5k
+4umq
+2l6e
+6drg
+6ghp
+2ksp
+1sps
+5hna
+6gnp
+3dxk
+5eom
+1e34
+2xfi
+2itp
+2zlf
+4iva
+2y4m
+4o1b
+2o9v
+4i7f
+4ezw
+1k4h
+2xgm
+4nud
+3m55
+6ezg
+4x34
+1czo
+5fqr
+5he4
+2h6k
+3fmr
+4kzb
+2g9q
+3tpx
+4hpy
+2bo4
+4nhx
+4mvh
+2nq7
+3n5u
+3rjw
+3emg
+2xm9
+1kzn
+5faq
+5f1u
+1c6y
+4j8g
+5lvr
+1xuo
+2vgo
+1grp
+4uxb
+2rol
+5cbm
+6hke
+5tr6
+1uyc
+3d6o
+5awu
+4wz8
+5egm
+1duv
+6b4n
+1t7j
+5aqj
+4jh0
+4q7s
+5cpr
+1bqm
+1g9c
+4fem
+4wnk
+4uv8
+5hgq
+1ayu
+6co4
+6ck3
+3uzj
+2qwe
+3u2q
+1nl9
+6bd1
+5yzd
+4r92
+2i0j
+4hvh
+4ool
+3s0n
+1ba8
+3b4p
+2ea4
+5k76
+2xzq
+4yur
+1pq6
+2y81
+3f8s
+2hwh
+1q4w
+4h39
+3zmh
+1ftl
+4ok5
+2qcn
+3qtf
+1g52
+3vw0
+3i4a
+5x79
+2am2
+2ym7
+5odu
+3adu
+6cea
+3t6y
+4pgc
+1k1j
+3d14
+3el4
+4nyf
+2y0j
+2w1c
+5l7k
+4n6g
+5cp5
+3ekr
+2isc
+5cbs
+5hvu
+4n00
+5mpn
+5zoo
+3v43
+1t69
+1oj5
+5lle
+5wef
+2ya8
+4ezk
+2c6o
+5vrl
+1b4d
+5za8
+1i80
+2ccb
+6ce2
+4cfx
+3m5e
+1k22
+6eea
+3s3m
+5t4f
+2xyn
+6br3
+3krl
+3upy
+2rm0
+5q1f
+4app
+7gch
+5hx8
+4ok6
+4zla
+5g3w
+1h27
+1fkb
+1idg
+5c8n
+1h46
+3w33
+3rzb
+4xg6
+2rib
+2uuo
+1r1h
+5agu
+6dcz
+1r58
+1nlo
+3g70
+2ofu
+4k6z
+4x13
+5ihc
+2yiw
+6bgw
+6gji
+1tve
+1gpn
+5tq5
+4ujb
+4l33
+6cqz
+4avh
+5uga
+5tzy
+5op6
+1m7d
+4aia
+2bxt
+4qgh
+3uuo
+5vkm
+4jr3
+1wdy
+4o4r
+2c3j
+2r9m
+5ual
+3vfq
+4c16
+6h7l
+3tv6
+2pyn
+2bj4
+4qvy
+4abk
+3m53
+3cwk
+3d52
+1a4q
+4rvl
+1d04
+3thd
+1ogz
+3luo
+2ftd
+4ran
+3uph
+3c8b
+2xp3
+2vtl
+1fj4
+2zm3
+3rxl
+1xmy
+3uo5
+1q1y
+2ql9
+1bhf
+2jg8
+6fuh
+2q2n
+4aof
+3ff3
+1heg
+4cae
+6fkq
+4i9i
+3ask
+1dbj
+4zbf
+2io6
+3d1y
+5t2l
+3kwf
+1juj
+3i0r
+4gui
+2hy0
+3i7c
+6g9m
+3nnx
+5u48
+2v2v
+6f5h
+5mwh
+5m0d
+5fue
+3k5k
+5edi
+4d2p
+2cm7
+8cpa
+3doy
+3k99
+1vyg
+6ckw
+6bmv
+5m4q
+5j4n
+4tpt
+2jbv
+3uqr
+4di2
+5ue4
+5enj
+3csl
+3m35
+4a9t
+5khx
+5f0h
+4wq3
+4ktu
+2qbr
+1sbr
+2f6z
+6cgt
+1t4v
+5nra
+3fed
+5l2o
+3mb7
+4bzn
+1efi
+3nrm
+5kjm
+2cle
+1j07
+3h9k
+4d83
+5n4s
+5yie
+5yz7
+5cr7
+4e35
+6di0
+3omm
+4w9n
+4z1j
+6b98
+4x61
+4x6m
+2pu1
+4q1s
+3n7a
+4qbb
+6fng
+5ya5
+3qiz
+5awt
+3sv8
+3doz
+6dlx
+4rkx
+3h91
+4uma
+1lvc
+1uy9
+5gso
+5gsa
+5drc
+4j5e
+2o22
+4loo
+5t66
+3rz0
+5unj
+4g17
+6bhv
+3cii
+5qcl
+4nw6
+4qqi
+3r0i
+5uff
+4lbp
+4bqy
+4l51
+5akh
+2yjb
+3dx0
+3v49
+3g42
+5xpp
+2c4g
+2bz6
+4fny
+3x00
+4nat
+6f8g
+4ppc
+3lpb
+3tlh
+4bcs
+3nok
+3lfs
+3o9h
+2vf6
+3we4
+2yig
+2kgi
+5llc
+2jgs
+1fkh
+5ogb
+5w8i
+1a42
+5web
+2llq
+5j7b
+2c6l
+4u0e
+2g9r
+1lt5
+5ai5
+3khj
+6hh3
+4uac
+5nob
+1ynd
+2on6
+6hd4
+3msj
+5ut4
+2wkz
+5jvi
+3gww
+4uja
+1pq3
+4u5v
+4z83
+6dh8
+1h36
+3vjk
+1unh
+3hv7
+2vxj
+3l38
+1a08
+5lgo
+1fgi
+5fqs
+5a5p
+5oss
+4o0z
+1gvx
+1xur
+5vqe
+2j7w
+2adm
+3b1m
+5zah
+1iwq
+4zx8
+3wkd
+5mtv
+5myd
+3t6r
+2c6n
+2c6e
+2gpp
+3opm
+5n9r
+3r0t
+6gue
+4b70
+3gnv
+5nyz
+1fh7
+5wae
+2bvs
+1tow
+1toj
+5een
+5fog
+6e9a
+5jek
+1jet
+5qck
+6coj
+1if8
+5t6g
+4b85
+2fm0
+5eje
+1ezf
+3twd
+3mo8
+4rvt
+6bbu
+4ejl
+1waw
+4heu
+4hgs
+6b67
+2cmc
+3arv
+4r4t
+1j80
+5ads
+6abp
+3el7
+6fex
+4m5o
+3qgw
+5t8o
+3pcn
+3nw3
+5w84
+3bqn
+4pin
+2w3o
+5yic
+4lwh
+3vo3
+4unr
+4cp5
+4b73
+6c6o
+1xz8
+5zo9
+2qn3
+2g1r
+4xc2
+1oe8
+2z94
+5aic
+4m12
+1sld
+2h96
+6cn5
+3bvb
+5x27
+3usx
+1ddm
+3fal
+2iit
+3fzt
+5vgi
+5jv1
+1py1
+5ktx
+4b35
+1m0q
+5kjk
+2vey
+6dkg
+5ei3
+6gbx
+3e01
+1p03
+4zuq
+4wbo
+4rfd
+3zlo
+5alb
+5b5o
+3liw
+2mwo
+4yjl
+4ab8
+5nau
+4at4
+1ke0
+1zz2
+2ccs
+5hz6
+2cll
+4m2v
+1q3d
+3ozt
+4ge7
+3pkd
+5t70
+3ts4
+5q0y
+5q1a
+3ogm
+5t18
+5c7d
+3g2k
+1ikv
+2j7b
+4obp
+4k6y
+2fb8
+3vbd
+5ung
+4b32
+2qlq
+3jzs
+3tiz
+1v2k
+6ee2
+3zzf
+1hpv
+5xo2
+5e2l
+1jn4
+5aly
+2jjr
+4agm
+3rwp
+3r7r
+3zbf
+4url
+2wm0
+6dcy
+2brg
+5dlx
+4g2y
+5jzs
+5fd2
+5nqr
+4jpe
+3tkm
+5ti0
+5lxp
+3avi
+5zuj
+3p3h
+3d9v
+4gs8
+4cwn
+5q1i
+6b7a
+1gjc
+4dbn
+5yp5
+5ukj
+6arj
+4iuo
+4ibk
+5ali
+6fgq
+1b5h
+2bvx
+2y36
+3f81
+4e3f
+3t84
+4na7
+2ow7
+4ob0
+1qb1
+2iku
+5kh7
+1eve
+1k2v
+3npc
+2w68
+6gqm
+1b2h
+4dfl
+4l19
+3dri
+5ab9
+4qzs
+3sqq
+1d4l
+4dn0
+9hvp
+2lko
+1lev
+3aau
+4mib
+3g2l
+1lee
+1o5e
+4os4
+4b34
+3m56
+4qvw
+4i4f
+5wf6
+3u7k
+4j5d
+1r4w
+5iaw
+3buw
+1eou
+3vbq
+6g5j
+5t4u
+3fuh
+2pj4
+2zn7
+4n4v
+2obo
+1tjp
+6b1w
+1ru2
+1fh9
+4wcf
+6hh5
+5orz
+4xmb
+4jfv
+6ekq
+5q0i
+3h1z
+4ycv
+4ura
+4prn
+6ev0
+1v3x
+2pqz
+2xjx
+3k5i
+5dtq
+4pn1
+5bue
+3ik1
+2h5d
+4jbl
+2x97
+2q72
+2qpj
+1acj
+4k6t
+6b1e
+3f3e
+2yog
+4kio
+3qem
+3ued
+5mge
+2bfq
+3mxs
+4k1b
+4p6g
+3td4
+5mnh
+5ien
+3lk8
+2zx6
+1rql
+3rm4
+5u4c
+4kb7
+5ap1
+4rj8
+5tkk
+4j04
+6i8b
+2nxl
+3zm4
+4dy6
+3h0z
+4zyf
+3e7o
+5li3
+4qjw
+4aq4
+3kpu
+5oq7
+3h2c
+2qnq
+5jn9
+5lvx
+1r10
+2l8r
+5ega
+2wi3
+3rxk
+1upf
+6chm
+4w9k
+1vja
+4zw7
+5ov8
+5wag
+1x8j
+5a6b
+6br2
+4owv
+5ev8
+3drs
+6fdc
+3tvl
+5mlo
+2xaf
+3h0j
+4yas
+1o35
+5dex
+2x52
+3lgs
+3fkt
+3dcr
+2ax6
+6cnj
+4y5d
+4umb
+3l4y
+6fkp
+5aoj
+1ony
+6b95
+5ueu
+5ahj
+2xel
+2g1y
+6iiv
+3b1t
+5dht
+4zup
+4rlk
+1axs
+5fnj
+1izi
+4gvm
+4xmr
+5yhg
+3g45
+3tao
+2q7o
+1pme
+4rrs
+6gl3
+4pct
+2tsr
+3qtw
+4uua
+5abe
+3ozp
+4lc7
+2bu5
+5kya
+2q6h
+1n95
+3k84
+3bl1
+3ui2
+2cgv
+4ez5
+3t3c
+3puk
+6css
+1ghw
+3ezr
+5il1
+1a2c
+5j6l
+2gss
+1n1m
+5f1x
+5vt4
+4y62
+1bcd
+1pwu
+4fht
+3w9k
+3qu0
+1zxc
+2vtj
+3dnt
+2pu2
+4hej
+4x3e
+5ive
+2wo8
+4ozn
+5i2f
+6dj7
+4a6w
+4q1x
+2g63
+1jak
+5knr
+5y7z
+5npb
+2izx
+3jpx
+2x38
+4knr
+3rwe
+4aze
+4nrm
+6er3
+6d59
+5a2k
+4cr9
+5i29
+4mbi
+6ht1
+2d1n
+1uj5
+3ljt
+1kyn
+5osd
+5whc
+4qws
+3uwo
+3th9
+2r2l
+5n8v
+2y54
+4m5n
+1yy6
+6fui
+3lvp
+2gdo
+3hyf
+2zva
+3zmu
+2pt9
+5nxg
+4zsm
+5eif
+4qsk
+4djr
+2ceo
+4apo
+5gj9
+1q4x
+4wz5
+2vx9
+2qzl
+4keq
+2b17
+4kzu
+1bdr
+4zjr
+3ftz
+3cic
+2c2l
+4yo8
+1vj5
+1w4p
+1aaq
+2q9n
+2yay
+4bea
+1szd
+5acb
+5db1
+5ne1
+1vj6
+3n5h
+3kpv
+2hvc
+3arr
+3v0p
+2fqw
+5jf7
+3udq
+2n14
+5mk3
+2pj6
+4pp5
+4c68
+5q15
+3u8h
+4erw
+1sb1
+5iz8
+6iiu
+3txo
+4nah
+4ir6
+4ufy
+6fer
+4zba
+1jij
+2piy
+5td2
+4yzc
+5qap
+5yhl
+1h1p
+5ni7
+4pf3
+4a6b
+4uy1
+2xwy
+4el0
+2w71
+2p8h
+4d7b
+4yoj
+2ms4
+4iwz
+5ctb
+3t85
+4lpg
+5bvw
+3uri
+3hfb
+4mr3
+3lq8
+3mlb
+4qwf
+4ixv
+4njd
+5tzo
+5qa9
+5ai8
+3o23
+4rn6
+5i22
+5u2j
+4o72
+4ej8
+1rv1
+4ixh
+1w82
+3arx
+3k02
+3uf9
+5wij
+4oq5
+1z4o
+2qf6
+3lq5
+4yw2
+3u4h
+4f20
+4mg5
+5ufi
+4n8d
+6mu1
+5zag
+4az2
+4fbx
+4emv
+4bqt
+3t2q
+5ajw
+4q81
+2fqx
+4kza
+5k51
+3u4i
+5ho8
+3lkz
+4ciz
+1yy4
+3bmy
+3vf5
+2w6p
+6fa2
+4flh
+4w4s
+3i28
+3oyl
+2nn1
+5lrk
+4e3i
+4xg9
+4d8n
+3lik
+4aj2
+4jt8
+5j7w
+5std
+3h2a
+2pwd
+6bjo
+3nik
+5lxc
+3qce
+1vrt
+4qxo
+2y1w
+1wok
+5lne
+5am6
+5er1
+3ob1
+1w4l
+6e2m
+5wh6
+3kf4
+4lnp
+3el5
+1w84
+4gm8
+3o6t
+1q83
+6fut
+2ycm
+3cqw
+5ot3
+5dp8
+1g53
+5va9
+4wx7
+5kww
+6bu1
+1q6t
+1uto
+2bks
+4fmq
+5opr
+3ies
+6hti
+5u66
+4qz6
+6b1o
+4gy5
+5mrb
+2xml
+3skk
+1c2t
+4in9
+4mss
+4kju
+1hti
+5l6o
+4wkb
+4dus
+2am1
+3t5i
+1dkd
+2onc
+6hvh
+1ypg
+6cvf
+6bkw
+2jt5
+3fdt
+4wh7
+5i3m
+1azx
+3fck
+5u6c
+4ded
+4k0u
+5xmu
+3wk9
+4kz0
+5a7c
+6g9j
+3iw6
+4ddk
+1icj
+6eyt
+4z2o
+4e3h
+4c70
+2y57
+3dzt
+5e0g
+4nrb
+4qr3
+5ltn
+3omg
+6bod
+4jpa
+3fzs
+6f5l
+5edd
+2oyl
+2bmc
+2xqq
+3kqy
+1oiy
+5y13
+4d1s
+3ked
+5djp
+3bxe
+4bo0
+3oy3
+4o2a
+5em6
+5tfx
+4d9p
+3tku
+4p1r
+5nwz
+4bs4
+3oys
+5lgs
+4nyi
+3l58
+2vxn
+1y98
+3ck7
+1nh0
+4an0
+1mcz
+2h9n
+3h0s
+2ycq
+2x4s
+4acc
+5ows
+3rxa
+4dhm
+5lhi
+5l8c
+3rxi
+3r00
+5oq5
+1jev
+4ztn
+4yb7
+4a7j
+4os7
+3e3c
+4acu
+5hvt
+3lf0
+4jfj
+4p44
+5dxh
+1d4k
+5ewk
+1gx8
+1x6u
+3g2z
+1i7c
+5opb
+4y85
+6fii
+6c7q
+4jkt
+4lbl
+1w76
+5dgz
+3run
+4qoc
+456c
+5twz
+4a4h
+6gl9
+6h7z
+4fxz
+2hu6
+2ohp
+4ayt
+4fgx
+1x8b
+2y7x
+4yb6
+5i8c
+2aoi
+3kgt
+3g2h
+3gwx
+5lyr
+3nk8
+1mrx
+1pf8
+6dj1
+4jss
+3tu9
+5cqt
+1yds
+3hy9
+5y5u
+6dko
+3s0j
+5hm0
+3s3o
+5om7
+2j9n
+4bfd
+5wzw
+1d3v
+4pvv
+4i0s
+4pmm
+6ftp
+4hct
+1c5t
+5i3v
+5urk
+5q0f
+5xv7
+4bj9
+1css
+4azc
+1hqf
+3kej
+2p8n
+5tg1
+3cvk
+6awp
+1e3g
+1t4e
+2nns
+5al5
+3cr5
+6equ
+5fl5
+2yex
+5lsh
+4fgz
+3mv0
+3l5d
+1q6m
+5khh
+5c85
+2f3e
+1npw
+5w1e
+4j0a
+4mvn
+5ioz
+4yve
+5tbm
+2fx7
+4oex
+4pjt
+1nnu
+3c4h
+3vp1
+1bky
+5ajc
+1jcx
+8hvp
+4wvs
+2vcx
+5hkh
+2llo
+2jkm
+1q6p
+5mf6
+5afj
+4l50
+1o3e
+6evm
+1pyw
+3l3a
+4mnx
+4fll
+6hx5
+3ivc
+6mx8
+5wf5
+5a1f
+4c4g
+1w5w
+2w26
+4an9
+5n8j
+3aqt
+4dho
+4l7r
+4aro
+1stp
+6g07
+4hze
+2cbj
+5wbr
+5afx
+3bh3
+4ca8
+6aam
+1r78
+6aol
+1gcz
+3bjc
+1o2x
+1b40
+4u45
+5nat
+4qiy
+3v30
+3kyq
+3unn
+5elf
+1xoe
+2oqi
+5mpz
+4hxz
+4rfr
+3vp3
+6upj
+5xyy
+4wci
+2fwz
+6buu
+3ocz
+3v3v
+2yq6
+4m8e
+4r6e
+5ihh
+5ngs
+2v77
+3nzs
+5tyk
+4l7g
+3dne
+1m5f
+2xk8
+4urx
+3ft5
+4br3
+1bgq
+3fa3
+2y07
+4lpb
+5k0m
+4tlr
+1ocq
+4j5c
+4awm
+1ado
+2n3k
+4mf1
+4oho
+2xv1
+5hh6
+5kmh
+4lkt
+3n2v
+3o7u
+2ieo
+1wbo
+5i25
+4mhz
+5mmp
+3qs8
+2ykj
+5igm
+4zji
+3zxz
+4b71
+1fpp
+2y76
+6eya
+5lch
+5gp7
+3mmr
+4qc1
+3sbi
+4bsq
+5yhe
+5hj9
+4qw1
+1px4
+1q5l
+4i9z
+1pbk
+1o5b
+1ryf
+3oeu
+3nx7
+4izy
+5lvf
+3hll
+4jbp
+5icv
+2yln
+5om3
+1e2l
+2oyk
+2c0o
+5w5v
+1b2i
+3w8o
+4x60
+6h2z
+2a3i
+5jnl
+6m9d
+4twp
+5gmu
+5wg4
+2mkr
+5e89
+2j78
+3elc
+3eb1
+5wlv
+3udk
+4p0v
+5ahu
+5two
+2jf4
+1mm6
+3c4c
+5a2j
+3ti4
+3pd9
+4rx8
+4yhz
+2vr0
+6mim
+2k00
+3fr2
+5q0o
+5dgw
+3tiy
+4jxs
+5xst
+5tpx
+4j1h
+5xva
+1o49
+2mlm
+4alu
+1tv6
+4g2f
+2xp4
+1lhg
+5aoi
+2bz8
+4kbc
+1h22
+5tho
+1wun
+1d8m
+3m17
+1slg
+3pcc
+1oh4
+2vio
+3v6r
+4n7h
+3fv8
+4f08
+2p33
+5a5d
+3n1v
+4j4o
+5o1d
+5ewm
+3ds4
+4b3b
+1sqi
+2nmx
+1gai
+1fax
+4deg
+3suu
+4rcd
+6f6s
+1juf
+4zx3
+1m48
+5ttf
+4man
+5ljt
+1y0l
+6hsk
+1gyy
+3bi6
+3fue
+1c87
+4byj
+4r18
+6bnh
+3qpn
+4r4i
+4e5h
+4ojq
+5aqt
+3fu3
+3oap
+3d1x
+4j48
+5uig
+5vcz
+5tzd
+4hva
+3mvl
+6b1c
+4e96
+4d88
+4ufi
+4uvu
+3met
+5aeh
+6dh5
+4mrh
+5fqv
+5t8q
+6f7c
+5l17
+3zim
+3k83
+2p3a
+4wsj
+4cd1
+2xk7
+5w6u
+4zs3
+4u58
+2qn1
+6f5m
+2c8y
+3vw6
+1o42
+5uor
+2ycs
+2i4z
+4q1d
+5m0s
+3qup
+1tr7
+5xpn
+2aa6
+5uv2
+3pgl
+4mk7
+5uir
+1p57
+4ezx
+3shj
+5csw
+3ibn
+4ag8
+6ceh
+2p98
+4yht
+3k0k
+5neb
+4ysi
+1bo5
+1a28
+3wha
+4nq6
+1czc
+1ckb
+5j6d
+3hr1
+4acm
+6hrq
+1py5
+3km4
+4c66
+1sr7
+1rnm
+3krx
+3d25
+5d6e
+5w4v
+1xbc
+2x8e
+4cmu
+4f09
+5afv
+6bkx
+3lqj
+2hug
+5dnu
+2woa
+5xvw
+3fi3
+4yho
+1h1s
+1w3j
+5y59
+1sdv
+5tmn
+5myk
+5f29
+1efy
+4ayy
+4ykk
+3m6p
+1utl
+3zls
+4ff8
+5f6d
+4k8s
+2mpa
+2gfs
+1c86
+5e5g
+3zvy
+2f3r
+4e5j
+3l5r
+5l01
+2vj8
+4b9w
+4i7m
+4qwu
+3rpr
+4i7k
+2c1n
+4ret
+1owk
+4mxc
+2ohs
+1akw
+5tyh
+5huw
+5w14
+5lb7
+1f8e
+4ciy
+5etm
+2pwr
+5o1f
+4r1e
+4ncg
+4abf
+1my4
+4jib
+4nxr
+2jbo
+1njd
+1pdq
+1qj7
+5fi6
+2vtp
+5dro
+1o3b
+4jln
+2yis
+1rt2
+4feq
+3gjs
+5eef
+2zx8
+3ujd
+1fkf
+5eto
+1c5p
+5tys
+6ayo
+4eh4
+3djv
+3mhm
+2pow
+1w0y
+4dbm
+2ywp
+5uhi
+1mj7
+5am3
+1pxp
+4uff
+2bq6
+1n1g
+5nxq
+966c
+3vws
+2of2
+2ydf
+4nbk
+1l0a
+4u43
+5trh
+1rpa
+5etv
+1h61
+2vcb
+3d1v
+1xq0
+1b3h
+3svj
+5fun
+4bw1
+1kti
+4jof
+4izm
+4phu
+3jy9
+4urw
+5ael
+4irx
+3avn
+5mm9
+5twl
+4w9l
+2ad5
+4b4g
+4a22
+5vfc
+3l3q
+6fn9
+4m14
+2gl0
+4c35
+3iob
+6apr
+6g3v
+4cik
+2xpb
+3loo
+3drr
+3e4a
+5thi
+5mgm
+3mrv
+2c4v
+4i8n
+5obr
+2qyl
+6ee4
+2wzx
+5j8z
+2itz
+5t2i
+4mdn
+5nw0
+5l9o
+1c5n
+3bel
+1nt1
+4g69
+5ovv
+1i9n
+1hlk
+5hh4
+2hhn
+5v13
+2ww0
+2rcx
+5zms
+2ncz
+1a86
+4uuq
+5g2n
+1o5g
+2gfj
+5d3j
+3dpe
+5ldp
+3veu
+2vaq
+4ui5
+1hp5
+5g1a
+3f39
+4owm
+1dl7
+3ce3
+2qmg
+4poh
+5azg
+6by8
+5l7f
+4awo
+6fh7
+4ifi
+6b7e
+5i3x
+5lws
+5sve
+3kn0
+3q2h
+5eoc
+6b7h
+5ekn
+3rqw
+4erk
+5fso
+5e80
+2bpx
+1yw8
+3sc1
+5o4t
+5c2o
+4u0f
+2w17
+5nxy
+1oy7
+6dvm
+3k00
+1py2
+4zx0
+4nkt
+3s1h
+2ym5
+6cis
+3hdm
+2yj2
+4ef4
+3kvw
+5cu2
+6c1s
+4j6i
+6bij
+5xgh
+5u2c
+3m3e
+1yid
+1b6k
+4gd6
+5lp1
+4ww6
+1zhy
+3eky
+2q38
+4jym
+2ymt
+3gur
+3suv
+1njs
+4na8
+4msa
+5nk8
+1o4k
+3l6h
+1i5h
+3t6b
+5e1s
+6gzd
+2eum
+1z71
+2yc5
+6hai
+2a14
+4aa4
+4ask
+4ea1
+3pxz
+4n4s
+5wkm
+4jg0
+3sie
+3pkc
+1w1g
+5y8c
+2c6m
+5ou3
+6d50
+5v3y
+5lzj
+5tya
+4c9x
+3eyu
+4xtm
+3agl
+2brh
+4mbf
+5d4a
+2zz6
+2gbi
+1t5f
+5jiy
+1nw7
+3hl7
+6h7f
+1zd4
+3suw
+5fdc
+3cct
+4pm0
+2y1x
+5n9n
+4gj2
+5d6f
+4joj
+5klz
+4jju
+2gg7
+5hn9
+4tjy
+4r91
+6ffn
+5usf
+2clk
+5n70
+2ohu
+3eka
+5eq1
+4qlq
+6eis
+4acf
+5t35
+4orx
+2hmw
+3fz1
+3qx5
+4zhm
+3rxo
+1atl
+5t1k
+3np7
+3zt4
+1bdl
+4u6y
+2jbk
+5l13
+5mob
+2igy
+5jlz
+4flj
+2zu4
+2w70
+5d6y
+4kip
+2uw0
+4msg
+6f8x
+1r2b
+4qvm
+2ydo
+1w4q
+5okt
+5op4
+3al3
+2rvn
+3dp1
+4luz
+3o87
+1w2h
+1fiv
+1uh1
+6c8p
+4g8m
+4hai
+2evc
+2q2z
+4yyt
+4od0
+5k9w
+3fyj
+5vh0
+5ale
+3th8
+4yhp
+4pvo
+4whs
+1c4v
+5hed
+5qbv
+2oei
+4y2s
+1z6q
+4px6
+2knh
+5kns
+1m4h
+2g96
+3cm7
+4gcj
+1qft
+3ztd
+4yzu
+1vwl
+4kiq
+6ax1
+3zyh
+2c1q
+6mx3
+5o9r
+1q8t
+4ue1
+2y7z
+3mdz
+4gtr
+6fh6
+4qvq
+3tti
+5ap6
+4pkt
+3tdj
+3qnd
+4lvt
+1xo2
+1pcg
+5fiv
+1g3e
+6gx3
+5dhj
+5ixt
+1xhm
+4po7
+1a5g
+1haa
+6egs
+3iw8
+5qag
+5o1b
+5brz
+3mpt
+3ral
+1j14
+5wdj
+4io3
+2irz
+3f35
+5jan
+6mrg
+2oi3
+1onp
+4bjx
+2i0v
+4hmk
+5mat
+4h2m
+4agl
+4ayx
+5iwg
+2xez
+5em7
+5kdf
+3vv6
+4y59
+3ti6
+5gn6
+4u7v
+3rz7
+5wxo
+1o2w
+4r0i
+1a30
+5nz2
+3rf4
+6ee6
+4oiv
+1a9u
+3iop
+5x4p
+3gl6
+1hqg
+2oji
+4ps3
+2w05
+1tnh
+3djq
+2c9b
+2flb
+5itd
+5k00
+1t37
+3g0b
+6hdo
+2rfh
+1b5g
+4bdi
+4ikn
+3inf
+1rww
+4g68
+4djw
+5yum
+4hxl
+3dbd
+4hw7
+5mys
+3p7i
+2j7h
+2vvt
+3r2y
+5ng9
+1y2g
+2vwm
+3if7
+2hl4
+3ffp
+4qwg
+3g6z
+5dhf
+4y79
+3m57
+4oyi
+3s0b
+1yvx
+3v7c
+4md6
+4g9c
+2w6m
+5tt7
+3pkb
+3upi
+2va6
+2weh
+4bao
+5gid
+2w1g
+3mxc
+2fw6
+3diw
+1ind
+3i3d
+5k0i
+4npv
+2wf5
+2q7q
+4wyz
+5xaf
+1gj4
+5hf1
+2aw1
+3dbu
+3o9i
+4l34
+5j0d
+5afl
+2w3i
+5ew3
+5e0m
+5jfu
+4fm7
+1o4h
+4w9p
+6eyb
+4onf
+4zhl
+3mj2
+3k22
+2xxy
+5m5q
+4aci
+5kab
+4dpt
+4rsk
+1rsi
+5c42
+3tb6
+6bmi
+4myd
+2i40
+5nhj
+5moc
+1tom
+3ds6
+1lt6
+3eft
+2w8g
+4d1y
+4yrc
+1tu6
+5hmh
+5nw2
+1k9r
+3mxf
+2q1j
+1u0h
+4ye3
+4k60
+5zop
+6htg
+2fx9
+4q83
+4y8y
+4gr0
+3uqf
+3vha
+2yaz
+1k1m
+4i2w
+5ovx
+3db6
+1v2p
+5njx
+1wtg
+3uvn
+3h89
+2q7y
+4o2f
+3fv1
+4eu3
+3pwm
+4tmp
+5ulp
+6di1
+4iks
+4dhs
+5hg8
+6b1x
+5nw8
+3tjd
+5yve
+4uwg
+1jd5
+5km5
+2w4x
+3umx
+2ony
+1bm2
+4pku
+2pj9
+2m41
+4z8a
+2yjq
+6mwe
+1prl
+5y1y
+1c5y
+5ak6
+1ggn
+2usn
+2pv2
+1w25
+1c4u
+4ba3
+6c2x
+4ijl
+3gxy
+3f3u
+4io2
+4drm
+3cjo
+4dst
+5c87
+5ni8
+2ll7
+2pem
+2x7d
+3zc6
+1m2q
+4gk2
+3lqi
+3c1x
+2az9
+1fv0
+1gz3
+1gfy
+5mte
+2nxm
+4yml
+4yog
+2g6q
+1t79
+3hdk
+3uij
+5vl2
+4ufk
+4aph
+6d1u
+4rn2
+1zxv
+2p4i
+1w80
+3bl7
+2vh0
+3kqa
+5vdu
+4b2d
+5jgi
+4u0i
+3g9n
+5var
+4lnf
+3e9h
+2fzz
+3cfv
+1ws4
+5h21
+2p7z
+4j8s
+5fdr
+5z7b
+6cj1
+5tts
+6bmx
+1g49
+1mqd
+5ost
+2qhd
+6e4a
+3fur
+3v1r
+4wwn
+4hni
+6e83
+1gqs
+3eko
+1dmp
+2wnj
+4i6q
+4d8s
+2jb5
+4p58
+4usj
+6eq6
+3o0j
+5apj
+4ty1
+5avf
+2xef
+1way
+1a7c
+5f2r
+5k1f
+2c9d
+3a9i
+3deh
+3q3k
+4xtw
+6f0y
+2r3l
+4mra
+5qa4
+4zpg
+4ijp
+3wyx
+1s5z
+1y2h
+3qi1
+1c83
+5dts
+5e1d
+5j9k
+5o4f
+2wxi
+5f04
+2qmj
+3kvx
+4l7j
+1d5r
+2vyt
+4cfv
+1qcp
+4hlm
+5d48
+3snl
+5uwn
+4yll
+2brm
+4dzw
+4q7v
+5teh
+1c4y
+5mro
+4ymg
+3b1u
+2z9g
+2rt5
+3gxz
+6e0q
+6f7b
+4qz5
+5dh5
+3v3q
+4do5
+3k0h
+5dt2
+3l54
+3ebh
+2hd6
+5hbe
+4wri
+2qi7
+1jao
+5ajp
+6ej3
+3gjq
+6f8w
+5y20
+1rwx
+4ibj
+5op2
+3c6u
+2w0s
+1c7e
+5yyb
+3ni5
+3t8w
+5ntp
+4dfu
+1o4e
+5tqu
+2xyd
+4pl0
+5tl9
+2vpo
+4ov5
+4qps
+5wdc
+4awp
+4a9s
+3atk
+4nja
+4ms0
+3fzr
+1i9m
+4b5s
+2y71
+3f33
+5q0h
+5x9p
+4p4j
+5ajo
+3c56
+4pnl
+4fmu
+3uli
+5nlk
+4cs9
+6hkz
+2vo5
+2fts
+3sfc
+3pke
+4dk5
+1lor
+2jdn
+3jya
+6bqg
+3p9t
+3qkk
+6fnj
+6fmc
+5c0k
+1mmq
+4fck
+5npe
+4j59
+4eol
+5m57
+4efk
+3bt9
+1oiq
+5dwr
+3wyk
+4qf9
+4c5w
+5tuq
+3oqk
+4xj0
+5mwg
+2vwu
+6dpy
+4ara
+3wi6
+5owf
+3nw5
+4fhh
+3sna
+1uye
+2vcg
+5j7j
+4gtt
+5jc1
+2ph6
+4zpe
+2hz0
+5l4i
+1y1m
+5vo1
+4y2v
+1ecq
+6eeo
+5jrs
+4l23
+6f34
+4r4q
+4quo
+6dq6
+4e3k
+3zov
+1cet
+5e2k
+5jyy
+2jld
+5jna
+4qls
+4ksq
+4py4
+6biy
+2hqu
+4yk5
+4i12
+4km0
+5ttu
+2za3
+4o5g
+2y1g
+4cff
+4ztr
+4jps
+4uin
+2q93
+2x7t
+2q6c
+6ayr
+1pwy
+5lwe
+5cxi
+4n9b
+1fzo
+5ukm
+1ork
+4mk0
+1jbd
+1k2i
+4ifh
+4yh4
+3fee
+1flm
+5tqe
+1jj9
+4li5
+4jc1
+5i8p
+1lah
+1apw
+3ppm
+4kn7
+1m5w
+3vbx
+4ie2
+3sur
+1nz7
+4trw
+4qmu
+4qnu
+4os6
+2bge
+5xvg
+4o6w
+1nfu
+2zjf
+1fpi
+5id1
+3ccz
+2d3u
+5tyj
+1fls
+1ggd
+5ofx
+4o12
+4wf4
+3pck
+4ayv
+5jt9
+5nb7
+4b84
+5ggk
+6h5x
+1r5w
+4zw6
+5d6j
+5ep7
+4duh
+4ps0
+4drk
+1los
+3s6t
+3be9
+4ipi
+3wf7
+4pow
+4y46
+5izj
+1d4j
+4qg7
+2avm
+1g85
+1r1i
+3g1v
+3l7c
+5wyr
+4oti
+4a4o
+1fvv
+3uok
+5n3v
+4tim
+3tv8
+4a4g
+6c0s
+4fak
+1w12
+4jn2
+4oyp
+4zt7
+5emj
+2rc8
+2ypo
+3kmc
+4hwp
+3q2j
+4k5l
+3hwn
+5i83
+1j4r
+4fyh
+1kfy
+5g43
+4bnv
+5fpu
+1o48
+6ei4
+6ekd
+2k3w
+3ksq
+5wkl
+4hzz
+2f8g
+2uw3
+6mi6
+4u91
+3f7u
+3zk6
+5n2f
+2qx0
+3mrt
+2f0z
+3qvv
+1l7x
+2oc1
+3gbe
+5he3
+1pun
+5m4i
+3pp7
+5o91
+4mjo
+4fns
+4xnv
+3tay
+3ejs
+4c1w
+3njq
+6guc
+4dv8
+4el9
+1iep
+1ta2
+3h0e
+1bsk
+4pri
+3pcb
+4jg8
+3lbj
+1i8h
+4a4x
+1lhd
+2wa3
+5tcy
+3wth
+4baq
+1ylv
+5bve
+4g34
+5u28
+1qbv
+5z5f
+4ot6
+2g97
+5n16
+1m5e
+3efk
+4h7q
+2v5a
+5vs6
+2cne
+1pxn
+1jqy
+5j6s
+4abv
+6d6t
+1t4s
+3zsx
+1p01
+2f4j
+2gh7
+3f78
+5i38
+3h0w
+5d3x
+3zst
+2lbm
+6fil
+2vk2
+6ew7
+2iu0
+3t0l
+5ai9
+5wbm
+4ark
+3hlo
+1np0
+2w6c
+6dk1
+5xih
+5hk2
+1qhc
+3v5l
+1vr1
+4no8
+5hmy
+2jql
+1a46
+3kf7
+4lv4
+5hbh
+3wf9
+4ieh
+2vr3
+5dyw
+4xuz
+1mmr
+5ar0
+4h4b
+5knj
+5elv
+2g79
+4w52
+2rnx
+4rgd
+2az8
+3c1k
+5ick
+4lv3
+4mwb
+5a83
+1prm
+4gg5
+1rlq
+1rq2
+5fbn
+4e49
+1f47
+4yfi
+2v10
+3pa3
+4j1f
+4jvi
+4gmy
+2v3d
+4cft
+1epp
+2hoc
+4r5g
+4k0y
+5zz2
+1d3p
+4ng9
+6fav
+5hor
+3qci
+4mbp
+4msu
+5hg1
+1pyn
+1hmt
+2lpr
+4yha
+5hoa
+1sln
+4h3c
+4k64
+2w0d
+4lmu
+3tf6
+4z2p
+2hny
+5ctc
+6cck
+5vd1
+3qcf
+5ia0
+3l7g
+1eld
+1xh9
+1h3c
+4urm
+4j7i
+4ly9
+1nu1
+5sz5
+6cdl
+1uvu
+2wl4
+4eo4
+5lj2
+2qrl
+3kc1
+5nqe
+6gon
+2xs8
+5gmj
+5eyd
+6c42
+4wks
+3ka2
+4p6e
+4nnn
+3zvw
+4q6r
+5v8v
+2jdv
+4lxm
+5qcm
+5e8f
+5y80
+3ud5
+4n70
+1jwt
+4cgi
+2uw8
+4ajw
+4ivb
+4m6u
+2xu1
+6bxy
+4ud7
+3k5f
+2arm
+5jin
+3fmq
+2nt7
+1p4r
+3hxf
+4zwx
+4xx3
+4yee
+6aak
+2xae
+5t31
+3bhb
+5llh
+5nge
+4b9h
+5oxl
+5ipa
+1q3w
+6d56
+6df1
+3ole
+2v16
+3g2s
+4led
+2uz6
+4pnu
+3nrz
+1f7b
+1j4k
+5ku9
+1a4g
+4uia
+5kv8
+2rke
+4e8z
+5ykp
+4z1n
+5sz1
+4oak
+3twu
+2r3p
+5cf6
+1ym4
+5l4j
+4exs
+4qpa
+2k1q
+4eny
+4i0r
+2xz5
+5fxq
+3re4
+6cf5
+2cbr
+5ece
+5t2t
+1x8r
+4n3w
+1klu
+5nxx
+5jcb
+5klt
+4usw
+5ek9
+2mov
+6bid
+1mjj
+1tlo
+2hb9
+2f94
+4luv
+5gu4
+3wf8
+5ufo
+4n5t
+5hcx
+5mtw
+1o38
+5z1c
+4w4z
+4v27
+1k21
+3jrx
+5l96
+1esz
+2m3m
+5ivf
+1o3p
+6std
+4auj
+2lyb
+2w87
+5j3l
+2fgu
+3snd
+2w5g
+2wsx
+4qfl
+2vuk
+4a23
+5iu6
+1siv
+4ht2
+2ltz
+1o39
+4p0w
+3p8p
+5ypy
+3zvt
+2c5y
+1ayv
+3erk
+6g4y
+4ucd
+3e16
+4i7p
+5alv
+1osg
+6gw1
+5wxg
+4bvb
+1nc1
+5ugg
+5t1w
+3tvw
+5hkb
+2r3n
+2iog
+1h1r
+5eyc
+1ogx
+3er3
+5q0p
+3rlq
+5itp
+4nuc
+4w9j
+2b8l
+5khj
+4mz4
+3zlv
+3okh
+2epn
+1pwq
+3gsm
+3hzv
+6dpx
+4hvg
+1s17
+6arm
+5jq7
+2v88
+5lay
+5nn4
+4hj2
+3pp0
+1rst
+3qcq
+3rx8
+5ewz
+5mgx
+4c1t
+3cso
+4wq2
+4gxl
+5isz
+2xj2
+4y5h
+3zcw
+1l8g
+5vex
+4tky
+4elh
+4ejn
+4pgh
+2wuf
+2xbp
+6gw4
+3d7g
+5ksu
+2qrp
+4jg6
+4eqf
+3hbo
+5yjy
+2gv7
+4hdf
+1yi3
+5edr
+5nmf
+4w7p
+4fz6
+1g4j
+4a2a
+5kx8
+5a0b
+4qta
+4mho
+4fn5
+1gj8
+3d94
+1kdw
+6euw
+6gpb
+6glb
+2xaq
+3vh9
+1aku
+5j4v
+2f1g
+2vfk
+2r5a
+1g1e
+3buf
+1bzj
+4abg
+3cho
+5t6j
+4jda
+5ywy
+6cdo
+6fo9
+1g74
+3oxi
+5t8p
+3pcu
+1fyr
+1fm9
+5ddc
+5hwu
+1xud
+5ai1
+4e9u
+4uyn
+5k05
+6fgl
+5iha
+4aa2
+1wzy
+2w85
+3s2v
+4yuy
+6dud
+1xdg
+3bgs
+4hiq
+3rqf
+1jmg
+2xeg
+4g93
+6c7b
+3zso
+4xjt
+4rsc
+5q0u
+5iyv
+1qxy
+5n20
+1you
+5c7e
+4x9v
+5tjx
+3rtm
+4b05
+3tg5
+1b52
+1w1v
+5lg3
+2bmg
+5wii
+5wew
+5iua
+2g00
+1nm6
+4wef
+1i7m
+4hrc
+6at0
+1biw
+4uhg
+5hvp
+1ugy
+3ery
+5ll4
+5v6y
+1ru1
+2o4l
+2evo
+5lvn
+2rc9
+2fci
+6evq
+3rah
+3g2v
+5ny1
+1p10
+3o96
+5vsd
+4cxx
+5kcb
+4g2l
+5vqi
+5svl
+5qax
+5h6v
+4aa0
+6g1w
+3ddu
+1pzi
+3sax
+4nmr
+4c1f
+5d26
+1elr
+3gc7
+3c84
+2y1d
+3vd4
+2uzo
+3v8s
+5mqv
+4zly
+4p0a
+5dlv
+3qqa
+4g3g
+3h9f
+4wr8
+5win
+4hzm
+3rqe
+4j1c
+4kii
+5mw6
+5mwj
+1xap
+4zv2
+1ol1
+1a69
+4btt
+5t4h
+2oi2
+6bbs
+6aaj
+2y4a
+5i80
+1hte
+6cfd
+1o4l
+5kbe
+3saz
+3kqr
+2b07
+3iue
+3bxf
+1is0
+5a4t
+5ktu
+4agd
+3nf8
+2nd0
+5yyz
+2ay4
+6fhq
+3f3t
+1zhl
+3vbv
+1g9s
+5aut
+4u71
+1q66
+5vb9
+3qtr
+4nw2
+3qin
+5mzg
+1d9i
+5h08
+3rv7
+3rv8
+6ckx
+2itk
+4ij1
+1pxo
+5nt0
+5x62
+4oc4
+5qao
+3le9
+5dtr
+4a6s
+3bls
+1em6
+1pus
+3asx
+4lng
+3mvj
+4q3q
+3s9t
+4gfd
+2r8q
+5wqc
+3ri1
+5ih8
+6g6w
+3cjf
+6duh
+3shv
+4pda
+3cpb
+1bp0
+4lmn
+1zm6
+2kup
+3ij0
+6cz3
+3v3b
+3vru
+3arg
+2o64
+6ayi
+4qab
+4xu2
+5wzu
+4lh2
+1w6y
+5eis
+1czr
+5jzy
+4ejf
+4ufe
+5wqd
+5f0f
+4aw5
+3iqg
+4p90
+2ank
+4n1u
+6eku
+2h23
+1j5i
+2y7w
+5alj
+5jfr
+5eak
+2gfk
+3be2
+5hln
+5uc1
+3ax5
+4f3k
+5qb0
+4llj
+5ldk
+4eyr
+5v3x
+4fys
+4crl
+2zxd
+2etk
+3ddp
+4q0a
+3el0
+2ica
+4yv8
+4jin
+5enc
+1pb9
+5jur
+5n1z
+5ou2
+4y8d
+4gby
+5acw
+4r76
+5hrv
+2jt6
+4hhz
+3ocb
+4qxj
+5er5
+3hu2
+1v2l
+3dz5
+4j8b
+5vqs
+5khd
+5fho
+2f71
+3kdt
+5gmm
+3sd5
+5fck
+3q8h
+4h36
+6bm6
+4qmz
+4ks5
+4dew
+1nf8
+5ap0
+3s73
+6chp
+4r3b
+2oag
+2w06
+5j74
+2a2g
+4ju7
+3i81
+5aol
+1af2
+5hda
+3nu3
+6das
+1swp
+5njz
+1b74
+1sts
+2co0
+5od1
+3pab
+5u06
+2gnh
+3uig
+2uw5
+2vrx
+1ele
+3hp9
+3fqa
+4mh7
+2xkc
+2w3l
+5w73
+5hip
+4xv2
+1fch
+2j4k
+1o33
+2v57
+1kqb
+1bhx
+4a4w
+5bmm
+3dcw
+4i9o
+4ydn
+5xii
+5duf
+5j1r
+4no1
+1gvu
+1ycm
+2l3r
+4wnp
+4yzn
+4p75
+1b1h
+5hwv
+1s4d
+3rlb
+1c5f
+4fqo
+1iau
+5mim
+5y93
+5qa6
+3wk5
+4mcb
+4qms
+6b5m
+6fba
+2q8z
+3r92
+5mo0
+3wvm
+4qh8
+6dgz
+2w66
+2joa
+6gih
+5fqt
+2viw
+4o76
+2j83
+3cm2
+3ion
+4y2u
+3fhb
+2rip
+2q94
+6eiz
+3ieo
+1uyg
+1vsn
+5fnf
+2vgp
+2pk5
+2f6t
+1zfq
+5ypo
+2nnv
+4jvj
+2cej
+3hqw
+2boj
+2h21
+3bzi
+5ut6
+2toh
+5urm
+5ml8
+2qpu
+6f5u
+3t3y
+2zx5
+5alu
+2vxa
+1z4n
+1c8k
+4w53
+5cf8
+4hcz
+1jik
+4b81
+5y86
+5ul6
+5l2n
+3dhk
+5vio
+2piz
+3jyj
+5ly2
+6do4
+1v2s
+4ezo
+1iht
+3hl8
+4q1c
+3sk2
+2xjj
+6cdj
+4w4w
+1bm7
+5lca
+3rxf
+4q7w
+5ceq
+5aen
+3kjf
+2psj
+4avt
+5nxo
+3qn7
+1d6w
+5ory
+3ckr
+6eyz
+4lxa
+2chz
+6eqw
+3nzc
+5umw
+1ilh
+1o4j
+2pks
+6evp
+2wks
+2yc0
+4xy8
+2jg0
+1u33
+5iez
+2zxb
+2qnz
+5wfz
+3ob0
+1g36
+4at3
+3pd3
+5jjr
+6eo8
+4omc
+1k3t
+2vl8
+4y8x
+3l4t
+3s2a
+6f4x
+2qrm
+4u68
+4o3f
+4rpn
+2zns
+4zvi
+5ul1
+3fwv
+1exw
+2vvv
+2aof
+5vfd
+2qcf
+3bl0
+3gk1
+5i2i
+5w8v
+4ptc
+3piy
+1nlj
+4ddy
+2vwo
+3kdu
+6bt0
+5l99
+3rtn
+2yk1
+6f28
+1zh7
+5uiu
+4eh7
+5epp
+1o6h
+1b05
+5t5g
+6b1k
+4k75
+5hva
+3ips
+5obg
+5a3o
+2q89
+5owc
+5ggo
+4xk9
+3cf9
+1czl
+2ewy
+3a1e
+1xlz
+5k0b
+2nta
+4gvc
+1axr
+5fou
+1kkq
+4u7o
+5i4o
+4cwr
+5fa7
+1i5d
+5hdu
+2zg3
+2pmk
+3mrx
+3s1g
+6mvu
+1wkm
+6dvl
+5fe9
+5odx
+3sbh
+2jiw
+2r43
+4ibb
+4o05
+2v13
+4io8
+4z2h
+2vmc
+5ggn
+2afx
+1g3c
+2cn0
+3oki
+4xty
+1xgi
+3mmf
+5v1d
+2uxz
+4wwo
+4kpz
+4xbb
+5up0
+2gzl
+4r5x
+3iet
+1oq5
+6afd
+1b55
+6chl
+1xp0
+2q5k
+5ti7
+5uqx
+6bm5
+5w94
+1k1o
+4lm3
+1ek1
+4ayr
+2xxn
+5lsc
+1o0m
+4ui7
+1ppl
+6beh
+3ud7
+4zxx
+6gzh
+1uu9
+5jgq
+2q95
+5l2y
+3v3l
+2gmv
+5bvo
+3snb
+5q13
+5upf
+5tq3
+3gw5
+4wwp
+5wbo
+2j7g
+3rmf
+1fhd
+2w2u
+4flk
+2r0y
+2wbb
+3da6
+3l4z
+1v2w
+2azb
+2bz5
+2oxn
+3f8w
+1w7g
+3moh
+5m3a
+5aqq
+3tz0
+3zyu
+2pzy
+5usy
+4mx0
+4cpq
+4aq6
+5eld
+5gmv
+3qbn
+6rsa
+3u5j
+5h1t
+1ms0
+1ctr
+5lub
+4h5d
+4n99
+4cc2
+4hzw
+5lbq
+3ati
+3pd2
+1f74
+1qyg
+5dhg
+4mwx
+5ohy
+4ob2
+2b2v
+3c0z
+2gtv
+4r75
+5w2s
+3wsy
+1rs2
+2r6w
+4agn
+4gk4
+4qq5
+6bfa
+1n7t
+1ocn
+1gnj
+4u79
+3hmm
+2vl1
+1ec1
+5f4u
+2zsc
+4u5j
+4jfk
+5kew
+4btw
+2cbu
+1vkj
+4jnc
+1nli
+6bw3
+3fyk
+3cl2
+6f3i
+1w3l
+3gv9
+4h4e
+2ylc
+2fgi
+2ha5
+5sz9
+5gjg
+2q1q
+4aza
+5yia
+2rny
+1o30
+3m1k
+6dj5
+1pfu
+4awf
+5oqu
+4k42
+2wcg
+3oj8
+4ngs
+4lq9
+1jzs
+6g9h
+6g9u
+3axm
+4qp7
+4mma
+1g98
+5wi0
+5iep
+2opy
+6eq1
+2pri
+4y2j
+4x1s
+1bux
+4cjn
+2vnn
+5d2a
+6cms
+3g72
+4wz4
+2las
+4his
+5ohi
+1apv
+2v86
+5nap
+3wdc
+3d9m
+4kzl
+5ixf
+1fd7
+5f25
+5js3
+3d7h
+4ceb
+3dst
+4yym
+4x7n
+4cr5
+1gar
+4z2b
+3iph
+5t37
+3zqe
+6m9t
+3m54
+3szm
+2oxx
+4mlx
+3uxg
+3qri
+4l0v
+1a07
+5qay
+4cwo
+3r0y
+1wbs
+8gpb
+6bgz
+1ecv
+1ha2
+2kfx
+2be2
+6cjw
+2p94
+5tq7
+5fsc
+4btb
+5yov
+6eox
+5am2
+2pvm
+3gn7
+2ql7
+5j58
+1msm
+2bjm
+3rt8
+3ds3
+1m13
+4nxo
+2vj9
+3fqe
+2m0v
+3so9
+4r5b
+1v2j
+4bks
+4tkj
+1jld
+2ojf
+1z5m
+3m1s
+4lpf
+4zls
+4af3
+5eue
+2jh6
+2kvm
+5etl
+5hn0
+3f6g
+3fdm
+2jj3
+2zjw
+3sfh
+1ong
+1noi
+3cen
+2p4j
+4rvr
+4az5
+2wqp
+6fnt
+3ejt
+5c2h
+3sow
+3ml2
+1ld7
+1yz3
+3m2u
+2wcx
+1ec2
+3iqj
+3ccn
+3r1v
+5foo
+3cn0
+2z4w
+4ks1
+3rtf
+4bfr
+2pvk
+3ppr
+3r93
+4x9w
+2z7i
+4hdb
+4bti
+1eef
+6bln
+4wyy
+4bw3
+3mks
+3iit
+2axi
+4urz
+3q2a
+2w12
+3aho
+6e06
+5dy7
+1gi1
+3az8
+4ew2
+3tgs
+3zc5
+2ggx
+4ih3
+5u00
+4qkx
+4oef
+6gjn
+1ewj
+2xii
+5al2
+1o4p
+2xck
+3dcs
+2g0h
+5adr
+1kv1
+6h33
+2wl5
+6h3k
+1qq9
+5vdr
+5x4q
+2xk3
+3p4f
+5prc
+4gxs
+1qm5
+4aw8
+2eep
+5l15
+3tk2
+5heb
+5aia
+4nh9
+5nki
+3eos
+4m8y
+5u49
+5yh8
+5ylu
+6f8r
+3nxq
+2x3t
+5exl
+4x0z
+4rj6
+5ly1
+3zo4
+2nw4
+3upx
+5btv
+4n3r
+5ek0
+3otx
+4r4c
+1y3n
+5cal
+4dld
+5z89
+5sys
+1ybo
+2cmo
+1k3q
+2ima
+3wv2
+3caj
+3vv7
+1xug
+4l2k
+3aya
+3gs6
+5alg
+4b4n
+3zyr
+4hxm
+2ov4
+3n9s
+4n07
+2jnp
+3zxr
+3v7x
+1aid
+2zis
+4u8w
+6e2o
+5yqn
+3oc0
+4b1d
+3g9e
+4qw7
+1c1r
+1zsf
+2nm1
+3cs7
+2x24
+5d7c
+1bji
+4yl3
+4k78
+2f9u
+4kab
+3rl7
+4n8r
+5etu
+5drs
+6es0
+2g8n
+5ol3
+2i6a
+5gvk
+2qrk
+3sx4
+4q4r
+5nq5
+3ctt
+5kxc
+1tyr
+4j0z
+5z4h
+2uzn
+4oon
+2ohr
+4re4
+3prs
+1f8c
+4jvb
+3eig
+1dbk
+4p74
+4zg9
+3wtn
+3l7a
+4cdr
+4fli
+2am9
+2jkp
+2bts
+1u9q
+5ea6
+6db3
+4b6o
+3nnw
+3p5k
+2c4f
+1rbp
+2xuc
+4hvd
+3ijz
+5n4t
+4jr5
+6e9w
+5f63
+4cjq
+1b46
+4j4n
+4kcx
+2a2x
+4h3i
+4k4f
+6bky
+2bak
+5csd
+4mgc
+4mwv
+1mzs
+4ym2
+4ztm
+5epn
+5v7w
+5hes
+3n49
+2zlg
+2kbr
+5fng
+2fgv
+1f4e
+2gd8
+3g6h
+5u0f
+4j44
+5v35
+1c9d
+6g6y
+4azi
+5kpk
+4ui8
+1no9
+4p3h
+3gcs
+5ngu
+6bl1
+1q6j
+2zda
+3b27
+1o2j
+3u2k
+6mil
+1n7i
+2wec
+1h6h
+3owl
+3g34
+1ec9
+4crf
+3hha
+3qiy
+4dh6
+4pid
+2y55
+5hhx
+5etf
+4xar
+2pvu
+5ito
+1li3
+3dya
+1xsc
+3ti3
+1tze
+6fmf
+1di9
+2yke
+5f4p
+5sz7
+1m2p
+1uu8
+4y64
+3rt6
+3vs3
+3nfl
+4bdg
+5jv0
+3wd1
+4pge
+5cxz
+5ak4
+4mdt
+2j27
+4lte
+2ayr
+5m6f
+5dv4
+2qbu
+6ciy
+3ekq
+3c7q
+2osc
+4r6w
+2ph9
+3qkm
+3hhu
+4xqb
+5jr6
+2uwl
+5ezz
+1y3y
+1i8i
+3sxf
+2i7c
+4m1j
+6rnt
+3g5y
+3hwx
+2p83
+1mik
+1mrn
+1sm3
+3py1
+5fah
+4pnn
+3tza
+3zmp
+2wxl
+5qaz
+4odf
+1eix
+5b2d
+5kdr
+3hcm
+2x91
+3frg
+6gch
+4qz2
+1hvh
+1g3m
+1tnl
+3s7f
+6cpa
+6g0w
+6ffh
+3kec
+1nj5
+6afr
+5kup
+6ftn
+4o97
+2i5f
+5nzf
+2f89
+5fsn
+3nex
+6b2c
+4zt5
+3gt9
+3sud
+5btx
+4fab
+3fxb
+2x4z
+3l4x
+5svx
+5u98
+1oim
+3lpr
+4pqa
+4pnk
+4ogj
+5k0s
+4oba
+6h7m
+3aas
+4zae
+5u5t
+4n6h
+4dsy
+1ywh
+4w9o
+2qbp
+5d45
+5lhu
+2bro
+1uz4
+1ugx
+6avi
+2aox
+5nb6
+4he9
+2oz5
+5n24
+1h8l
+3pwd
+1it6
+4rvk
+6b27
+2qu5
+4x7j
+5tyo
+2ym3
+5mjn
+3h26
+5ufs
+2wj1
+1rt1
+1kz8
+2zbk
+4krs
+5j87
+3u3u
+3g1d
+5bms
+1u9w
+2z7g
+4xv1
+4ngm
+4j8t
+5d11
+5c11
+3f17
+4alw
+3q3b
+3rwd
+1c70
+3iiw
+1hww
+5kv9
+4wey
+4ok3
+2ctc
+4bcb
+6e8m
+3shy
+4mnw
+5ut0
+3lcd
+5yc3
+2vwz
+2xbx
+1o0o
+2vj1
+6b33
+4hnp
+1w70
+4bh3
+4gng
+2q8m
+4rrr
+4rqv
+1hih
+3u1i
+5ovr
+4ach
+6c0u
+4qiz
+4k18
+1e72
+4ocq
+1b11
+3gds
+3mwu
+5ivs
+3bux
+3qc4
+5ovc
+1pu7
+1y3w
+4v01
+6e7j
+1uu3
+5dh3
+3tu1
+4q3s
+4r5a
+4z16
+3uyr
+5wir
+4exg
+5kby
+4iur
+4zv1
+5xsu
+4gtp
+2o8h
+3wqw
+5th7
+5tv3
+1tfz
+2v3e
+5ntw
+3cyy
+1ny2
+4q6f
+2pjt
+5w7x
+5jap
+3b68
+5tyl
+2a4w
+2v00
+4nks
+1ets
+2fq6
+5hbs
+4r59
+2ama
+2web
+6e4t
+4xm6
+3bkl
+6fty
+5v0n
+6ge7
+2boh
+1ikt
+2r5d
+5alh
+5szc
+1cr6
+4zzy
+2p1c
+1phw
+5w92
+3l9n
+3k8q
+3eml
+3nij
+5amg
+3bv2
+2w73
+1h35
+3tne
+3jyr
+5l9g
+2xsb
+4f70
+2iuz
+1ml1
+3ewj
+4mk9
+2lo6
+2jiu
+4es0
+6dqa
+3kab
+5mxk
+2i0y
+4zy2
+6cqt
+2i47
+2i0d
+2i80
+1nfw
+5uzk
+6hd6
+1y6a
+1vcu
+1e2k
+4wz6
+1nc6
+4zyv
+1wcc
+4og4
+3il6
+1kzk
+4frk
+1nms
+5kax
+1fwu
+5djr
+3a1c
+1hdq
+5ml6
+5l4h
+4rj3
+2x4t
+5nk9
+2xj1
+3mfw
+3gy3
+5nvv
+4rh5
+4jxv
+4zs2
+5ivv
+4axa
+5lli
+3q5h
+4wmu
+2z50
+4lkf
+1rjk
+6g93
+4nni
+6exs
+1nki
+3i7g
+6ei5
+6erv
+1z3c
+3r5n
+1lol
+3qk5
+1rhm
+4jfe
+1xm4
+4gny
+3ti1
+4hhy
+5b5b
+4fjz
+3w2o
+1oxq
+3wk8
+6bl2
+3oe8
+3f7z
+4j3i
+5ogl
+4my6
+6fob
+4o1l
+2uy0
+5dqc
+1g46
+5orl
+1uwu
+185l
+4z07
+2w8w
+3l3l
+3vvy
+3jwq
+1o2z
+5lsy
+3iok
+3g6m
+1sdu
+4xtp
+4qt2
+5fdd
+5ia2
+1ux7
+5jhk
+4ca4
+3qfz
+1qvt
+3t9t
+4j0y
+4x49
+3lpf
+2bpv
+3jzr
+1fcz
+3qps
+4p7m
+4hvb
+3cp9
+5jm4
+2q9y
+3eid
+4o4y
+3cid
+3nc4
+6b1f
+3rth
+3rxe
+1sl3
+4ap0
+4mu7
+1pvn
+4jfm
+1rej
+3ens
+3e9b
+4hby
+6guf
+3iiy
+2z4y
+4to8
+5t1i
+5upj
+3c89
+4dju
+3pcg
+6gu2
+2euk
+3vbw
+3px8
+3g4k
+6dh0
+1ukh
+4rxh
+2ien
+2upj
+5m44
+3gzn
+3eio
+2hk5
+3emh
+1o3f
+5ipj
+6dai
+3su5
+3sh1
+4iic
+5e0h
+4fs4
+4m3q
+4wf2
+2osf
+2pj3
+2r0z
+1jrs
+1i9q
+2zgx
+2jfh
+2ow9
+2aa9
+5vt1
+5l0c
+4ybk
+3zmt
+5kr1
+2r7b
+2lh8
+2wyn
+3ckz
+6lpr
+1usn
+3v2n
+5oq8
+6bhd
+2pym
+4xaq
+3puj
+4ek9
+1eb1
+5o1a
+5k5c
+2rnw
+5qa8
+2zg1
+4gtm
+3uec
+4ncm
+4alg
+5hvs
+1lxh
+1y6r
+5aux
+2qwb
+2h2d
+4qir
+1s89
+1qwf
+2avv
+5knt
+4odn
+3o1e
+3lj7
+3eov
+6cpw
+1qca
+1jii
+3lkh
+3rul
+4g4p
+1qk4
+5nkh
+1w0z
+1jlq
+4ngt
+4meo
+4pmt
+2v2h
+4lm2
+3bu6
+1bzs
+2vvc
+1v0n
+5a7y
+3d67
+4u0c
+4uxj
+4b78
+4bo2
+2jsd
+4aa1
+3ga5
+5hka
+3vtd
+2gm1
+4x6x
+6ma2
+2ces
+4ayq
+1tpw
+2uxi
+3gc4
+5ye7
+2cbv
+1v0p
+2uzb
+5xvq
+3r7b
+3umo
+3b8q
+2y6c
+3bc5
+2bes
+5wa7
+4f6x
+3uvk
+1hgt
+6m8y
+5twg
+4avu
+5vwi
+1ms6
+5xo7
+3dcq
+5km9
+2ygf
+1zsr
+1f40
+5m2q
+1bl7
+4k7n
+4hvs
+4z2i
+5voj
+3ck8
+5d6q
+3g0e
+5y0f
+3dpk
+1bjv
+6bw5
+3dow
+2zif
+1mzc
+2nng
+2xnm
+3f8e
+2ndg
+4lh5
+3e8r
+2e9o
+3uih
+4hlc
+5moo
+4epy
+2i3h
+6fby
+6ew6
+3ama
+4l0t
+1vj9
+3pup
+2xct
+5fh8
+5ivz
+3pxq
+1str
+2xxx
+2pqj
+5mra
+4o28
+1d6n
+1pr5
+1rpf
+4tt2
+3bhx
+2cma
+3dct
+4qt0
+6f1j
+3gvu
+5yy9
+2x81
+2cv3
+6fa4
+5jo0
+3l0e
+3p3j
+1eed
+5aib
+3qbc
+1n7j
+5fpi
+6bnl
+5v8q
+6gxw
+1rw8
+5zw6
+4e3d
+2mji
+6ayh
+3lcv
+1w2k
+3zns
+5mty
+6c3u
+4ks4
+3ds9
+4hld
+3elj
+4j45
+5v41
+2yne
+5nad
+5lvq
+3h0a
+3d9p
+4tu4
+3ip5
+1w22
+4kbk
+5lqq
+5wyx
+3ps1
+5ou1
+5j9x
+2xru
+3t4p
+3uzp
+2o2u
+3dei
+3u93
+4z8m
+3lhs
+2f81
+3fn0
+4iqt
+5laz
+1xot
+2z52
+1zs0
+1oau
+5d47
+3o4k
+6dcg
+1o4q
+1csi
+3tpp
+5nwd
+2wca
+4ie5
+1u1w
+2w92
+3r4n
+5aae
+4gja
+2pj2
+5f2p
+5hjq
+2yme
+3cbs
+5u0y
+5w3i
+5qaw
+5d3h
+3sb0
+5alx
+5g57
+3uhm
+6afj
+5jf5
+4ccd
+5mhq
+5etx
+1clu
+4k6u
+6g28
+4zyz
+1sj0
+4eh3
+2r5b
+4pyn
+5eq0
+1cil
+5le1
+5nyh
+6bdy
+2auz
+5vm0
+2ff1
+3f8f
+2rka
+3dda
+4iu4
+5bs4
+5a0a
+2qn2
+4fyo
+1p2a
+2uze
+1x38
+4ps8
+2g19
+3nzu
+3gi6
+5dtt
+1gaf
+3nii
+2xm8
+4att
+2vex
+4jx7
+6h34
+4tvj
+4lh3
+3pvw
+1ai7
+1kc7
+2rl5
+6c0r
+4rcp
+1nhv
+5mw2
+4mg7
+1drk
+4mzh
+1r6g
+5xpl
+4o74
+3bu8
+3cgf
+4nie
+2a3b
+1xos
+2wlz
+1a37
+5lpj
+1yvh
+4xhe
+2lct
+3lau
+3zki
+1g2l
+4uiv
+6b4u
+3jqf
+3ixk
+4pvt
+5lpl
+3sdk
+5km3
+1t7f
+4z2l
+4tk1
+2j9a
+3bbt
+3tjc
+2qi4
+4acg
+5tuz
+4lbo
+5uuu
+5ckr
+2h2e
+5qai
+1qxk
+2wyf
+3f0r
+1rd4
+2ipo
+1ukt
+2xx2
+3hvh
+2bmz
+1jqd
+5d24
+4cpy
+4bdk
+2xu4
+2evm
+6alc
+4r5y
+4oc5
+3kdm
+4gk3
+5wvd
+4pz5
+1x1z
+2duv
+6fnx
+5u4b
+4dfn
+4agp
+1n9a
+5mny
+3hqh
+3o8h
+4qem
+3l81
+4c0r
+1utp
+5z68
+5tuy
+5fb7
+3wv3
+3n7r
+3bze
+2bvd
+2fx6
+5vcw
+4ibf
+1zom
+4f6w
+2fdp
+1i91
+1d2s
+3ikd
+4w9e
+3fej
+4ntj
+4wk7
+2a29
+3f15
+4u6c
+1ing
+3wq6
+4ock
+2yb0
+3wtl
+5yl2
+2hz4
+2gbf
+3r8z
+5cso
+2j95
+5jxq
+4ofb
+2fr3
+5b4l
+3ovx
+6cha
+4qwl
+1tl9
+3mo2
+3c8e
+3k5x
+1uw6
+3fhr
+1b3f
+5a8z
+5bvk
+4o62
+2hb1
+3qxd
+3i4y
+6g9d
+6gop
+5ajv
+1oar
+5flt
+2zhd
+2xey
+4ikr
+4ckr
+4ufg
+5xvk
+2vmd
+5oci
+1syo
+4lq3
+1dtq
+5bpp
+6fg6
+4cki
+5wei
+3fl9
+5cnj
+4os5
+6awn
+3atm
+1ypj
+1odj
+5msb
+4f2w
+2h5a
+2w3k
+5w1v
+1pi4
+4kx8
+5m25
+1dhj
+2qki
+4b72
+1n0t
+3oag
+5dyo
+1ql9
+5u7k
+5mod
+5mtx
+4u6w
+1yk7
+3d8z
+2pnc
+5em8
+5cil
+3iny
+5l2t
+3u6a
+5fow
+2owb
+1k1y
+5nf9
+3ijg
+1qan
+4ad2
+2p09
+4xir
+4tqn
+5q11
+3zzh
+3qel
+5jha
+3rxh
+2ha0
+5xqx
+3q2g
+5wl0
+4bic
+6ccn
+1mh5
+2ai8
+1xp9
+4ef6
+5jzb
+2h8h
+5sxm
+3dm6
+2ym8
+6fmi
+5f1h
+4qyg
+2qtu
+3vfb
+1vwf
+1ivp
+3vf7
+5typ
+5igq
+4h1j
+3sug
+3h85
+2viy
+6hf5
+2poq
+5q17
+6afa
+4mpc
+6c0t
+4zsh
+5dit
+6aom
+3h8c
+2e9v
+2hfp
+4u5n
+5caq
+4m2w
+4tv3
+3oy8
+1pot
+3owk
+3arn
+2dri
+4ymj
+3qto
+5ix0
+4jr0
+5i5x
+1nzl
+5u4a
+4eg4
+1wbw
+3nee
+5fdp
+4cqe
+1hvs
+2w6q
+1n8v
+5ct7
+5dx4
+2fdd
+4ydf
+5c2e
+3qsb
+5zk3
+5vgy
+5ot8
+6boe
+1c29
+2pjl
+2euf
+6cnk
+3as1
+3abu
+3dpd
+5aku
+4k6w
+4j3l
+4nau
+3fpm
+5w7u
+3c3q
+3mg7
+1c12
+3buo
+3rhx
+5lyh
+1b0h
+4ynd
+5y0x
+5enb
+2a5b
+5mqe
+3l5e
+1tl1
+3way
+3gs7
+3ex3
+4np9
+5mxq
+2cnh
+1swr
+3t1m
+4eyj
+6mj7
+5f01
+3rkz
+4ln2
+4jia
+4l4z
+4qr4
+2er9
+5vnb
+5xff
+4prp
+2jqk
+1gi8
+2ynr
+1gfz
+5nk2
+1rsd
+5umz
+3jzf
+4k69
+6b5o
+4o4k
+4jal
+3nht
+4j1i
+5xof
+4ie7
+4f9w
+5tvn
+2qoh
+2jdy
+3wym
+1w31
+5c6o
+3nim
+1vso
+4z2g
+1o37
+3lbz
+4bqs
+3tfv
+5i8g
+3utu
+3msl
+1bra
+2k46
+4mg9
+2w76
+4wyp
+5jcj
+3b92
+3zha
+2i4x
+3wbl
+5trk
+2n8t
+3ge7
+4unn
+2ria
+4wgi
+4bds
+2y06
+6bhh
+4raq
+5i56
+3vv8
+2xk4
+4az0
+3ov1
+4ycw
+5ei4
+4u1b
+3bv3
+5etp
+3kcf
+6fod
+3eq9
+1awf
+1eat
+4giu
+3vyd
+2pvh
+5aaf
+2hdr
+1lgw
+2lsp
+4l02
+6b7b
+3ioe
+4c1m
+5kxi
+5yof
+4obv
+3r6t
+1uj6
+1a09
+4yk6
+1rzx
+3cy2
+6f8v
+1xor
+4la7
+1qwe
+5mxo
+2opb
+3psl
+1aqi
+5y5w
+2j4q
+5vfn
+3e37
+4fzg
+5qim
+6da4
+1oxr
+5hgc
+3at3
+5ieg
+6cq0
+2ioa
+3lpg
+4mzo
+1w7x
+2aia
+5v2l
+5abg
+5x5o
+2xcn
+5toz
+5tp0
+4z22
+4wlb
+4pd5
+3zbx
+3pb9
+3f5l
+6bke
+4p7s
+6dhc
+5enh
+4m3p
+4jmg
+1k03
+4pni
+4asy
+1a4r
+4deu
+4hy9
+4g0p
+5b5p
+3wqv
+2r3m
+6q73
+5ar5
+1drv
+4ua8
+4loy
+5eh8
+4wy3
+3jzh
+2aov
+6dj2
+5t27
+1kdk
+2og8
+10gs
+4uw1
+4hge
+5qcn
+6c4d
+1bai
+3i5n
+6epa
+3dgq
+5tzx
+4h3b
+5a4q
+1e5a
+5n93
+2exg
+5k4l
+1qpb
+1f3e
+4a0j
+5ula
+4cc3
+2oym
+1dis
+3ns7
+3uz5
+3ewz
+5q0r
+1qbo
+1nd5
+2qzr
+4oq3
+6e9l
+2n0w
+2hr6
+5vse
+5jsm
+1ssq
+3l4u
+1y2k
+2r3k
+3e3u
+4ibi
+3kgu
+1tni
+5hfj
+1dva
+5mno
+1qbu
+3px9
+5xpm
+5q1b
+4zjw
+4gqp
+3r21
+3lpp
+4bdt
+4ks2
+2v58
+3bft
+1il5
+5yco
+4p5e
+3rhk
+5fq9
+5ko1
+5bwc
+4q7p
+1ro7
+3lil
+4mwq
+3wf6
+1tft
+3hpt
+5t3n
+4oo9
+2b5j
+2bet
+3nuy
+5nea
+3kdd
+5ls6
+6f2n
+4rac
+4j47
+3iut
+4uvv
+2i4v
+1a8i
+5k48
+2hpa
+1mrs
+6fv4
+1q8u
+5aqz
+4ytc
+1ibc
+4eft
+2v95
+3me9
+4o0b
+4kn0
+3g2j
+1ai5
+1n51
+1gnm
+4zur
+4zsj
+3v8w
+4tzm
+1swn
+5ih9
+3gen
+4r6t
+2ykb
+5u7i
+4e81
+3v01
+3n4b
+1jmi
+4dcx
+3at4
+2c6c
+4mzl
+5ybi
+5tcj
+2w54
+4o13
+5ufc
+2xs0
+4flp
+4yqm
+3o8p
+3tpu
+5yft
+5mql
+5w6e
+2yjx
+4pqn
+4m48
+2iiv
+4ohm
+4a9c
+6en4
+3wk4
+3p50
+3bpr
+5gwz
+1mai
+5wys
+4oel
+1uy8
+4u01
+4ibc
+1qti
+4jpy
+1q41
+2gvd
+5aip
+5i0b
+4zyt
+4yxd
+4qt1
+4w4y
+1c8l
+3e5u
+3kqd
+4uj2
+3k4q
+4o42
+5o1h
+3b0w
+3t64
+1hty
+5dgm
+3bjm
+1b8o
+4l3p
+2b1z
+4yyi
+5ceh
+3l3m
+3ebl
+4bo8
+4fvq
+3fup
+5ot9
+3eax
+3vhk
+3cda
+5c1y
+3r2a
+1obx
+4tzn
+1d7i
+3qd3
+3wka
+3ovz
+3ihz
+5iis
+5vo6
+5tc0
+3fqk
+1dm2
+4y63
+6mvx
+4ucu
+5oht
+4jff
+1n8u
+4xua
+2ybp
+2cgf
+3d4q
+4gsy
+4i32
+2psv
+2ych
+5lck
+1xxe
+5wrs
+5trg
+5dp6
+2lnw
+5kr2
+4utx
+1a8t
+4qw5
+6emu
+2itt
+4m7y
+3kqo
+5ku3
+4n7u
+1oit
+1w1t
+4j26
+4nrp
+4yh3
+3nzw
+5zh2
+3vw9
+2hyy
+1qj1
+1csr
+2we3
+6mnf
+4ijh
+2jdk
+4kcg
+1kmv
+3buh
+4ovh
+1mxu
+4mg8
+3lka
+4twy
+5vsb
+4ofl
+6cdm
+2f3k
+5aqp
+4tkg
+5w4e
+3zot
+2yak
+3qkd
+2pv1
+5mz8
+5gut
+3avz
+2q8g
+3ilq
+3zw3
+4ckj
+1qbr
+4q08
+3zpu
+5u8f
+2ydv
+3rtp
+4b95
+2f9b
+2uz9
+4ehg
+4xsz
+1bkj
+6f22
+1tet
+3f3w
+4abd
+3sjf
+2byh
+5jsg
+4op3
+6fsy
+3twv
+5d25
+2vto
+2oa0
+6csq
+6dh3
+5jq9
+4xkx
+4tmf
+4brx
+2ghg
+2gvj
+4wnm
+5ayf
+4osf
+1u65
+3lzs
+1i90
+3gdt
+6eru
+5ops
+2mpm
+4cc7
+3ckp
+1o4d
+2vvs
+6ep4
+4bkz
+4aom
+1wc6
+1mpa
+2pl9
+2p53
+4zyr
+3gqz
+4og8
+4mlt
+1mf4
+1xxh
+3p3g
+3g0c
+5dbm
+4xhk
+5lny
+3g8i
+5g60
+2zas
+2z7h
+3kfa
+4a9u
+4gj7
+4nwd
+4ei4
+5yjf
+2won
+2f2c
+3mg8
+3i06
+2wa4
+3hzm
+3wkc
+4j7e
+3dxm
+3m9f
+1gzv
+4clp
+5a09
+5tg5
+4wkn
+1njf
+3ce0
+1tpz
+6bh0
+3vfa
+3su1
+2vtn
+5yp6
+5twj
+4i74
+5zk8
+2al5
+3bh9
+4djo
+4oi6
+5m4u
+1wva
+3uvx
+2aod
+1pqc
+4n4t
+5fdi
+1lcp
+4ere
+1ci7
+1ykp
+2qju
+1ftj
+5hk9
+5y3o
+5q0v
+6ayn
+5os1
+2j9l
+2qm9
+2x6j
+5ei2
+5izc
+3u18
+4zek
+4s1g
+2wq4
+1nfy
+6bnt
+4jai
+2jb6
+5ahw
+4uit
+3bmn
+4gzx
+6gxu
+4gki
+3ccb
+4ezq
+5v7a
+6dzq
+4az3
+4ucr
+4pnt
+2wtj
+5amn
+4ux6
+3t8v
+1i33
+3r7n
+5f00
+5nvy
+3k41
+4aq3
+3tdh
+3wdz
+5nib
+6b8y
+1v2h
+5jpt
+1xge
+4umn
+2xrw
+2vvn
+3t07
+2vpp
+1ghv
+4ydq
+2dbl
+3ho9
+5w88
+2g71
+2lk1
+2pwg
+4ai5
+1o41
+1tsv
+5al3
+4fk7
+2o4z
+4a51
+4gwk
+3ay0
+3own
+4e5g
+1lcj
+1hk3
+4rhy
+4yoi
+3prf
+5z99
+3slz
+4tmk
+1au0
+2qwg
+1bdq
+5h8b
+4emt
+6bec
+1g9d
+1qf2
+2r0u
+1f2o
+1b9t
+2c1p
+1hn4
+3skg
+4ee0
+6c2y
+4pg3
+3uw4
+1wdn
+3mho
+4yl1
+6cex
+3b7r
+4pop
+5f02
+3d7b
+1xbb
+2zft
+4i8z
+4oya
+3otf
+2xhm
+5c20
+1zrz
+4rqk
+5k8o
+1tvo
+6fnr
+1dwd
+4zlo
+1nyx
+5aad
+3kqw
+3nw7
+5ti6
+6e59
+1ywr
+2xn5
+4cy1
+1yfz
+5e3a
+4a7c
+5ljq
+5h9r
+1gi6
+2b1p
+1kc5
+3kpw
+1q5k
+4aml
+4ibg
+6e49
+2w8y
+3kmm
+2w6o
+3hdn
+6cef
+1u8t
+4e8y
+4gj6
+4hbm
+4g3f
+4eok
+2rox
+1t48
+2xd6
+2fjn
+5ndb
+4j77
+1n43
+3ui7
+5e73
+1h1h
+4mhs
+3ohf
+3kdc
+4edy
+2wzs
+5h9s
+3cth
+2j34
+1h62
+4jnm
+6bil
+3mhc
+4zw8
+3g90
+4f7v
+4eh8
+5wkh
+4ycm
+4mk8
+2xj0
+2tpi
+4clz
+4ko8
+5qaj
+5dus
+4xm7
+2uzl
+1k1l
+4g95
+1c3e
+5g10
+4nnr
+3e64
+1fq8
+4c71
+4b0c
+3ft3
+1ybg
+4y2t
+5dk4
+4dds
+3ggw
+5axi
+3ueo
+2r9b
+3ppj
+1meu
+4z7o
+3n8k
+2jup
+3p9j
+3hxe
+1e06
+3ika
+5yyf
+4kc4
+3su3
+4z7h
+4x2l
+5hcv
+5ir1
+1f9g
+5law
+3o99
+1yci
+3g3d
+3aje
+2o5d
+3b7i
+6af9
+2gvz
+4qmt
+3qj0
+2y9q
+1nw5
+5wg8
+4bzr
+1k6t
+5hd0
+3fui
+1lek
+5kql
+5d21
+1iig
+5ewh
+2vhq
+2gh6
+4x6y
+4mbl
+2v54
+5ug9
+3hvk
+3qts
+3q43
+4nvq
+3co9
+4uxh
+2gj4
+4dwb
+5q16
+1uyf
+5jbi
+2il2
+1yt9
+3t6j
+2zu5
+5buj
+2xiy
+2fs9
+1al8
+4cig
+3cgy
+4mdq
+3st6
+4dzy
+6fcl
+1m0o
+3spf
+1lkk
+4uu7
+1zky
+5d9l
+1zd3
+5t6f
+1g2o
+2i1m
+3khv
+3o88
+5fi2
+3idp
+2pov
+4x11
+2qe4
+3qtz
+5alm
+4pli
+3fts
+2x4u
+4o45
+1tkx
+2q8h
+1mue
+4eop
+2j7e
+5akk
+5wtt
+3h9o
+3v4x
+5b6c
+5boy
+5hdz
+5nkg
+6ajg
+4dum
+3whw
+6b59
+5aan
+6fo5
+4pmp
+5i0l
+2xk9
+5mon
+3d4y
+2xib
+3c2r
+5mfr
+2ra6
+2xl2
+4w5a
+3zm6
+3zm5
+3zhz
+3ogq
+4rrq
+4wxi
+4asd
+5txy
+5u0w
+4xta
+3t2v
+2q9m
+5lyw
+4kon
+3sv2
+2j7d
+3hmv
+6g92
+1z6f
+5a46
+3gnw
+5k03
+2bgn
+3ohi
+3acw
+3hng
+4j06
+1owj
+4fbe
+1gbt
+4fmo
+4g8v
+2fzc
+2vx0
+2o1c
+5unp
+4o7c
+4rxe
+5t90
+5jyo
+2wzy
+1tfq
+1tqf
+3s78
+6g0v
+2ei6
+4r95
+3peq
+3o9f
+1mq6
+1k6v
+6gfs
+6q6y
+5m39
+3wz7
+1nwl
+3ard
+5e3d
+2woq
+4gq6
+3iw5
+2h15
+3avh
+2azr
+5vsk
+6ckr
+4n84
+4eh2
+4cpw
+5f91
+4k3r
+4dgn
+2yer
+5ayt
+4cpz
+4y29
+5tq6
+3bcn
+3roc
+5lwm
+3gjd
+3k5v
+4zip
+4und
+6exm
+5qak
+4u0n
+5wo4
+1rrw
+6b2p
+5hls
+3pjc
+1cin
+3b2w
+1kmy
+4ys7
+4f0c
+1zgi
+3vqu
+3uoh
+1qr3
+5i8b
+4l7h
+2aay
+4jlm
+3wnr
+4en4
+5w0f
+4lxz
+4cjr
+4jj7
+2h5i
+2avi
+3ezv
+4j09
+3hkw
+3uxm
+4b5d
+5vcv
+2vtt
+3bl9
+2y1n
+5nin
+3uvo
+5anw
+5nvw
+4b9k
+1j4p
+4io5
+4dkp
+2jdt
+1iem
+4lm0
+5wyq
+5nr8
+6f7t
+5a81
+3lnk
+2xuf
+5ort
+1h1q
+4twd
+1e1y
+5lo5
+4anm
+1e1v
+4tki
+4wv6
+4htx
+1t13
+4euo
+6gy1
+4civ
+3dux
+5knv
+5hn7
+6gi6
+5vb5
+3f5p
+2kfh
+4b3u
+5nf5
+5mes
+4w55
+1sv3
+3p7a
+6frf
+6eks
+5afm
+5z4o
+3cow
+5hu1
+6f9g
+3ds1
+6a6w
+2g8r
+2ze1
+5q18
+5cau
+1s38
+1zge
+1me4
+3cj4
+4pr5
+4gzf
+1q54
+5l9i
+1z6j
+2ez5
+3unk
+3qwc
+3o86
+4j51
+6fyi
+3cj2
+2viz
+5ekm
+3f9y
+2uy5
+2brb
+5g45
+5jgd
+4j1e
+4ikt
+5opu
+4mwc
+1t7d
+2viq
+3ocp
+3wuv
+4yay
+3d7f
+2vh6
+4foc
+2ra0
+5fwg
+1o3c
+4bi0
+1kll
+3qtq
+3cl0
+3ho2
+3vw7
+6h37
+4rlw
+6dak
+3p3t
+3m11
+6fdp
+4c73
+5ksx
+6bcy
+1okl
+4gqq
+3sji
+2x2i
+1hkj
+5hu9
+5vom
+5fdo
+1uu7
+5iay
+5nu5
+6ghh
+4q06
+3apc
+1re8
+5uey
+2ha6
+4aaw
+1tcx
+3lpj
+3w0l
+3s45
+3n5j
+1rhj
+5d0r
+2qp6
+2vqj
+2zaz
+2zir
+4mq1
+5alt
+5he5
+3atp
+5w5j
+3ekw
+4xas
+5mwy
+4mwy
+2wzf
+2z1w
+1d4t
+4yk0
+3hnb
+4h3j
+6g36
+1rhr
+2r3f
+6bcr
+5owh
+5idp
+1odi
+3vw1
+4n6z
+3qcx
+4z89
+4pkb
+2ewp
+3d9z
+4c5d
+1xoz
+4meq
+3so6
+3tiw
+6h7u
+3u0d
+1ur9
+4lar
+4css
+2a3c
+1h3a
+4aqc
+5l2s
+5vi6
+6cmr
+2wf4
+3chs
+5uc4
+5tca
+2x6e
+4gne
+5ey9
+4ih5
+1mto
+4qwi
+5nai
+4ym4
+3prz
+2nnp
+1vij
+5bvd
+5swf
+3pn3
+2pql
+3ekv
+3nq9
+5c7n
+1els
+4k5m
+6bq0
+1o2k
+2aux
+4x69
+5mts
+1e9h
+4ykj
+1mkd
+6i3s
+1zpb
+5mo8
+2q92
+4u70
+4wph
+4xy9
+4ith
+5aph
+5cy3
+3fat
+4ly1
+2vti
+1f0q
+4wsy
+3ej1
+3ine
+3rux
+2vvu
+5a3t
+4qtd
+5a5s
+3t7g
+1xon
+4gah
+5flq
+2hw2
+3aig
+4hl5
+5wzr
+2imd
+3oxz
+1ndv
+5yji
+2hxm
+3l59
+4jpx
+1ws5
+4uix
+2iv9
+4dwk
+4qvp
+2c1b
+5cw8
+2r6n
+4cws
+2d3z
+2oc9
+4ehz
+1ett
+1fdq
+2avo
+4r0a
+1zzz
+3moe
+4mmp
+5muc
+5zh5
+3zqi
+6f08
+6elo
+6dif
+3mf5
+5qas
+3fu5
+3rv9
+4jne
+5urj
+5oxk
+4y3y
+5hki
+4tyb
+2glm
+2xnb
+3tt0
+6gmq
+4i5p
+1f92
+3psd
+5hcy
+1q6n
+4tmn
+1tmn
+2gg0
+3zep
+1c88
+4fea
+5nwb
+1ppc
+5myx
+4ci3
+1z6d
+4kww
+4ixu
+1leg
+3a5y
+4bo3
+5cyi
+5uzj
+4mw0
+5j18
+5ml3
+2bmk
+1e66
+2brc
+2fwy
+5oxm
+1bn1
+4qlv
+4qip
+3ms2
+5wxf
+5yib
+3ig7
+4pgb
+4p73
+2y4s
+5zh4
+4ch2
+5tex
+3b4f
+3poa
+4zyx
+4m0y
+2ot1
+5e91
+1yc5
+3iqi
+2xpc
+1o2s
+1pmu
+4l58
+4mx1
+5am4
+2hs2
+4yne
+1ctt
+3l5c
+2zm1
+1lxf
+2f1b
+4zcs
+4zw5
+2oh0
+5n2z
+3f19
+2obf
+6f9t
+2f01
+6euc
+5vee
+4oyo
+5moq
+3ft4
+6cbg
+5wuu
+5f5b
+4xs2
+1ype
+4rwl
+4hpi
+4e34
+1yc1
+5jy0
+2fxr
+5e1m
+5oq6
+1nhz
+3pjt
+5jzi
+4elf
+4fxy
+6dpz
+5a6a
+2waj
+2fqt
+3mxy
+1y91
+6biv
+4j1k
+3zmi
+5xjm
+2fv5
+5db3
+3ryj
+3b95
+3i5r
+2ajl
+6c99
+3bx5
+5ave
+3muz
+2ye9
+2rf2
+2j2i
+5g0q
+4yuw
+2r3g
+4cwf
+3t3h
+5ajz
+2bgr
+4l9i
+4igk
+6cz4
+4qfp
+5ty8
+4odm
+3hs8
+4ele
+5zia
+2pvj
+5w5k
+4m5l
+3wz8
+4egh
+5w7j
+4jfw
+6eda
+5tpb
+5m5d
+4o71
+3kag
+4u93
+6eed
+4zam
+2jbp
+4z3v
+2pu0
+4zzd
+4f14
+4h4d
+1pxh
+2hwp
+1u9e
+4wko
+5kqx
+1tog
+4yqu
+2yi0
+2xf0
+4uof
+4io7
+4ds1
+5diq
+3ti5
+4x8u
+2lp8
+3avm
+3kqm
+4zph
+2wou
+4jyu
+3ip8
+5ti3
+4cku
+3h78
+3vdb
+3chd
+4i5h
+5i2r
+3tyv
+4uwk
+5j6a
+3lk0
+5k13
+2hj4
+3bfu
+4ajk
+3n45
+3te5
+3ujb
+5v9p
+2bzz
+3a6t
+2y6o
+3b28
+1hvl
+5a2i
+4mx9
+4g8y
+4dko
+3kdb
+1lf9
+4qmp
+5q10
+3hqr
+4d2d
+3w32
+4ksp
+6f8b
+3h6z
+5k0h
+3h2n
+5ke0
+5w4w
+5mxr
+5igl
+2h9p
+4iuu
+4d8a
+3dvp
+1lox
+3ao4
+4h38
+4bdb
+4j1p
+1o4b
+4mro
+5eel
+2ay3
+3e8n
+6g8m
+2pyy
+4m0z
+2mas
+4ks3
+4p72
+6ccx
+3ttn
+4jsr
+6aqq
+6em6
+5o3r
+4rqz
+4hkk
+4qll
+2q6f
+5he1
+4fe9
+2wu6
+2pj7
+1bty
+1ule
+6dry
+5ezg
+5h17
+3s7b
+4x48
+3atw
+1kug
+2hdu
+3odu
+3l9m
+1d1p
+2ihj
+5mgk
+2n1g
+6hk6
+1mq5
+4kp5
+1yyr
+5al4
+1pzj
+6axl
+2z4b
+5mym
+4gv8
+2cni
+4xub
+6cjh
+3ig1
+4m2r
+1tt1
+5ime
+5oei
+6d6u
+1utj
+1dfo
+3ixj
+5anu
+2ogz
+5bwb
+5vad
+2x9e
+6epz
+4yv2
+2jdh
+1w13
+4eos
+4x1f
+4gao
+5c8k
+4k5n
+3pj1
+5hz5
+1x7a
+1h9z
+6gjm
+3ipe
+3fuj
+4m3d
+5g11
+4asj
+5xig
+4ybs
+5gmi
+3d04
+3dv1
+2fie
+4twt
+3ary
+4anb
+6hmg
+4n9c
+2lsk
+2ksb
+5mwd
+1gfw
+5gja
+3u3z
+2f7o
+3mzc
+5w9g
+5yz2
+4nmq
+2y2k
+5lz5
+4qz7
+1bug
+4zmf
+1o8b
+5jf3
+2qft
+5jop
+4jaj
+3s9i
+6cvy
+3oaf
+4hlw
+5ta6
+5mae
+2yg2
+5m2v
+4zxy
+1g9r
+3iqu
+5i9y
+3pch
+5kkt
+2fqo
+5mn1
+3e3b
+4jlj
+4tq3
+4zga
+4jlg
+1pf7
+5ewd
+4dhf
+4ywa
+5fxr
+4e1k
+2zmd
+4jyv
+1fkg
+2lwi
+6iik
+3zxe
+6dgx
+2vx1
+3hav
+3pkn
+1trd
+3u10
+2a0c
+3ly2
+2xn6
+1irs
+2ntf
+3rum
+6iin
+5hzx
+1urw
+2xb7
+4umr
+5apr
+5ngt
+3q71
+3og7
+4oeg
+4tte
+2ow6
+3sff
+3eu7
+2zju
+5tx5
+3v8t
+1nqc
+3t2c
+4afh
+3vhu
+2d1x
+2h2h
+5nvz
+4qy8
+3w2s
+4w9h
+4r6v
+5esq
+5v5d
+3rbu
+3d51
+4q07
+4jrv
+2in6
+3bki
+5ov9
+6ayd
+4zyc
+4ybm
+4xip
+3ibu
+5kde
+1br5
+1lan
+1gwq
+3ggv
+3cke
+4av0
+4qpd
+4urk
+4o04
+3d4z
+4uct
+5uwj
+4b8y
+5nvc
+6f23
+5w0e
+4m84
+2hdq
+5c6p
+2amt
+2rgu
+4ax9
+6g9n
+6hu2
+6cho
+3v7d
+5iz9
+2vev
+3dej
+2hxq
+4mvx
+2c3l
+5c29
+1we2
+6f9u
+1m6p
+2f80
+2ooz
+4py2
+4zcw
+5kx7
+4m13
+4dhp
+4kz6
+5ypw
+2zdn
+1qf5
+2i4w
+2xu3
+1wbn
+5lt9
+3zmv
+4efs
+7cpa
+2i72
+4rpo
+2h13
+4obo
+3f88
+6g47
+2y6d
+3o57
+5a0c
+2fl6
+5mja
+5jal
+3aqa
+3g9l
+5g22
+5znp
+1hi4
+1y2a
+6ay5
+3cf8
+3f5k
+1a0q
+3et7
+4eky
+6bvb
+5u94
+1m1b
+3ryv
+2peh
+3vw2
+5e1e
+3i51
+5fb0
+1rtl
+2i4p
+1a9m
+6au5
+4jyt
+4k3k
+2c5o
+2qu6
+4hwr
+6bqk
+4bqh
+5cs6
+4dea
+4x3r
+4iid
+6gzm
+2y2n
+3fuk
+2ypi
+1b3l
+4zsq
+4o44
+2e1w
+3rdo
+4zme
+4zz2
+4ucs
+3sl0
+6ey9
+5arg
+3p3r
+6an1
+3eor
+2uue
+1n4h
+3gk2
+3oot
+5i5z
+3hv3
+2hzi
+5yc2
+1g48
+3bxg
+5wbq
+4a1w
+5q0q
+4axm
+6g91
+4q0l
+1d8e
+1bb0
+1lpg
+1gmy
+6gin
+5yy4
+2pvv
+6ht8
+5j82
+2x5o
+3orn
+2nmb
+1pk0
+3rik
+4lkd
+1v2n
+4ufl
+6ahi
+5bsk
+3uza
+2w8f
+4nka
+6bef
+5abw
+6bkh
+5l30
+2clx
+4an1
+4ufz
+5nk6
+1syi
+3zsy
+5ul5
+3irx
+5ux4
+4r5n
+4bdc
+5fut
+3e2m
+3lgp
+2ivz
+1akv
+2ydi
+4i11
+3zxv
+4m3e
+4kwg
+4g2w
+3o95
+6gu4
+2doo
+5yun
+3n5e
+3mqf
+2y5k
+2vnm
+3g0g
+1qfs
+4o7b
+3ty0
+4uil
+3st5
+3f36
+6fqo
+4pkw
+5j6m
+4xbo
+1hxk
+6ghj
+1h8y
+2pq9
+5em3
+1rlp
+1yvf
+4cqg
+3srb
+6fmk
+4o6e
+5xmv
+4agq
+2j3q
+1ke9
+1mfd
+1a5h
+4b1j
+4i0f
+3up2
+2ay5
+5mqx
+3lpu
+3bmq
+5wfw
+4j5b
+5alp
+5ee8
+4qj0
+1ka7
+3k3h
+4np2
+3bwj
+5osl
+5all
+4jze
+3b3x
+3sm0
+2xa4
+3t3v
+3t1n
+4yp8
+4bc5
+5f08
+4qw6
+5ofv
+2bdl
+4y83
+4y4j
+2prj
+6cfc
+5eth
+1oky
+2ban
+4blb
+3f38
+2uy3
+6fdq
+4qmx
+4bo5
+5wh5
+6cvv
+2oph
+3owb
+1xlx
+2cf9
+4av4
+2wxv
+5v8p
+4eqj
+4aoc
+5jk3
+4xtx
+2aoj
+1v7a
+3lzv
+3bum
+3fxw
+4qhc
+1b6j
+3f80
+4or0
+6dq5
+5lsx
+4bdd
+4x8v
+4od9
+3vye
+2yix
+1i7i
+4eoy
+2k4i
+2bv4
+4z0f
+5dp7
+4xiq
+3d20
+5llo
+2ymd
+4hn2
+5n31
+4ffs
+4b5b
+5jf1
+4i73
+2pqb
+2i19
+1cbx
+1gsz
+5ws3
+4h3q
+5a5q
+4uvy
+4guj
+1i7g
+3c8a
+4ll3
+4pp0
+5hk1
+4rux
+3coj
+3d6p
+1ndj
+1o79
+3nsh
+5hl9
+2e27
+5iui
+4og5
+3wdd
+1lnm
+3igb
+4x24
+4tsx
+2gmk
+5ox5
+1bqn
+4e8w
+5xyf
+4uco
+1eas
+2b7f
+4m5u
+5evk
+3ipu
+4p5d
+2fsa
+4ay5
+4bcp
+5thj
+3tam
+4jsc
+3ug2
+5e7n
+1ofz
+3uo4
+5clm
+3gr2
+2m0u
+6f7q
+3sdg
+1sme
+3g2i
+1oyt
+3h3c
+1xff
+4oc6
+3b3c
+1xa5
+1fe3
+1g32
+5olb
+5a85
+2oz2
+4d8i
+5swh
+1y2d
+1inc
+4obz
+1c7f
+2clh
+1qkn
+2etm
+4i5c
+3vd7
+1p28
+5j7s
+1y2j
+1aht
+4iq6
+2pe2
+4f3c
+4ih7
+1hkk
+4puj
+3o1g
+4w9i
+3qrj
+1qj6
+4azg
+4joo
+6dxx
+4sga
+1gja
+5k4j
+4yef
+1sbg
+3s00
+5fcw
+4txe
+4mr5
+1bxo
+5xxf
+5jic
+4abh
+3nzx
+5edl
+3pcf
+3k98
+3qqu
+1hgj
+2qch
+2yim
+4dtt
+4l09
+3f07
+4hki
+4cl6
+2z5o
+4e7r
+5amd
+1c1v
+1ym1
+1s64
+2w6t
+3tl5
+1w1y
+4eoh
+5i58
+4z5w
+1u2y
+6hu1
+3wav
+1nxy
+3n4c
+5c91
+2kzu
+5kre
+2jkk
+4hkp
+4y2y
+4w54
+4bnu
+3lp7
+3bla
+1gni
+2idw
+2xy9
+2q6b
+3uxd
+5o9p
+3ao1
+3v2o
+1i00
+5ewj
+2rgp
+2xln
+6ma5
+4d2w
+6eww
+4lkq
+4o7a
+5m4k
+5g3n
+1wm1
+4fzj
+2nyr
+2mc1
+2qky
+6eh2
+5v1y
+4glw
+3arb
+4gq4
+2vin
+4rss
+3q7j
+2h4g
+2x2c
+1gt4
+1fo0
+1ftk
+4kby
+1ch8
+5fnu
+3sv7
+2vvo
+5uah
+3k97
+2wpa
+1a4h
+5lhg
+6eeh
+4mse
+3nnv
+6eq4
+5f1l
+3q0z
+3elm
+2uzd
+3p2e
+1r0p
+1j16
+2qhz
+5h0b
+4nyt
+5ij7
+4l2l
+1afl
+5vd0
+4i10
+1opi
+2ves
+5tyn
+3gf2
+3nuo
+5w2p
+4uv9
+3sut
+2f3f
+1dmb
+3wd2
+4xm8
+4kp4
+1hy7
+4xbd
+5m6h
+1g05
+4zh3
+5d0c
+2e7l
+3kba
+5nhh
+2pax
+1p0y
+2l7u
+3wns
+5u4d
+3axk
+4qpl
+5jt2
+2rk7
+5lss
+3sni
+5aaa
+5wfm
+4pb1
+3oe4
+5m63
+5xg5
+3lle
+2yac
+4odk
+2x95
+2ig0
+2zdl
+4ay6
+2c1a
+1nny
+1nw4
+2fj0
+3kah
+4db7
+1bxq
+3ifl
+5f9e
+4p2t
+4exh
+6d2o
+5t2b
+6eee
+2oi0
+3lpl
+4tk3
+1o6i
+6eux
+2ojj
+3pyy
+4edz
+3jy0
+3r5t
+4xdo
+3qo9
+4u2w
+5nme
+6hdn
+4mre
+4ryg
+5ez0
+3c14
+5kzi
+6bix
+3uug
+5ldm
+1o2v
+4llx
+3lw0
+2op9
+4nhc
+5wqj
+5wi1
+4e20
+6fjf
+2x2m
+5l98
+3qsd
+5lcj
+5t23
+1adl
+2wa8
+4q9y
+2rkg
+4gid
+3f70
+2xk6
+5t8e
+4lxb
+2xp2
+3juk
+5uf0
+3l1s
+6evr
+1lag
+1awi
+6ej4
+1pmx
+4kln
+3wgg
+5g61
+5n69
+4o9v
+3kqc
+1him
+3cd0
+2kbs
+1moq
+2f10
+2zb0
+2d41
+4bdh
+1ebz
+5lcf
+5u13
+4qwk
+4cd8
+4d85
+1tvr
+4l6s
+3pd4
+1hq5
+5tiu
+1o44
+6g22
+2ckm
+4btk
+2cli
+6cb5
+3o0u
+5u4x
+4lp6
+5afk
+2o3p
+1zpc
+3qo3
+3d1z
+2z3h
+1hms
+5ox6
+5fpp
+3f3d
+5t78
+5hg5
+3mxe
+4i7l
+1tuf
+5lmk
+3nu9
+5om9
+2qi3
+3l5b
+4v0i
+2nv7
+2wkt
+4ndu
+1aq1
+3ddq
+1vyj
+5ods
+1b8n
+4xhl
+4jft
+3n7o
+4og6
+5j9z
+5ufr
+4qjm
+3g4g
+5jox
+2std
+4uvx
+2yb9
+3qaa
+5mwo
+2onz
+2hiw
+1xnz
+2agv
+4qgg
+5x28
+2dw7
+2ohk
+4oue
+1o4a
+6ds0
+3t5u
+3ij1
+4klv
+3lgl
+2wey
+1qk3
+4i0d
+1rti
+3clp
+4f8h
+4qwj
+1bmb
+5jar
+5a5z
+5ofu
+2yiu
+2z5t
+5emm
+4ibm
+5lzh
+4xh2
+3t09
+2p95
+1fmb
+1gj5
+3s72
+5ia5
+4kp6
+1s9t
+2fhy
+4del
+3ao5
+4xii
+5cp9
+3vb4
+3sha
+5el2
+2uup
+6bfe
+4jv6
+3e8u
+3ubd
+1tys
+4p10
+5ekj
+1d5j
+4bpi
+3n23
+3amb
+4gue
+4n8q
+1o0d
+4d1j
+1a9q
+2vtq
+1h1d
+5uw5
+6bfp
+3tfu
+5xyz
+2q1l
+4m7j
+3m8u
+5kbf
+3btl
+4ctj
+1koj
+2ajb
+3unz
+2xgs
+4pd9
+5g1c
+5w19
+5vqw
+5yc8
+4ufd
+1w83
+4qdk
+1tyn
+6buv
+4amw
+3dnd
+1d7j
+1nzv
+4ryl
+2f6j
+6bbx
+5lgp
+6ay3
+1fwv
+2qg0
+3mg4
+3qmk
+4dma
+2r3c
+1d6s
+5dsx
+1tg5
+6do3
+5n7v
+1g5f
+1c5q
+1lb6
+4pv7
+3ii5
+4rab
+4eke
+2w10
+5i86
+1uho
+5vsf
+3g08
+2zxg
+4btl
+3hb4
+5j4y
+5lgt
+6boy
+5kva
+5mka
+5wa8
+2i0e
+3ptg
+5vp9
+4mw2
+3djf
+6b7d
+5t1a
+5hrw
+5e2o
+4xuc
+3vg1
+6c4g
+2csn
+3q32
+4q9m
+3vry
+3fv7
+5lhh
+3fh5
+2qry
+4f9g
+4c8r
+4m6p
+4riu
+3dp3
+2r3i
+3bwk
+6ew3
+4bs5
+4g3e
+2wvz
+3wc5
+4nb6
+5xmr
+4ps5
+4dvi
+3uvm
+5os7
+2xix
+5alw
+3iaw
+2xnp
+6dvo
+1cgl
+4xv3
+5dya
+3c79
+2e9a
+6dil
+4caf
+4tpw
+5fov
+5n3w
+5ttv
+6f3b
+5cas
+5aqg
+3dz2
+1xhy
+3t01
+5th4
+4i80
+4q3t
+2ceq
+4yff
+3bti
+5vll
+4hy4
+6g3y
+4ykn
+5lu2
+5wg7
+3fnu
+6g84
+6ap6
+6hpg
+1c3x
+5if4
+5wgp
+3d6q
+2wer
+2qrq
+4cfl
+3uib
+6fgf
+5tx3
+3plu
+1zzl
+6db4
+5tpg
+3k8o
+2fxv
+3c2u
+1w8l
+1di8
+3udh
+4cu1
+5gvl
+5nvx
+1bt6
+6d9x
+1vjb
+6b31
+2qo1
+3daz
+1v0l
+5nuu
+2qtb
+5ewy
+4i9u
+4afj
+5hzn
+2ohq
+4eqc
+3p8h
+5y53
+1kat
+4kwo
+2hiz
+2cmf
+4bgx
+4hys
+4lww
+3k39
+3rdq
+4ega
+5fsl
+3ehw
+5gnk
+5n8b
+6h29
+1dy4
+3nu6
+5dxu
+5jf8
+2qd9
+5zaj
+1oif
+2v8w
+1ywi
+5uiq
+1xr8
+4yvz
+2y82
+1pkx
+2qbw
+1ebg
+3a4p
+3ful
+4d62
+5qb2
+6bgx
+1ela
+5j59
+2q63
+6cmj
+2p9a
+5nwk
+3w54
+4rrg
+2l1r
+2gsu
+2wva
+2vpn
+1qji
+4gts
+6gnw
+4afg
+5ty9
+4ps7
+3r2f
+3qak
+4dgg
+5f67
+1kak
+4mc9
+4dpu
+4u5o
+3h06
+2b4m
+4q99
+4dgb
+4hw2
+5uvc
+1yxd
+4cix
+4ehm
+5his
+2ihq
+4kmd
+5cin
+5v7i
+3itz
+3il5
+3brn
+1h4w
+2ilp
+6bnk
+1me3
+5fh7
+1rgk
+3iub
+3p4q
+1njt
+1ya4
+2x6i
+3e81
+4yl0
+1at6
+3pr0
+5wf3
+4uzh
+5f2k
+4j93
+4dwg
+5ti2
+4pcs
+3qcj
+4qjo
+4dpf
+4qer
+2kp8
+4o2p
+4pd6
+2op3
+6got
+2weq
+4lh7
+5l97
+6bed
+5o1e
+4ezt
+5y8w
+1err
+4rj7
+2yol
+5t68
+2ltx
+5g17
+1mxl
+4c7t
+3uvu
+5a3r
+4yuz
+3nf9
+2fix
+3pgu
+4kn1
+3mi3
+5dpw
+3e5a
+1y1z
+5j8i
+4e5d
+3cd8
+6ccl
+4x9r
+3rtx
+2zmj
+1xp6
+6cks
+3fw3
+4znx
+4nyj
+1q1g
+2ews
+5o5m
+4w4v
+2w1d
+5d1j
+5fbe
+3ipq
+2qk5
+2ydj
+3gv6
+1mxo
+5l9h
+3odk
+6cze
+6c98
+2wmx
+3qa2
+5oax
+6gnm
+3q6z
+2oqv
+5exn
+3s53
+5lzg
+3mo5
+6drz
+5nn5
+4x1q
+2mow
+3ow3
+4ebw
+4a4c
+6ema
+2h6q
+2fjm
+5uez
+6et8
+6ep9
+1avd
+2q80
+4nj3
+4mdr
+5ely
+5dri
+1v2r
+1hk4
+5xfj
+4z0q
+4heg
+5h3q
+2buv
+4l7d
+3nb5
+1g9b
+3oay
+4aj4
+1bxr
+2w9h
+2iws
+3oy1
+3qpo
+2hog
+3rl8
+1zgb
+1a4w
+3vvz
+5ia4
+6f1n
+1x70
+3q5u
+4do4
+2pcp
+5m77
+4a50
+1elb
+1rri
+2ydm
+3ldp
+4k9g
+2ptz
+3uo6
+2qwd
+5fdz
+6fiv
+1hqh
+5q12
+5anv
+5k5e
+5aer
+1jys
+2zu3
+2oiq
+5lwn
+1i9o
+1npv
+4piq
+2w0j
+6c5f
+1w8m
+5gjd
+5yql
+1m83
+4djv
+3ns9
+3zn1
+4eg7
+3u4r
+6hu0
+4awq
+4l6q
+1ad8
+3d9k
+5ucj
+3piz
+3ekt
+1g9a
+3m2w
+5q1c
+1mv0
+6emh
+1ttv
+5u12
+1lhf
+1w1p
+2i4t
+4z68
+4gqr
+3uqp
+1rhq
+3oxc
+4mc6
+3adt
+3ivi
+3r8u
+1udt
+1o36
+4ou3
+6apz
+5gmp
+2l84
+2gg9
+3o64
+3uo9
+4qht
+6f6r
+3hy7
+1nox
+5u6v
+3ans
+1mdl
+3d2e
+2fsv
+5t2m
+3p79
+4je7
+3gta
+4q9z
+6f55
+1gu1
+5y6k
+3ert
+3ant
+5a3h
+3zdv
+3o1d
+6ar4
+2ym4
+3s3v
+3fk1
+3pbb
+3r6c
+1xh8
+5lh4
+5q0w
+2psu
+5gty
+2qi0
+2nn7
+5vb6
+4b8o
+1zp8
+5uqv
+4pci
+5f5z
+4rj4
+6dh4
+3lce
+4ea2
+4y6m
+5d3p
+1gsf
+4l7l
+1tkt
+3a29
+5ljj
+4gpk
+3ayc
+3qkv
+3ern
+3zhf
+1f4y
+6bgg
+5uz0
+2znt
+5du4
+1utn
+5ayy
+1lzo
+1gym
+5k8s
+3kb3
+3ijy
+5oa2
+5wio
+1yei
+4hlh
+2eg8
+5tkd
+1kpm
+3tpr
+3mt9
+3tcy
+4qxr
+3uyt
+3o9l
+5ejv
+6ekn
+2fky
+5vp0
+4rxa
+2xpa
+1kav
+3ecn
+5vlh
+4lko
+2g83
+4m5r
+5iql
+5g53
+3fhe
+2xj7
+3u78
+3zt3
+4hkn
+4h58
+4f1l
+5q0t
+2jkh
+5w13
+3r88
+1yvz
+4qmq
+5aag
+1b6m
+4i47
+6d5e
+6c91
+1z95
+4ca6
+4ysl
+4gbz
+1aj7
+1wcq
+3m96
+5t92
+6gjl
+4yxo
+5l2m
+4wov
+3l2y
+5v5y
+3a3y
+2gmx
+2fo4
+2h5j
+2r05
+2qtt
+4jn4
+5npf
+1gt5
+4jfd
+2qhr
+2sfp
+3rx5
+4e4l
+4mao
+1zkl
+3syr
+3jzj
+5h9q
+5one
+4io6
+3lp2
+2i4j
+3rlp
+5eta
+4yth
+3r24
+3gsg
+2of4
+5inh
+3ed0
+3zlr
+5llp
+6esj
+5c8m
+3dkf
+4z6h
+4x6o
+2j47
+1usk
+5w8j
+4x1p
+1xb7
+1lqe
+4apr
+5khm
+2ga2
+2ly0
+4zro
+4e26
+4g16
+4uvw
+1f5l
+4ipn
+4hwt
+4g90
+1kds
+4bck
+5k6s
+5eqp
+1vwn
+4uye
+4gtq
+6fe1
+6gjj
+3hb8
+2zdm
+5alz
+3eyg
+4lkk
+4os2
+1j4q
+4o36
+5dtm
+2wgi
+6faf
+1mx1
+3lfn
+2i1r
+3hzy
+1a4k
+4zl4
+1dub
+3qi3
+3i90
+3qgy
+6gfy
+1o2u
+4xyn
+3rk7
+6atv
+2wu7
+4tnw
+2uwo
+1z6p
+3p8o
+3my1
+3s3q
+5w0l
+4tw8
+6b8u
+5jg1
+2azm
+4qw3
+5ntk
+5jq8
+3gcq
+2zcs
+2pvw
+1o0n
+1ph0
+3nu5
+5loh
+1t08
+4m7b
+1qpl
+4ge5
+4w9s
+5fat
+5l2w
+5nx2
+2gst
+2pix
+5myn
+3ump
+2xoi
+4r7m
+5ful
+1kf6
+2on3
+5wa9
+5tw3
+1q4k
+6fu4
+5umx
+2bdy
+4ezy
+4crj
+4pyx
+5t97
+2cet
+1fao
+1n46
+2ooh
+3dgo
+4io4
+1xjd
+1q72
+3mag
+6euv
+4g5f
+4dgm
+5o1c
+3ken
+4d2r
+5fnq
+4z7f
+2o3z
+4xgz
+4i2z
+6g15
+5ypp
+2x09
+5lt6
+3k5g
+2xej
+1uyh
+4tln
+4mrw
+3c10
+3sdi
+4tw6
+3pa4
+3q4k
+5l3a
+3b9g
+4ohp
+3nq3
+5vfj
+4rqi
+4x7k
+2q8s
+1ca8
+5a3x
+4wf6
+5oh4
+2qu3
+5dq8
+6ffs
+4loq
+4prb
+5vwk
+2yhw
+5eob
+5dgj
+2i4d
+5wlo
+5lo6
+5aln
+3p5l
+4q1y
+6c4u
+1gi7
+2jkt
+4yv1
+3ctj
+4qfr
+4wmy
+5c84
+3ie3
+5mnn
+3l8v
+6eij
+1o2n
+4nmv
+5ok6
+4yab
+2vnf
+3r0w
+2uyq
+5j19
+1y2c
+3nth
+5ni0
+4tpk
+1vyq
+4txs
+2wgs
+5gv2
+1b3g
+3kad
+6hgz
+3dng
+4u7q
+5fnd
+2c5x
+5ho6
+2aqu
+1ke6
+5g5f
+3wff
+3m36
+4bb4
+3ai8
+3ee2
+5u14
+4xe1
+4ggz
+4av5
+3bgc
+5i4v
+3g0d
+4bkj
+2aei
+4hup
+4yv9
+4uwf
+2ovy
+2oo8
+4dlj
+2v83
+1ct8
+5qan
+4nwc
+4msc
+4iti
+3guz
+6i8l
+2yof
+1oyn
+3qti
+5nf6
+5n7x
+3t0t
+3oyw
+4ab9
+3aw0
+2x2l
+2lkk
+3oyq
+6c28
+2ofv
+1b58
+4cpt
+4em7
+1t5a
+4xyf
+3duy
+4xh6
+1j15
+3b7j
+2isw
+3eqs
+3tkz
+3dgl
+3qw5
+6df4
+2jjb
+5auy
+2rcu
+5mqy
+2aoe
+1d8f
+1kl5
+4zsg
+3e0p
+3su6
+5tz3
+4hco
+3i6c
+2ow1
+2yjw
+4nue
+1uyd
+4hod
+3oik
+2viv
+4wtu
+4gih
+1b9s
+5i40
+4zb6
+5fns
+4x6h
+5hg9
+5naw
+1h37
+1uj0
+3e9i
+3i5z
+1v2v
+5wb3
+3lpt
+3k4d
+3nhi
+1cbr
+5uis
+3f68
+5nk3
+3okv
+3krr
+4gnf
+3zyf
+3ivg
+5d3n
+4rs0
+5oxn
+1owd
+1w7h
+5o4s
+3hrf
+2r3h
+1yp9
+5oyd
+4iif
+4uvb
+1b9j
+2bgd
+6giu
+5eek
+3tnh
+3gp0
+3bxs
+2ay1
+5cap
+3ta0
+3mkf
+4uyf
+4zx6
+5a3n
+3hek
+6b5r
+1fbm
+5wou
+3g76
+2rkf
+5j86
+5lav
+2rkn
+3jzo
+3kr5
+2vo4
+4lzr
+2srt
+4wbg
+1vyz
+3ebi
+2qbx
+3gbb
+3vnt
+3erd
+1o34
+1lbk
+6czi
+4d1c
+1k1p
+2bmv
+1jlr
+5hx6
+2igw
+4o0r
+2isv
+3kr8
+4k3m
+5xiw
+2z5s
+2w97
+4u03
+4p4d
+1pwp
+5j27
+3w5e
+5a3s
+1zz3
+4y5i
+4rx9
+4ty7
+6g4z
+4j3e
+1s3k
+6ajh
+2wei
+5v5o
+4p45
+4nbl
+4rxd
+1cp6
+1r5h
+4nnw
+4cga
+3kb7
+4phw
+3zpq
+4b3c
+1wv7
+5hja
+1mfi
+3jzk
+1hsh
+6aro
+3vyf
+2v7d
+4k66
+5eeq
+5mos
+3ska
+4bg1
+4ql8
+4lgg
+4anw
+6gr7
+2x7s
+4ie4
+1p05
+4zbi
+5c6v
+3qzv
+5etq
+1nhw
+2xx4
+1uys
+3zsq
+4h75
+4d0w
+4pml
+3gyn
+2fm5
+4r93
+5usq
+4e3b
+2hwi
+3kl8
+2vr4
+3o0e
+3dab
+5h8g
+4ynb
+1kr3
+4w9d
+4msl
+5k8v
+5ix1
+5zaf
+6cve
+4h85
+3d5m
+3v78
+5uox
+5d1u
+4fcm
+2zz1
+4i1r
+5hkm
+4y8c
+1fkn
+5he2
+3n3g
+3g5v
+2evl
+3k3j
+1utr
+3ixg
+4u44
+4ahs
+1o2y
+3fcf
+5dde
+3bgm
+2nxd
+4yhq
+4k19
+5y7w
+3v3m
+4ph4
+1nja
+5i23
+3rsv
+5n17
+4bt5
+2ojg
+5x72
+6eq7
+3ehx
+1a5v
+6clv
+3qcs
+5cy9
+3bgp
+3nf7
+4dmn
+2dxs
+4msk
+2wmu
+4hws
+5c2a
+3lco
+4x3u
+5c7a
+1pa9
+5he7
+3m67
+1j7z
+3s5y
+3oe0
+4kql
+5l8y
+5cks
+4cc5
+2rcb
+3bi1
+3jzc
+2qzx
+4u73
+2an5
+2uvm
+6g2n
+1dzj
+6b97
+5y6d
+1qvu
+4ymx
+1yej
+6f3d
+5tg7
+2w1i
+3dv5
+1vyw
+3sw9
+4qz3
+8a3h
+3o3j
+5gn9
+4b7j
+2i4u
+3m8q
+2fu8
+4hfp
+1swi
+2vd0
+3pz1
+2jo9
+3q4c
+4bzo
+4tyl
+6fuj
+2x7c
+3hg1
+3pty
+5f3i
+4q9o
+3e1r
+4j78
+4ucv
+6h38
+3wtk
+4lno
+2h65
+3s4q
+3lp0
+4ajo
+5maj
+3d7d
+4xe0
+6hpw
+4p0b
+5fsb
+4f6u
+5h0g
+2zdk
+6hoy
+6bib
+3rpv
+1kel
+4v24
+5k1i
+4j4v
+3kai
+3c9e
+2v59
+5cs5
+5jqb
+6as8
+5dw2
+1mq1
+4gw1
+4gj9
+6e2n
+4qlu
+5jog
+4iu1
+3tyq
+3r2b
+5alr
+5c4s
+3pa8
+2qq7
+4qn9
+2uwd
+1yye
+1oir
+1gpy
+6ert
+3d7z
+6dq8
+4lsj
+2wxm
+3pj2
+3nwb
+3qxc
+4hwo
+3afk
+2rk8
+6acb
+6g85
+1kna
+4pvx
+5hex
+3jqa
+5xms
+4an2
+3lcu
+3rdh
+2r3y
+4w7t
+6few
+4cd4
+2gg2
+4ga3
+6anl
+1yyy
+3r16
+4al4
+4uru
+2ovv
+4ya8
+4re3
+3bgq
+5lpr
+5fap
+5cei
+3v5q
+1maw
+4p4f
+4jx9
+4xqa
+3ghe
+3fci
+1zfp
+2l12
+4lil
+6eq2
+5vdv
+6b0v
+4l0i
+4zsl
+5yu9
+1p04
+1rnt
+2chm
+1nhx
+3ckb
+2vot
+5n1p
+2jke
+5ztn
+4j82
+3hxb
+3qvu
+1hk5
+4pyo
+3atl
+3jzi
+4ad3
+4b6r
+3g8o
+4kc2
+7upj
+5eyz
+6do5
+5wfj
+2j4a
+3kmx
+5oy3
+4o55
+2qi5
+4jfs
+4h3g
+2y68
+3fv2
+3ioc
+5jyp
+2gde
+4mm8
+4jjm
+5ccl
+5umy
+4n7e
+4f64
+3p9h
+4hz5
+4kin
+6cqf
+3feg
+3aaq
+3sl1
+1uxb
+4oee
+4i7d
+5ll5
+4qlk
+3s9y
+3t3g
+2p3b
+2jbj
+2x8i
+4ui3
+3po1
+5ji6
+3u90
+4lts
+4xrq
+4jzr
+5nkb
+1km3
+2qnx
+3myg
+2va7
+4ymh
+4nms
+5ar7
+2h3e
+4lp0
+2ay9
+5trs
+4fse
+6cdg
+6ful
+4x8n
+4w4x
+5m29
+4ocx
+3sov
+4fxp
+3ex2
+5uwl
+4wpn
+3bi0
+5nwh
+1vik
+4uj9
+5tg4
+5hrx
+3wz6
+1mm7
+5v37
+5eol
+4egi
+5n21
+3vhe
+3cj3
+4w9q
+2qtg
+4z93
+3jqb
+5f3c
+5ll9
+4zts
+5aei
+1o9d
+2iw9
+5b5g
+1g54
+3v5p
+2yk9
+1ez9
+4c6x
+2g24
+1m9n
+5f60
+1vzq
+1rt9
+5jq5
+3ha6
+2yj9
+4nra
+2h9t
+6afi
+3el8
+4kni
+5v79
+4fxf
+3ncg
+4x3s
+1dif
+4aa7
+4d1b
+5av0
+2clm
+4l10
+1exv
+2ltv
+6g1v
+3rwh
+5yfs
+5xkm
+5mjb
+2i03
+2v0z
+3udm
+3i3b
+1h1b
+6esa
+4ai8
+5ak3
+5j20
+5jjm
+3wto
+1std
+1jjt
+4kxb
+1jdj
+2chw
+3ll8
+5j79
+5adq
+4r74
+5n53
+3ljg
+2lsr
+3rk9
+1add
+5ja0
+5x02
+5cc2
+3u9c
+5d12
+1o6q
+1xn0
+5f20
+4lgh
+5etr
+3mj5
+4km2
+4f9u
+6el5
+1hmr
+5n9k
+3are
+1lv8
+5n99
+4gly
+1sh9
+4uns
+4cpu
+2or4
+2rkd
+4psq
+5wcm
+5vsc
+4nl1
+4um9
+4wa9
+3fxv
+2obj
+2ohm
+4zw3
+5jao
+6htz
+3n2p
+5dda
+2vwy
+4ge2
+5d7e
+4wx4
+3sgv
+5w12
+4gmc
+4dij
+1r5g
+4r3w
+5hn8
+4fk6
+5ggl
+2ks9
+1tq4
+2qu2
+4oyk
+1my8
+6mv3
+3kqs
+3gz9
+3eta
+1uv5
+2bok
+1b42
+5mlw
+5d29
+4qr5
+1r6n
+6f6u
+4pio
+6cee
+3mp1
+4pz8
+6fgy
+5moe
+5yfz
+5zwh
+4kb9
+2wmw
+1nax
+1o4n
+2znu
+5gvp
+3ma3
+3k15
+5qae
+1v48
+2psx
+4d8e
+1m5d
+2v2c
+2ql5
+3h0y
+3wpn
+2qa8
+1rs4
+5tgy
+2b7d
+4pl6
+2w8j
+3tdz
+2wxj
+5g2b
+4nrk
+3gwv
+1kne
+3kqe
+5khg
+4uiu
+4kyh
+4e93
+2qg2
+1a4m
+4byi
+5ngz
+4c1c
+6fam
+5lgq
+2hb3
+4c1h
+3ms9
+2pnx
+4qmw
+1lzq
+3aid
+3psb
+5w6r
+4c4e
+4jxw
+5ngr
+1utz
+3r7o
+3umq
+3hw1
+1tkc
+1abt
+3mbp
+3i9g
+1rwq
+4iz0
+6dh2
+4rx7
+4kwf
+2cn8
+3bar
+1cny
+1npz
+2bys
+4y6p
+4y6s
+5q0s
+2uxx
+6eaa
+5cs2
+4c1e
+4mgv
+2wxf
+5q1e
+5o83
+3r4o
+6fyk
+1x8s
+5b25
+4f39
+4tk5
+5nhp
+4lov
+1kui
+3art
+4d2v
+1bnv
+4kov
+6cyc
+4fkk
+6hvj
+2j4z
+2p3c
+1f0s
+3dy6
+5lz7
+3mww
+1hvk
+1x76
+2aez
+1bio
+1lhe
+3k48
+2krd
+4bbg
+5otc
+3kqp
+4esi
+2rku
+2kpl
+1erb
+2xx5
+2fl2
+4a4v
+1lqd
+1lfo
+3hf6
+3bm8
+1a99
+3zly
+6cwn
+3d8y
+5icz
+1hgi
+4qgf
+6fug
+3tn8
+6f3g
+5kq5
+3tz2
+1xws
+5uov
+2o9j
+3coh
+3h98
+1mes
+3ot3
+6f1x
+5gs9
+2hha
+1pfy
+4j3j
+4rn1
+4ze6
+3zpt
+4f9y
+4lbu
+2nno
+5v42
+2fyv
+4j8m
+6hjk
+4ha5
+3tu7
+1izh
+3o9d
+6g4m
+2r2b
+2xc0
+1g30
+3anq
+5uv5
+4ty8
+2vew
+3br9
+1df8
+3fgc
+1utt
+6cd8
+6f4w
+3l8x
+4fe6
+5adt
+5nzq
+6htt
+5k0t
+3pn1
+4ryc
+4nga
+4crc
+4xcu
+3f3c
+3obx
+6agg
+6cq5
+3w2r
+3dsu
+3zs0
+4wkv
+3w07
+1ntk
+4dk8
+3kgq
+4anv
+4ery
+4jzd
+3tjh
+4jwk
+4hyi
+4kne
+3u9q
+5ih2
+2h02
+4c6u
+1i8z
+4bid
+3lmp
+5ohj
+4j81
+5dey
+5k8n
+1pz5
+1shd
+3hhm
+5n0f
+5kh3
+3nba
+4u5s
+2ndo
+3shc
+3vid
+1ik4
+1al7
+1fq7
+2x4r
+4kz3
+6cen
+3s8l
+2aoc
+2xh5
+2p93
+4qfg
+5huy
+3ebo
+1agm
+1fv9
+4jnj
+4c72
+1vfn
+4xg3
+3hl5
+2xah
+5n58
+3mof
+3r22
+5o45
+1utm
+3p8e
+1kl3
+5o5h
+5lpk
+2zdx
+6q74
+3miy
+5bpe
+2vta
+1bnq
+6f90
+4myh
+5llm
+4poj
+1q6s
+1jws
+5xmp
+4ahv
+4e4n
+4qgi
+4o75
+3ogp
+3dk1
+4bdf
+2ybs
+4ght
+1e5j
+1veb
+4wrq
+5cqu
+1ow7
+5nsp
+2wtx
+2rqu
+5kpl
+3g19
+3gy7
+4bhi
+3rjm
+3f3v
+4h42
+4y16
+1o3g
+1utc
+3wzu
+6b5q
+1u2r
+3sx9
+1nju
+3drp
+6azk
+3r9d
+4awi
+1ce5
+5vd2
+5kzq
+2bal
+1y2f
+2xxt
+5hlp
+2y8c
+5auu
+1bwa
+6gbe
+2zv2
+3f7b
+1e4h
+6arn
+3ryw
+6fr0
+5wbz
+1c8v
+5kau
+1v41
+4oew
+5fbo
+3l9h
+5hi7
+3igg
+4hy0
+5enf
+1cwc
+5yjo
+2e92
+186l
+6epy
+3qk0
+6cq4
+2jkq
+3gb2
+5xmt
+2ybu
+3arz
+2o5k
+5c13
+2amv
+4std
+4gfm
+2fm2
+4pv0
+3qx9
+1hnn
+3bl2
+5dhp
+6c7i
+5pzn
+5ur9
+1pye
+3upf
+3ooz
+3zcl
+2vkm
+2jdl
+2uyi
+1usi
+1ezq
+4qz0
+3g5d
+3ry8
+2vnt
+5ehy
+1mqj
+5gs4
+5xg4
+3h59
+1bv7
+1br6
+1mns
+2ybk
+1bcj
+3ktr
+1e02
+1ykr
+5vja
+3r5j
+5edu
+3kga
+6b5t
+5f74
+5qaq
+1jpl
+5hmi
+3mjl
+1eei
+4n8e
+1hrn
+2o4s
+1agw
+4yz5
+4wn1
+3tc5
+2pjb
+6ea1
+5epy
+3vc4
+2i0g
+3ah8
+1ogg
+1onh
+1tmb
+6bau
+4ad6
+1a3e
+4drn
+1aw1
+5fl1
+2h2j
+4prg
+4ibe
+2yem
+3fql
+5zwf
+1x0n
+3pi5
+6ehh
+1cps
+5akl
+3g2t
+3vsx
+4dhl
+1hsg
+3vhv
+3ro4
+5myl
+4lyw
+3sl5
+3e92
+4o91
+3ipa
+1imx
+1u71
+5dx3
+4hvi
+1h5v
+3p7b
+5v8o
+2rg5
+5nx9
+4i8x
+4rcf
+4yrd
+4bcn
+4d09
+5f8y
+5mwp
+3n76
+3gba
+4lyn
+1unl
+3e6v
+4dce
+3rsr
+1nlp
+5dxg
+3rcd
+2cji
+5zwi
+3o9a
+2x2k
+4n5d
+5l4f
+5gx7
+5yqo
+4gzt
+3tki
+5dd0
+5ivt
+5mnr
+3asl
+2oc0
+3l3n
+5izf
+1i6v
+2y56
+4x0f
+2h4k
+3ivh
+4yy6
+1drj
+3hku
+3kfn
+6gmd
+2c3k
+1w5v
+5oha
+4myq
+3new
+1o5c
+4k5y
+5kbi
+5of0
+5dhs
+1y19
+3hky
+5nka
+1wax
+6h9b
+3i7i
+4b6c
+1h3h
+5xw6
+4jql
+4cra
+3d4l
+2nwn
+2zyb
+3rm9
+3q6s
+3g4i
+3zqt
+4bbe
+3olf
+6gxa
+5h4j
+4ljh
+4a4f
+4nw5
+2vi5
+3eqy
+4tpp
+1pi5
+4bt3
+3kiv
+1pb8
+4xyc
+1zd2
+2vur
+2k2r
+5j1x
+3pfp
+6g37
+3nsq
+6eq8
+2wi6
+1bxl
+5eci
+5jrq
+5ehr
+4qz1
+1nmk
+5zkb
+4f65
+5a7b
+5sym
+3wti
+1j1a
+4nwk
+5l8t
+5k0f
+1mqh
+1h3b
+1wxz
+3tt4
+5k0x
+2ez7
+5wbp
+4ddl
+2ygu
+3e7a
+1aqj
+5m7s
+2odd
+4eon
+3fq7
+3vf9
+1pxm
+4umt
+4rlt
+4f8j
+5dp9
+1ung
+4r5w
+5ais
+4btu
+4hyb
+3nyx
+3nif
+5hmk
+2qyk
+3tz4
+3qxm
+6isd
+6g97
+5ccm
+5u0z
+2wbd
+3aav
+5vzu
+2hai
+6evn
+2yga
+1p17
+5fus
+5elw
+4azt
+6ea2
+1xbo
+2yhy
+4u0m
+6b3e
+5hff
+3iu8
+1dqx
+4o0v
+5t1u
+4gs6
+2xei
+6gn1
+2pze
+2j87
+4hy1
+3udn
+5nmg
+2pgz
+1wdq
+4er1
+2l6j
+4cmo
+4gbd
+3itu
+5a7j
+3su4
+3r6g
+5ea7
+6eiq
+5qb1
+4o61
+3grj
+4xjs
+2x85
+3b66
+3le6
+5t19
+3itc
+5ett
+2esm
+3dc2
+3p2k
+3u7s
+3c6w
+4fcr
+2clo
+3d0e
+2jqi
+1ekb
+4ovz
+3soq
+1hef
+5os0
+4htp
+4tk2
+4j22
+4etz
+3gus
+3s76
+4pyv
+3c49
+1ktt
+2yz3
+1xnx
+1elc
+5i24
+6eif
+5t9z
+2qt5
+5h1e
+5ekh
+5vdw
+3fbr
+2za0
+5wej
+2yel
+4anx
+1rtf
+1r5y
+1pzo
+2ydw
+5kam
+5i13
+5exw
+2xyr
+3qs1
+2x9f
+2np8
+5ci7
+1m7y
+1fcx
+5ok3
+1hp0
+3l79
+4jq8
+4bhn
+1i9p
+4hyh
+1jil
+5f2s
+1bwn
+1xh7
+5kbg
+4bg6
+3qfy
+4ycu
+3lxk
+4w57
+4kxl
+5mk9
+6er4
+2rcw
+4jpc
+6h1u
+3iae
+3fnm
+3mxr
+3c2o
+5eva
+4kbi
+3ogx
+5owa
+6bmr
+4mgb
+5ka9
+2bza
+3sad
+4wj5
+4o1d
+4pnc
+4mzf
+1f9e
+4whl
+2imb
+4llk
+4ih6
+1vjd
+3stj
+3usn
+5tnt
+4i4e
+4hg7
+5dxe
+1hc9
+3dcc
+3d27
+5azf
+5c5h
+4m7x
+1ysg
+5nve
+4dcy
+2qrh
+1r9l
+3l0k
+4iuv
+5j31
+5ulg
+2wc3
+2jxr
+4qwx
+4joe
+4djh
+3sm1
+4prd
+5etn
+1ihy
+3ut5
+4gjc
+5c1x
+2oxy
+5can
+5hz9
+4uxq
+3nzi
+4hfz
+4mn3
+1jif
+1svh
+4zsa
+5y24
+5f9b
+4mka
+5wf7
+4o0y
+5w8h
+1t1s
+5w7i
+4o2e
+3g58
+4jmu
+3dg8
+5u7o
+5owt
+2oic
+3mt7
+5ka3
+2wgj
+1orw
+1vea
+1j01
+4yec
+6b7c
+5wg9
+2fv9
+1e55
+3cfs
+5nhy
+4x6s
+5ak0
+4q1w
+4o5b
+1b32
+2fzk
+2p3g
+1h24
+3qfd
+4xu0
+3pe2
+5ow1
+4uyg
+3du8
+4cby
+1mtr
+3bug
+4uj1
+4omj
+2y58
+1onz
+3esj
+1y2b
+1zyj
+1fig
+3cd7
+2hf8
+2v3u
+1o4f
+5w0q
+1ql7
+3kv2
+2wez
+2loz
+4zg6
+5f4l
+4mo4
+3mam
+2h44
+3que
+4y32
+5cwa
+5hyr
+4mnp
+3wix
+5tw5
+4dem
+5di1
+3zlq
+6en5
+4ogv
+2r6y
+4k8o
+6c7w
+6eqv
+3dx4
+5qa7
+2qlm
+4tn4
+5vm6
+4x5r
+3k23
+2q54
+2pe1
+4k2y
+4rfc
+6hqy
+4ie3
+2qbq
+5o07
+7std
+4mm7
+3axz
+1a85
+5etb
+4ylj
+2vpe
+2qln
+5zfi
+5wbk
+4bty
+1nvs
+5ftq
+3dyo
+2c9t
+2h6b
+5nz4
+5ar4
+1xfv
+6g6z
+4e1n
+2q2y
+2zwz
+2ozr
+3fei
+2xe4
+3nin
+4py1
+4j52
+4mrf
+1ngw
+1m7q
+4m8t
+4qhp
+1my3
+5fgk
+2ff2
+5qat
+3chf
+4kox
+2nsx
+5v1b
+1det
+4cq0
+2iwx
+4isi
+4bcg
+1pro
+4ebv
+2jh5
+4mm6
+4b82
+4m5m
+4pp3
+3v31
+4yt7
+3vp4
+3imy
+1rbo
+4b1c
+2w7y
+1k9q
+4erq
+5tzh
+2a3x
+5f61
+5orj
+3efr
+5g4o
+3el1
+5lkr
+1mn9
+5wdw
+4oym
+2iko
+1h39
+5ddd
+5nu1
+4qz4
+1o5p
+4i9c
+3wmc
+4ajl
+4zud
+3kig
+3wde
+6fdu
+4psb
+4bw2
+1hk1
+1pph
+3wut
+2ate
+3nw9
+5ceo
+1g2m
+1o3i
+5ald
+2hjb
+2cvd
+5za1
+4cnh
+4pnw
+4uwl
+5c1w
+1m0n
+2nsj
+4imz
+4x3k
+4zgk
+3pvu
+2fah
+6ex0
+6gz9
+1ke8
+3tkw
+2xfj
+4bo9
+4cqf
+5sy2
+3fh7
+5u4f
+1k27
+1msn
+5ow8
+5ekx
+4re2
+1dar
+6iil
+1bmm
+3tsk
+3tzm
+3ttz
+4ibd
+4zx5
+2c4w
+1qf4
+2xgo
+3bm6
+2zfp
+2uyw
+3h8b
+4acx
+3ouh
+4zy1
+1c1u
+3btc
+1azl
+4ivd
+1o4i
+5evb
+5dpx
+4k9y
+3wtm
+6mxe
+5ygf
+4mpe
+1o2q
+3ru1
+4qtb
+3qw6
+4yb5
+2hzl
+1b5i
+1dbb
+4gj3
+6bfw
+5y5n
+5jdi
+1kv5
+4der
+2xc4
+3s3r
+5lso
+5cu4
+1ysi
+1n2v
+4jje
+5akw
+5f3z
+6dl9
+5wgd
+3muk
+3uw9
+1o32
+1jq3
+2zpk
+3hk1
+2f1a
+2haw
+4psx
+4i54
+5d3s
+5m56
+2bkz
+5dxt
+1qw7
+4abu
+2ggb
+5fjw
+4k5p
+5als
+2xjg
+5eqy
+4g2j
+2a4g
+3hf8
+5em9
+4ztl
+2g1q
+4phv
+2ylq
+5tdb
+6dh7
+5kz0
+3rz3
+2vww
+5t4b
+5xs8
+5j7g
+1f4f
+5mev
+5cgd
+3vrw
+5t8j
+2o48
+4cgj
+4cts
+4cbt
+5ofw
+3q2m
+4ce2
+2qqs
+2j75
+4yw7
+5wip
+6afl
+6b16
+2cgu
+6exi
+3mt8
+4o24
+1z34
+3fvg
+1w6j
+3py0
+5nkc
+1os5
+3t70
+2nwl
+6dq4
+1n3z
+4rme
+5z95
+1tx7
+4yc8
+3t0m
+4e5w
+4f1s
+1hvr
+1xzx
+3fty
+6au9
+4eu0
+4d1a
+2p7a
+4z2k
+5ap7
+1sqb
+3wt5
+3eqb
+4o3c
+1rgl
+3r7q
+2ay8
+5enm
+2ha7
+5ak5
+6dug
+3wgw
+6bfn
+2fx8
+4rxc
+6bny
+6h36
+5ugm
+5trj
+5xpo
+1jmf
+3ugc
+4jhz
+3p44
+1zp5
+1v2o
+5tbe
+1o2r
+4wsk
+5vdk
+3rni
+3kc3
+4mi3
+4pb2
+5euk
+5ukk
+2k2g
+4a4q
+3m6q
+3btj
+4gtv
+5u0g
+4zh4
+3v2p
+3kqt
+4im0
+1ni1
+3wp0
+3vi7
+4tkn
+6bg3
+1o1s
+5d3l
+2qc6
+5lp6
+2br6
+5ku6
+5d3c
+5ny3
+5hjb
+4txc
+3waw
+5b1s
+4trz
+2pwc
+3nu4
+4x3i
+3djk
+3qw7
+3ivx
+6b8j
+1y0x
+1ros
+4rwj
+6bto
+3ztc
+1jlx
+5h13
+3l0v
+4ycn
+5qam
+4ke1
+3wv1
+4mep
+5tzc
+1bil
+4fl2
+4u6r
+4k6i
+4g2r
+4pg9
+2hzn
+3r8v
+5d9k
+4gg7
+3d9n
+6cct
+2c93
+1uxa
+3nfk
+5f27
+2w6n
+3k8d
+4pl4
+4b9z
+4uiy
+5fl4
+2bbb
+3b50
+1qon
+6ce8
+1ydk
+4jv8
+2bw7
+1li2
+4qag
+3hyg
+2qtr
+2cen
+4k3n
+2fes
+5kat
+1dzp
+3gcu
+4nbn
+4ctk
+1okx
+1nzq
+5fb1
+4r5v
+4b7n
+4det
+2xm2
+3exe
+1jje
+4wup
+3sjt
+6h8s
+6bx6
+5tqf
+3lpk
+3h1x
+4ps1
+3c72
+2dua
+4w97
+5diu
+2cbs
+4zy4
+4alx
+1y6b
+1tnj
+2vo7
+4qye
+4no9
+5ct1
+4elb
+5sz2
+4u0x
+4ztq
+2nnd
+1xom
+4u0a
+4abj
+3uh4
+1pop
+3hv5
+3zll
+4rak
+5u7m
+5tuo
+5ur1
+4iqu
+3kac
+1t7r
+4lqg
+4um1
+5mwa
+3djp
+4zeb
+3r91
+4o77
+4r5t
+4ijq
+4mxa
+3d0b
+5ld8
+3hdz
+4bxn
+3vb5
+5q0j
+3rx7
+1k9s
+6cjy
+4zjj
+5i3y
+2wos
+1bbz
+4rx5
+4nzm
+1m5c
+4djs
+5ect
+5ejw
+4fvr
+5vtb
+6f20
+1u9x
+1zea
+2zzu
+3rxp
+2z3z
+1oeb
+1gux
+4g8r
+1hk2
+1g4k
+5dfp
+5vc4
+4g55
+4o7d
+1nc3
+6cdp
+4gu9
+5gvn
+3l8s
+4bis
+4lwi
+2baj
+5wik
+5a7i
+4pnq
+2xde
+6e4u
+1b38
+3wzn
+5nn0
+4fl1
+3q6k
+6eab
+3s68
+2dq7
+1yw7
+4x8p
+5vgo
+2o9k
+4yo6
+4lv1
+5ose
+4ogt
+5e8a
+4k76
+2pr9
+1z3j
+2g72
+2l1b
+4gdy
+1lq2
+6g0q
+6cv8
+5tri
+1g7q
+1o4g
+4cg8
+2ohl
+1v2u
+4b83
+3wyj
+4hrd
+4kfq
+1no6
+5lax
+3egk
+3f7g
+4deh
+5tmp
+4rad
+3d78
+5l72
+3avj
+4rg0
+4qev
+5nt4
+5svk
+2w08
+3lxo
+3ljj
+4kif
+2jdo
+1v0o
+1k3n
+1rgj
+4zt8
+6dub
+1zdp
+187l
+6dkb
+3n6k
+1tnk
+2h2g
+4mjq
+4a9i
+4pre
+2gni
+3bgb
+4ddm
+2p99
+1mwt
+5nhl
+6b4w
+3sou
+3sm2
+5qik
+1loq
+4yqh
+5mi7
+4ynl
+2f9v
+1qbn
+2r2m
+3uod
+2w0z
+4ewo
+1fsy
+1tmm
+3va4
+6b0y
+2adu
+5dtj
+3k3a
+1sm2
+3ozs
+3o8g
+5g42
+4h3a
+2fwp
+6fbw
+5tzw
+1xkk
+3tdu
+3efj
+3as2
+2ity
+4zwy
+2nmy
+4amy
+5kad
+3qtv
+3ncq
+3p58
+6h1i
+5x13
+4d1d
+3zev
+6aum
+1ftm
+4dsu
+4hla
+2qoe
+5em5
+3u0p
+3dx2
+4kn2
+4ce1
+3np9
+3t2w
+4b7r
+5o9q
+1rth
+3eq7
+3i0s
+2pgl
+2yj8
+2z6w
+3v9b
+6fs0
+1dhi
+4hdp
+5svz
+4mz5
+1a1c
+4p0n
+3n7s
+2aow
+1j19
+4k3o
+1m7i
+3s9z
+1igb
+4o9s
+5xyx
+4mex
+5jy3
+4jht
+3hxc
+2c3i
+4itp
+2fl5
+4b12
+3o6m
+2vd4
+3tib
+5yj8
+1aze
+4yad
+3w37
+5e8z
+3pz4
+2p4y
+2xda
+3n4l
+1i1e
+1tbz
+2bxu
+1g6r
+4h5e
+3fuz
+1f2p
+1b51
+5lm6
+5gx6
+4wn0
+3h21
+3vd9
+1x39
+4bjc
+6d5g
+3nnu
+3d32
+3l9l
+5i3w
+4kqo
+5w38
+2p2i
+4ohk
+5l2i
+4jlh
+4whh
+3u6i
+2khh
+4gu6
+1snk
+1ogu
+2p0d
+1zeo
+1iq1
+4knb
+3lrh
+5a5r
+5iok
+5t6p
+4dx9
+2a0t
+1c5x
+1qxw
+3d83
+3pwh
+5g46
+4wy6
+1yw2
+1bm6
+4yhm
+1tka
+3qpp
+4g19
+1pdz
+4fnz
+5mby
+3zt1
+1y3v
+3tv4
+2qmd
+2phb
+1bzf
+1m2r
+6g46
+3hvj
+3mhl
+4w9c
+4imq
+5k4i
+3abt
+3vuc
+4oyt
+5qb3
+5yv5
+4n1b
+1pip
+5iu8
+3zke
+5f1z
+4qvx
+4zlz
+4qfs
+5ai0
+4op2
+3tfn
+5cnm
+6dge
+1htg
+3rey
+1o6r
+3w5n
+4l52
+4b76
+5qar
+4nj9
+2kaw
+4ezz
+3hkt
+5e1b
+3ufa
+4btx
+5mgj
+2ha2
+5cst
+5ea5
+5kgn
+3jq7
+1qy2
+4fpk
+3hq5
+1at5
+1hge
+5ty1
+4b3d
+1lyb
+5mrd
+4zt4
+4ovg
+4pp7
+2w5i
+6ced
+4elg
+5j71
+3i6o
+4mx5
+4pnm
+2g0g
+1x81
+4wkp
+4kxm
+1wht
+6q3q
+4gee
+2f70
+2wd1
+6aah
+4rfm
+2wvt
+3dx3
+5t4v
+1owh
+4k7o
+4z7n
+5lla
+5ehw
+3jsi
+1ohr
+4k3h
+5y8y
+1pu8
+2mg5
+3l5f
+3tfk
+1t31
+3t08
+4iho
+3i91
+5fdg
+5ntq
+5wa4
+2iqg
+5tol
+4tz2
+2d1o
+5ene
+4mc2
+6ax4
+4aoi
+1u3r
+4ngq
+3src
+5zzw
+1sw2
+6g2m
+5fwa
+2q7m
+1xn2
+5g6u
+1puq
+5ha9
+3s56
+3ckt
+2jaj
+2qlb
+1l6y
+4tzq
+5u9d
+4nzb
+3ccc
+3isj
+5kbr
+3u92
+5duw
+3ijh
+1uti
+5tw2
+2hd1
+4wuy
+1riw
+3u8n
+6flg
+2qzk
+5w6o
+2v2q
+6e86
+4e0w
+5jf4
+5a8y
+2nq6
+1tsy
+1rek
+2xht
+2gtk
+4kom
+6en6
+3zmz
+3efs
+4whr
+5j1v
+1ld8
+4o37
+5qa5
+4c9w
+2zdz
+1l5q
+3kek
+2olb
+4dru
+2h42
+5jnn
+1w0x
+6chh
+3av9
+3uzc
+1l83
+2osm
+5i89
+5wb6
+3vdc
+1b4h
+5e0a
+1nyy
+4bqg
+5otr
+5axp
+2w1f
+4z1q
+4q4q
+4o07
+3g0f
+4alv
+4jfx
+5kjn
+2jal
+4u0w
+2k0g
+3uwk
+3k26
+5iq6
+6hjj
+3o0g
+3iu9
+4xg8
+4xya
+1zfk
+3jvk
+5wxh
+3t0w
+5yg3
+1xpz
+4b60
+4j2t
+3ds0
+2l0i
+5y1u
+1pxk
+4da5
+1z6e
+6fi5
+4pry
+8lpr
+2ya7
+3zv7
+4bnz
+6bh5
+4x2s
+2vth
+3tvc
+5fky
+1i43
+5jau
+4qsw
+1m21
+4xsy
+2pvn
+5htl
+5o7i
+3skh
+5wic
+1fvt
+4rio
+3si3
+6dnp
+4c4n
+4ufu
+2ork
+5dkr
+3m59
+3tmk
+3btr
+5nps
+2oah
+5ywx
+4nxs
+3r8i
+1z1h
+4jik
+3oui
+4pzx
+5o1g
+5auw
+6cjv
+1t46
+1uwb
+4lgu
+5xvu
+4x3t
+5dtk
+3qkl
+3meu
+3ove
+5mw3
+2zcq
+4n7y
+4e4a
+2avs
+4e3l
+1hbv
+3tl0
+3g15
+6c5t
+4ayu
+4aba
+5ddb
+5mmg
+5k09
+3fj7
+4y3j
+3dxg
+5a6n
+5lvd
+3c2f
+1w51
+4m3m
+4x6k
+6h7b
+5gow
+5n6s
+2f18
+4kvm
+6d5j
+2qhn
+1ppm
+5y2f
+3fi2
+3vzv
+4r73
+5h5q
+5gic
+1g6g
+5j47
+4tmr
+4inb
+1w6h
+1inf
+5alk
+3u8l
+2bvr
+5bnj
+5nk4
+2c6g
+5tbn
+5ug8
+2wbg
+6bgv
+3fzc
+5mgf
+4ngp
+4akn
+3k27
+4gfn
+1vjc
+6ckc
+3bqc
+2k0x
+4o0a
+4m1d
+3cfn
+2ds1
+3wnt
+2ogy
+6d28
+6ffi
+6eu6
+2it4
+6h78
+3s0e
+3u1y
+3r04
+2nsl
+4knj
+5q14
+1e1x
+5cs3
+5hfc
+5zob
+4erz
+5ant
+6ft4
+3th0
+1ijr
+4lnw
+4pyq
+2r3t
+3ik3
+5mkr
+2gz7
+1joj
+1j17
+1c5o
+3avf
+3u0t
+1c85
+5w99
+6d4o
+2qic
+2tmn
+4cc6
+5q1h
+5k5n
+5n55
+3rxj
+3njy
+3srg
+4bgm
+4m0r
+5nxv
+5j1w
+4bbf
+4mot
+1tl7
+6gf9
+4ivs
+1bmn
+1bju
+1w4o
+2pmn
+1k6p
+5akz
+4ca7
+4joa
+1mfa
+4qqc
+5e2q
+1dgm
+2fxu
+3ejp
+1a1b
+5k0j
+3h5u
+5ha1
+2rg6
+4b77
+4hzt
+5fe7
+4pgd
+5uwm
+4rro
+4rr6
+4o70
+5cfa
+4ke0
+4ju3
+2wtw
+2qhm
+3m3z
+2vqt
+5q0d
+3gcv
+4fmn
+3s7l
+5kgx
+2n9e
+2vmf
+4x63
+3fc1
+1ok7
+4pzv
+5dd9
+4nru
+1udu
+6fmp
+5fxs
+4dk7
+4e5f
+5ml0
+3v6s
+4eeh
+5cjf
+5e1o
+6bw8
+3h2f
+1yly
+4avs
+1sri
+1avn
+3i25
+4fm8
+4jg7
+3fu6
+1g7f
+4l1u
+4dr9
+3ocg
+5wal
+4dq2
+1uz8
+4qjp
+2onb
+4hgc
+2lyw
+5xgl
+1ydr
+2r0h
+4q4s
+5j64
+4riv
+4cp7
+5d9p
+1il4
+2qaf
+1idb
+3ft8
+1hos
+2xnn
+5myv
+5ti4
+3gi5
+2wxh
+4eo6
+3spk
+1g3f
+5iy4
+5h1v
+1mem
+1rdt
+2c92
+1n7m
+5dif
+2py4
+4k9h
+4ehr
+1nfx
+4cst
+3h0v
+5z7j
+5ta2
+5nu3
+3k3i
+3gpe
+3zq9
+5cqj
+6axk
+4len
+4nmp
+3b25
+3ch9
+5djv
+4hxw
+4euc
+1hvy
+2jk9
+3ncz
+3bu1
+4bb2
+3wqh
+6alz
+3qch
+4jv7
+2jmj
+3nan
+5vds
+1oth
+2aie
+3dln
+3mv5
+5egu
+6dki
+5am0
+3vrt
+6dpt
+1ew9
+2w6u
+2ggu
+5jsj
+2xwd
+3zya
+5uwp
+2exc
+5e28
+5k32
+3g4f
+2xng
+4x5y
+6esm
+1gi4
+3d2t
+5ttw
+3ekx
+2uwp
+1klg
+4y2x
+2dwx
+5lrq
+1od8
+1ceb
+4ci2
+4m5i
+1o9e
+1nq7
+3ejr
+4wrs
+4eor
+1g1d
+5ofi
+2kce
+5eko
+4mhy
+1ndy
+2cc7
+2nnk
+5ybe
+1svg
+5aes
+4pkr
+5j5x
+1cka
+3n7h
+3pxe
+4kz7
+1tcw
+3v5j
+2xp8
+4bv2
+1gyx
+4i6b
+4oyb
+1ox9
+2wf2
+3zvv
+5f90
+1rdj
+4azb
+3o5n
+4uub
+3d1e
+1n5z
+6b1y
+2o7e
+3o9v
+4hf4
+6ety
+2qi6
+5jas
+3gss
+1c5c
+5l9l
+3ifp
+1czk
+4fi9
+1cs4
+5e88
+1mui
+5vp1
+3vbt
+3sh0
+4avw
+2asu
+1qf1
+3hjo
+1iiq
+2r2w
+1ztq
+2c8x
+3wtj
+4lys
+4lv2
+1u9v
+1dx6
+3mtd
+5ii1
+6d5h
+4nku
+1w1d
+1e37
+4pte
+3c7p
+4glx
+1hvi
+5f4n
+5f6u
+1y8p
+3kwb
+5ggj
+3gx0
+3ztx
+4uiz
+3ueu
+2vl4
+2wnl
+4cl9
+5aqn
+3ipx
+1i72
+3dsz
+3mhi
+4ap7
+2xcg
+7prc
+4e3o
+4int
+1zz1
+5bjt
+2xas
+4ny3
+5fi7
+1owi
+3qd4
+1ew8
+4hdc
+4ac3
+3h52
+5nzp
+1lbf
+1b6p
+2ylp
+3qxt
+3kyr
+1ksn
+3twp
+3ti8
+6ewe
+3t19
+3eu5
+4dvf
+2flr
+1oe7
+2or9
+1v2m
+5fjx
+1m2x
+2c80
+3nm6
+4nak
+5y12
+2vwn
+5chk
+4dv9
+5akg
+5x9h
+5m7u
+6axp
+1upk
+1j36
+1uv6
+3f2a
+1yet
+1ov3
+4kwp
+1eoj
+1yms
+4o4g
+3p0g
+3tsz
+2hrm
+4tz8
+5k4x
+3hzk
+5w5s
+4ekg
+6gh9
+5l26
+2lsv
+6eqa
+5fp0
+2zc9
+2jk7
+3h03
+3wb4
+1xow
+4hra
+4mk1
+1y20
+5yc1
+5lx6
+2er0
+5x4m
+4yih
+3mbl
+4a7i
+5koq
+3cx9
+4zom
+5sz6
+2vqm
+4yyn
+1syh
+1kwr
+6cyb
+3pm1
+2zym
+5fv7
+3shz
+4dmw
+6b5j
+1wn6
+5vd3
+1njc
+3ew2
+5ia3
+4a4e
+4a9n
+4iue
+2ewa
+1akq
+5ew9
+5n8e
+5a54
+4kow
+5w6i
+5mgg
+5oh3
+4hmh
+4f3i
+3kd7
+1w9u
+5da3
+4n6y
+5jid
+1juy
+1f0r
+5iw0
+4ui4
+3tkh
+2gc8
+5qc4
+3wzj
+5l7h
+2kdh
+1uyk
+3sv9
+3uex
+5os8
+4b7q
+5a6i
+5om2
+5lgu
+5uwk
+5u1q
+5unh
+5nw1
+4ck3
+1g42
+2e9d
+5k4z
+4l7b
+1so2
+6esn
+4pzw
+1ida
+3jzb
+1o2h
+5qby
+1nde
+6dne
+1f8d
+1nvr
+5ult
+1db5
+4fr3
+1g27
+5alo
+4o15
+4ryd
+4oar
+4zy0
+4pd7
+2aq7
+6etj
+2zga
+5luu
+2l8j
+2wky
+4yrg
+2v22
+2ndf
+1jsv
+4u6z
+5wdl
+5nxi
+6i8y
+4u6x
+4mw5
+4ufj
+3ith
+1wb0
+3arq
+3dp2
+1lst
+3fqh
+2ivu
+1s5q
+6ct7
+3pma
+2ay6
+5e90
+1re1
+4l32
+3g5k
+4bw4
+1pig
+2xne
+4qme
+5ugh
+1cpi
+3wd9
+1o5m
+5wfc
+4tun
+2y77
+5laq
+1ax2
+1b39
+3hrb
+5dah
+5iza
+3thb
+4bi1
+3arf
+4gwi
+6h1h
+5yzc
+5tg6
+3vap
+4lkh
+2q2a
+2fle
+2vwf
+1g2k
+2zz2
+5xhz
+4zim
+5fwr
+6g9i
+6bqa
+1g45
+2ao6
+3is9
+2qmf
+3wt7
+5vlr
+5fnr
+6cf6
+5z9e
+5t1m
+3uda
+3ur9
+3dp4
+6c7g
+5kmf
+4hev
+6d8e
+5acx
+5ouh
+2fde
+6c0n
+4wt2
+6cgp
+5zun
+5dyt
+4cwb
+2xcs
+4z1k
+4b11
+2i2c
+5ko5
+2ccc
+1xgj
+6ayt
+1ao8
+1c5z
+1b9v
+5toe
+4bup
+4pox
+1gnn
+3a4o
+5kqg
+5am1
+1i37
+5uln
+3gpo
+2g01
+3fx6
+5nba
+1ke7
+3my5
+4inr
+1jp5
+5eqe
+4muf
+5xhr
+2wmr
+3ebb
+4mwe
+5ap4
+5nwg
+4bde
+4x8t
+5tbp
+1nl6
+4oz1
+3hvc
+1flr
+3pz3
+4pkv
+4drp
+4awk
+2ww2
+4tw9
+5b4k
+4o3t
+4fev
+1o45
+4bcj
+3upz
+1qkb
+5nho
+2x8d
+1aq7
+5y21
+5i12
+3bsc
+4wki
+1ouy
+4jv9
+5o9h
+4ez3
+3dd8
+5h7g
+1sqt
+3pn4
+5aab
+2ha4
+4enx
+1bap
+6enm
+5za7
+3dbs
+4mvw
+3vrv
+2azc
+3mss
+3i60
+5hld
+1zkn
+2wxq
+4y87
+4m8x
+5f6v
+4fhi
+5auv
+4l7c
+3wcl
+3gqo
+5dhh
+5u7d
+6grp
+1xm6
+1lpz
+2wf3
+5cvd
+3zmj
+1g6s
+5eud
+3wyl
+4wh9
+2oq6
+2who
+2nmz
+2xiz
+4hbx
+6bee
+3ryx
+5ans
+5zvw
+4zwz
+3q96
+4v05
+2pyi
+5jf2
+1wbv
+6c7x
+4qsx
+5m6u
+1m0b
+3qg6
+3nyn
+1rm8
+3lhj
+3g86
+1ny0
+3zj8
+3o6l
+2vb8
+3p9l
+4bnx
+5uch
+3sv6
+4ynk
+223l
+5orr
+4o0j
+3rxm
+3rw9
+4po0
+4z84
+4j21
+5yjm
+5n25
+4bky
+5svy
+5gmn
+5u7l
+1f1j
+4aji
+2oj9
+3oli
+4r4o
+6ayq
+4ql1
+3f7h
+1skj
+2wfj
+1wc1
+2r3o
+3m5a
+4lwc
+4j5p
+3n46
+3ttp
+1jjk
+5vb7
+2pj0
+3dog
+2bve
+5vfm
+4l2x
+3uvp
+4wno
+4y38
+2c69
+6eol
+2zx9
+4q9s
+2alv
+3ip6
+3sio
+4xoe
+3sn8
+5y48
+5u7j
+2g9u
+5f3g
+1xh4
+4j79
+5ghv
+2i2b
+1gah
+3kr1
+5npd
+5mks
+1jwu
+4inh
+1tsl
+2wq5
+6hkj
+2yek
+1hpo
+2e9u
+3mwe
+6aff
+6e99
+5vlp
+2uxu
+4u5t
+2hwo
+4ury
+5byi
+4v04
+5e2s
+3lj3
+4bt9
+4jt9
+4rce
+6b5a
+4mk2
+4idv
+4bah
+5lj1
+3nal
+6d1m
+5kpm
+5sxn
+5doh
+3m6f
+3u81
+6bsk
+5nfa
+1gui
+5i1q
+4m4q
+4ow0
+5ur6
+5iu7
+6d55
+1bma
+5uxn
+1oiu
+4ln7
+2a5s
+5u5k
+5e4w
+6afg
+1oyq
+4xtv
+6eji
+2xdl
+3at1
+1njb
+4u0g
+1tou
+2pj5
+4q1a
+3avk
+6fmj
+2lto
+6czc
+3u8m
+2y5l
+3fjg
+5uwi
+6m9c
+5m3b
+3jxw
+4zqt
+5vc6
+4bj8
+5knx
+3pix
+3e6k
+4f7j
+3koo
+5n3y
+3jdw
+4pl3
+4nmo
+1ody
+5tpc
+6bsm
+2cm8
+5gn7
+5hu0
+1yq7
+2gga
+4wy7
+2brn
+2fmb
+5h14
+4zsp
+3nus
+5n8t
+5x5g
+5yls
+5u3b
+4x14
+4puk
+5wa6
+2y2h
+4nmt
+3uol
+4jzf
+4um3
+2wi2
+5k7h
+2ccu
+5x74
+3ddb
+1xd0
+4mm9
+1ulg
+3coz
+5epl
+4ear
+3vzg
+4qac
+4l5j
+1ai6
+3d8w
+4mi9
+2wtv
+5iee
+6ge0
+4q3r
+4qb3
+5cdh
+2vip
+3pz2
+3f34
+3rin
+5n2t
+5oqw
+3p4w
+4n5g
+2pvl
+1b0f
+5a3u
+4p6x
+3rsx
+3rcj
+5i9z
+5ai6
+3c4f
+1o4o
+1sje
+3alt
+5f41
+5yid
+3bgl
+4rdn
+4hso
+4ttv
+3ok9
+5y0g
+5twx
+1wug
+4a6c
+3gxt
+4lzs
+2zx7
+3jvr
+4as9
+3f1a
+4cd6
+4r06
+5h5s
+7gpb
+5w0i
+4wp7
+5e3g
+2ydt
+3hec
+4abb
+5ji8
+4olc
+5ieo
+5i2z
+4zy5
+4qga
+3t82
+4dff
+2wr8
+3i7b
+3v4v
+4gw8
+2hxl
+4bkt
+2wj2
+4xwk
+1xk9
+2fgh
+3tv7
+1ajp
+1tps
+3ag9
+3ls4
+5wg3
+1cwb
+3mea
+1bjr
+5auz
+2y67
+1q9m
+2i3i
+5ivj
+2vd1
+1p2g
+4l7n
+5efj
+4jzi
+5mt0
+5vlk
+3rt4
+3zmm
+2wn9
+2a25
+4hmq
+5jim
+4zyy
+4bi7
+1f3j
+2b52
+2xkd
+1yys
+3vtr
+3zrl
+4kao
+5fnc
+1ke5
+6hwz
+4d0x
+4mzs
+5mi5
+4a6l
+4eg5
+1sgu
+1w11
+3dkg
+1s19
+4wyo
+3ppp
+4rlu
+2jjk
+5ovp
+6cd9
+3qio
+4wrb
+5h9p
+5na0
+1wbt
+4ael
+5eu1
+3nw6
+1c5s
+1bl4
+4ual
+2vsl
+5vil
+4o7f
+4dhr
+1pw6
+1b5j
+3iad
+3rwg
+5wev
+2pjc
+3vi2
+4xum
+2zmm
+3fv3
+3ukr
+5myo
+3lbl
+4ppb
+1q0b
+4abi
+6bfx
+3kmy
+1q6k
+5ygd
+4z90
+4omk
+5bqh
+5lrj
+1ta6
+5jzn
+3ad8
+5izu
+5ief
+4frj
+2pre
+4ty6
+5nut
+3zps
+5e2v
+4k43
+5e7r
+6ftz
+3rjc
+4uvc
+3uix
+5up3
+4i31
+2p15
+1ag9
+1gu3
+6cw8
+5ei8
+4psh
+4i71
+6f9v
+4yoz
+6hu3
+5e2m
+5ap5
+4fob
+1uod
+3e63
+3mke
+3io7
+4whz
+4mt9
+4x21
+1ppi
+3t2p
+5q0g
+5jjs
+5icp
+4g1f
+1csh
+4mwr
+5al1
+4qxq
+3o9c
+2wxo
+4xmo
+4ii9
+5mlj
+5hm3
+4wr7
+3iqv
+4xjr
+4w9w
+5fhm
+5o2d
+5osk
+3bv9
+3i7e
+4kz4
+2oz7
+2vzr
+4d4d
+6eog
+1sfi
+5ehq
+5uy8
+4u0d
+2l11
+5v2p
+2vtm
+5ef8
+1zgv
+4b2l
+3g32
+4q4e
+4x50
+2oax
+4r8y
+4c61
+1w2g
+5er2
+6b2q
+3up7
+3r01
+5izq
+3l6f
+3p9m
+6ey8
+4b74
+6fyz
+5g1z
+3t4h
+4wkc
+2o4j
+4bdj
+5mek
+4gvd
+5qil
+5go2
+4m3f
+2e2b
+4bi2
+5l0h
+5ugb
+2ynd
+2wzz
+5wyz
+5may
+4eev
+4kiw
+2y7i
+3tcg
+3s71
+2qcd
+1vru
+1h00
+4uat
+3nyd
+3bkk
+4n9d
+1bim
+5zt1
+5ml2
+3fpd
+6h7o
+3p8z
+4ewh
+4uva
+5a3q
+4dai
+5o55
+3f69
+4puz
+6f05
+3g35
+4qij
+4dxg
+3n1c
+4dt2
+5u2e
+6c7j
+4cj4
+2w7x
+6ccq
+4qsv
+5upz
+5iu4
+4x0u
+3ffg
+1y6q
+5yba
+2f35
+4gsc
+1jt1
+4qvl
+1mwn
+1jeu
+1c3b
+3ual
+1rdn
+6dh1
+5iub
+5mav
+3t83
+3iof
+6cwf
+2fpz
+5ml4
+5f2f
+1fpc
+7kme
+1erq
+5t9u
+1lhv
+2qlf
+5jmp
+4s3f
+6hm6
+2qt9
+4p4i
+5vih
+3mnu
+6djc
+5db0
+4ca5
+5mar
+2w77
+2avq
+5eqq
+1tsi
+2vw2
+2ggd
+6f6d
+1f4g
+3ndm
+3l17
+5fni
+1c3r
+5trf
+1pau
+3vf3
+6ewk
+4re9
+4li8
+1n4k
+4l7o
+5kbh
+2gj5
+2oh4
+4wht
+1joc
+1qxl
+6elp
+4qge
+2yfe
+3g8e
+1cea
+6czu
+2fs8
+4mk5
+4i60
+5h5f
+5tcc
+4dpy
+3fh8
+4h1m
+2anm
+1uom
+3oim
+6ezi
+3da9
+3r17
+1l6s
+2uzv
+4e1e
+3d28
+4bcd
+5kez
+1bn4
+6biz
+4lwt
+3jpv
+5fsm
+4mds
+4e6c
+3o9p
+2ylo
+5fyq
+4hxq
+3mj1
+1zm7
+1s63
+4nk9
+4qmm
+3g7l
+4zyw
+2q8i
+3gfe
+5ih6
+4owo
+5o5f
+4x7q
+4wy1
+1qbq
+5du6
+1oe0
+1g2a
+4iie
+4x5p
+1fhr
+1ntv
+1o5f
+3os3
+2cbz
+4arb
+3lxs
+3lxg
+2g94
+4gv1
+5n87
+1gz9
+1b6l
+5kcx
+2xni
+3ig6
+1p4u
+3r9h
+5n2x
+6aud
+5nee
+1bnw
+6bo6
+3kaf
+2fvd
+2xaj
+4a95
+5iv2
+3n2u
+4q6e
+1i2s
+2f34
+6evo
+5ksv
+2e9n
+5cgc
+1m51
+1d4h
+5ncy
+5kls
+3s2p
+4o09
+1zaj
+1f0u
+4k3l
+4tww
+5f88
+3fcb
+4arw
+4o9w
+2ay7
+5dcz
+5dqf
+3ttj
+3lc3
+2xg3
+1uml
+3qt6
+4oty
+4wzv
+1hxb
+5l87
+4ah9
+5ma7
+3npa
+4hs8
+4qgd
+5zkc
+4d2t
+5elz
+1jk7
+5cuh
+3jqg
+5w4s
+4n7m
+3tct
+5ie1
+4n1t
+5aqh
+5oku
+5nhf
+2yge
+4gs9
+6bu3
+5vqr
+2am4
+5wr7
+4j58
+4zsr
+1yon
+2bkt
+4buq
+3u7m
+3u51
+3w55
+2gfa
+3fum
+4hyu
+4pra
+4kyk
+4gzp
+3ekn
+3v2w
+2pog
+2bb7
+3qj9
+5d3t
+2w67
+5fs5
+5qaf
+4crb
+6fe0
+3zmq
+5t1t
+2xm1
+4oaz
+6e8k
+2fys
+5egs
+4aac
+1akr
+2wi1
+2qbs
+5h2u
+1lke
+3smq
+5eds
+3wmb
+4azy
+3cz1
+1s39
+1n94
+5lc0
+5wg5
+5ttg
+1ndz
+6gmx
+4cfe
+3fsj
+1cim
+3kry
+2f2h
+2g9x
+1o47
+6aqf
+4knn
+1t29
+4k5z
+2ycf
+5ijr
+5kks
+5tg2
+4yxu
+4i7c
+3ftv
+1ke3
+4bib
+3coy
+5orw
+5bns
+4bs0
+3igp
+2uw4
+4avj
+3qcy
+1yqj
+3a73
+4uu8
+4h3f
+4rwk
+5j5r
+2rkm
+1w14
+4edu
+3e73
+5o1s
+4ngn
+1mnc
+2ypp
+3wq5
+4j74
+3rz5
+5upe
+4eh6
+1c84
+4fcf
+1sc8
+3agm
+5w10
+1gzg
+4qvv
+4qp6
+4why
+3ip9
+5he0
+4yv5
+3ril
+4uyd
+1nu3
+3ozr
+5qin
+4utv
+5x9o
+1okv
+5m4c
+4xhv
+5ll7
+3cyx
+4gm3
+2w78
+5q1d
+3bym
+5aqo
+1ga8
+1p5e
+4c4f
+2mps
+4cxw
+4w9f
+3lmk
+3sz1
+4tya
+4qjx
+2v85
+5bw4
+2ael
+4pv5
+5osy
+4m8h
+3i02
+6fs1
+3twr
+2vt3
+4rrn
+1h5u
+2qoa
+4lk6
+3qtu
+3k5d
+2qhy
+5jmw
+3bun
+1ghy
+4l70
+6cd4
+4ymb
+6bh2
+5h0h
+1ppx
+3r4p
+6gnr
+2x6k
+3f9n
+4oys
+3rf5
+3hub
+4fcq
+5o1i
+3ohh
+4mqu
+4gw5
+5m4f
+2r9w
+5yea
+1xp1
+5lm4
+2csm
+5t2g
+3n1w
+3t8s
+5nhz
+4zk5
+1e6q
+4cg9
+4bf1
+2ycr
+1c50
+3hd3
+4ehe
+5tku
+5wle
+2r64
+5l3j
+6ft8
+5ti5
+4tyt
+5lma
+3oct
+3zt2
+5ai4
+3zh8
+3fl8
+3old
+3fqs
+4xct
+4mjp
+3g2y
+6g8n
+5kkr
+1fjs
+6dxg
+4cxy
+3che
+3qfv
+1r5v
+1pr1
+3nsn
+4p1u
+3szb
+5u11
+4o0t
+3ux0
+4q4i
+2nww
+2qp8
+4rsp
+2p0x
+1fsg
+1u0g
+4djx
+6cco
+3opp
+4lg6
+2clv
+1hps
+6bhe
+6bir
+5h09
+3kwj
+2bed
+3e51
+3msk
+6axj
+3udd
+1zvx
+5u6d
+4qnb
+4eox
+2rly
+3ob2
+3osi
+5m7t
+5vqy
+5ei6
+3mbz
+4f7n
+3ipb
+1w6r
+4zx9
+6gy5
+2hh5
+3nmq
+1vjy
+4hy5
+4ts1
+2qcg
+5lwd
+1kjr
+3oyn
+3rbq
+3ujc
+3ekp
+3p7c
+4urv
+3gws
+3fw4
+1ow8
+4pov
+4k4j
+4m0f
+1ow4
+3qaq
+4ovf
+1ps3
+4gw6
+5a4c
+2ovx
+5q19
+1gwm
+2kwn
+4uce
+4y2p
+2q2c
+5lgn
+5oxg
+3i4b
+4qmo
+2wyg
+5eay
+2b4l
+2xd9
+3sus
+5zu0
+3gjt
+2b1v
+3ctq
+5tzz
+1h28
+4b13
+2q15
+2aac
+3kw9
+3qox
+4wku
+5wqk
+2fjp
+5eij
+4mga
+5ak2
+5dp4
+6f8u
+4inu
+4nrc
+5drt
+1tng
+3vhc
+1xh3
+1toi
+2lya
+3mpm
+4k55
+1qb9
+5hlw
+6ej2
+4i33
+4cwp
+2fvc
+3r02
+6ezh
+4muw
+3d91
+3img
+4op1
+4s3e
+3qxp
+4f5y
+4qmv
+5u62
+4qtc
+6tim
+4h2j
+1x7e
+1b6h
+3opr
+4gjb
+5u5h
+5v3o
+5l44
+2c6i
+4ezr
+5u4e
+1fta
+4z2j
+3n3j
+5eg4
+2yiq
+4jmx
+4q2k
+2x2r
+2fzg
+1ajx
+2yki
+3ccw
+5u0e
+5j32
+1y2e
+6gwr
+6gu7
+5dxb
+1hyo
+3lc5
+3arw
+3rz1
+5kr8
+5jv2
+4xpj
+4l2f
+3vi5
+5etj
+2a3w
+3bra
+1h79
+6ed6
+5w85
+6mub
+5od5
+5ejl
+3ubx
+2c6k
+4rn0
+3pdq
+4unp
+5mng
+4w5j
+5afn
+6h0b
+5yc4
+1ahx
+2jst
+1o3h
+3rz8
+4na9
+1f0t
+4eym
+5vew
+4k3p
+4zs9
+3b9s
+5wcl
+4mmm
+2l65
+4lph
+3frz
+3rz9
+4kba
+5a8x
+5eps
+2wor
+5f1j
+2oxd
+3ldq
+4g0k
+1u32
+5gr9
+4fl3
+3uxl
+3fud
+4x68
+3k2f
+5lsg
+1h23
+3zrm
+5v6u
+4lke
+1o4r
+5mw4
+4fcb
+4qp8
+1htf
+4q0k
+4yc0
+3ebp
+3q4b
+5dh4
+1bqo
+3sn7
+4waf
+3gst
+5ecv
+1hvj
+3m3c
+4ocv
+2ewb
+5h19
+3ryz
+3a2o
+4ea3
+4wx6
+1i32
+4lge
+3cwe
+2hmu
+1h0r
+5m9w
+3jwr
+2q70
+4iu0
+1tl3
+2gqn
+3bc3
+4gk7
+2wd3
+2x8z
+4ybt
+5dms
+1nhg
+4mw1
+3cgo
+5v4q
+4x6n
+3bh8
+3f6h
+4au7
+5i3a
+3l6x
+5d0j
+4c1u
+6fzx
+3ook
+5mt4
+4hzx
+3udp
+4g8o
+4mrz
+4tyo
+4umj
+4rfz
+1rxp
+2zyn
+5t54
+6bh3
+4cd5
+4hlk
+3t0x
+2az5
+5dy5
+1ft4
+5org
+5to8
+6eqx
+3b2q
+6ez9
+6aox
+2y5f
+4x7o
+4y4v
+5oae
+1h25
+5d2r
+2xch
+5ndd
+2cf8
+2yi7
+4nzo
+3mct
+5dia
+1iih
+3ime
+3tll
+2jle
+1h26
+3l7b
+5v88
+1o2o
+3e90
+4j46
+3dp9
+5ngb
+5jh6
+3rpy
+2vfz
+5xn3
+1e6s
+4xx4
+3ppo
+4qfo
+1o3d
+3s43
+6ar2
+3cyu
+1qf0
+1ow6
+5eml
+4amz
+5ye9
+4mti
+6f29
+5mk1
+1qsc
+1uvr
+2off
+4whq
+4d08
+6gu3
+4b6q
+4agc
+4uu5
+3blt
+2zb1
+1g37
+3krd
+3lm1
+1noj
+1t32
+3i6z
+5kbq
+1gwr
+4k6v
+2vba
+3zrc
+4nmx
+4mbc
+1ie9
+3bgz
+2k62
+1mqg
+1qy1
+1mau
+2x7u
+3iqq
+5h8e
+2aig
+5n9l
+2g5p
+4fri
+4q3u
+5flo
+4bo1
+5tyr
+5x26
+2drc
+4bbh
+3gkz
+5e6o
+4qy3
+1nq0
+5e8w
+2gu8
+1w9v
+5uci
+2veu
+5hvy
+3ewu
+1jm4
+5yas
+4ojr
+1h0w
+5eou
+3as0
+4e3g
+4euv
+2wly
+3lir
+3znr
+3q3t
+6ap8
+3g3n
+3c52
+3g2w
+4iku
+5bry
+4wvu
+3neo
+1abf
+5db2
+1sjh
+3hc8
+2liq
+5dtw
+5dhu
+1qhr
+2pl0
+4wcu
+3exo
+1jvp
+4mrd
+5jxn
+2yjc
+2o9r
+5en3
+5hpm
+4qf8
+3k05
+3h0q
+2a4l
+4xue
+3ewc
+1dtt
+4yzm
+4q4p
+3d9o
+3s8x
+4mq2
+6ek3
+5tgc
+2hds
+5fwj
+3kme
+1zog
+6ce6
+4c6v
+1xdd
+5bvf
+1dud
+3lzu
+4c1d
+2er6
+3f48
+2xuz
+5fto
+4zei
+3c39
+6fqu
+4fc0
+1bzc
+3pj8
+2auc
+4kp0
+2y7p
+4ono
+3nc9
+4bjb
+4qo4
+3mz3
+5edc
+2xzg
+6bqj
+1ga9
+4oq6
+4b80
+5ee7
+1bcu
+4y8z
+5ka1
+5lpd
+3qi4
+5vii
+5ele
+2ew6
+1s9v
+4bqw
+2x6w
+5mnc
+5eni
+2xi7
+1wqv
+2b8v
+6f92
+4jve
+3oy0
+6h41
+4zx4
+4q15
+6czv
+2wti
+5j9l
+1o5r
+5n1v
+2bq7
+4fic
+2ole
+3umw
+1a1e
+1ax0
+4ux9
+6bhi
+3pj3
+5m55
+2uw6
+2bqv
+2ea2
+5dl1
+6g7f
+5o8t
+5nze
+5fkj
+5t1l
+4wj7
+3ibi
+5h22
+4bgh
+2j9h
+2iyf
+5m28
+2v0c
+6ez6
+5b6g
+2q3z
+4des
+3pcj
+5y0z
+3u6h
+3dtc
+3cr4
+2igv
+4qaa
+1yhm
+3n35
+5ylj
+5dls
+5h1u
+4qw4
+4hv3
+4jfi
+2go4
+2m0o
+5vzy
+2h6t
+2qnn
+2aqb
+1zd5
+3pxf
+1p1n
+1bnt
+2hm1
+4z46
+5e2n
+2p8s
+1h2k
+3uh2
+1b2m
+5eiw
+4bfy
+5csx
+2z92
+2rjp
+5o7n
+3d3p
+4ban
+4l0l
+3zo2
+3zhx
+2hah
+3dx1
+4dkt
+3sz9
+3n0n
+4l4m
+6asz
+3hv4
+3imc
+3eju
+5t6z
+4jaz
+1gjb
+5fl0
+3r9n
+3sw8
+5ggp
+1btn
+5j8u
+1x8t
+3bva
+2wtc
+5lom
+2vwc
+6bw2
+6m9f
+4n1z
+3lzb
+1fwe
+2w16
+2i6b
+5ytu
+4j8r
+4je8
+3t60
+2n9x
+6f8t
+4lwu
+6ebe
+6ajj
+5wap
+5a6k
+2xp7
+3ql9
+1q7a
+5dkn
+2gnl
+3rwc
+5npr
+2e9c
+1ajv
+6e4f
+1o7o
+5alf
+2xys
+6h5w
+2jfz
+2xdx
+4bda
+3rxd
+1f57
+9icd
+3zpr
+4ciw
+2zcr
+2w9r
+3hnz
+4r1y
+3olg
+5lce
+1kwq
+5lrg
+5f0c
+3u8j
+1bl6
+3amv
+1ej4
+4ht0
+3u6w
+4q09
+5jss
+3qlc
+2b1i
+5cls
+3eyf
+6eln
+5mrh
+3nox
+2qe2
+3u6j
+1qka
+3k5u
+1ws1
+5ew0
+6ft9
+3kr4
+5iug
+1ajn
+3zn0
+2i3z
+1y3x
+2r9s
+5vpm
+4occ
+4nym
+3osw
+3hy5
+4j86
+4dfg
+5f1v
+5l4e
+2qve
+4erf
+1wss
+1ajq
+5h5r
+1ym2
+1hee
+3rq7
+1dxp
+1vgc
+3huc
+5fqb
+4mue
+1qbs
+1ujk
+6fa3
+4xo8
+4ww8
+4qxt
+3in3
+2wxp
+5j3s
+2h1h
+5d7a
+4lhm
+2rjs
+5nvf
+3l0n
+5b5f
+4oow
+3ej5
+5fum
+1li6
+5swg
+6eum
+3fas
+5klr
+2vv9
+3v5t
+2byr
+5nr7
+3suf
+2xb9
+1f4x
+5v86
+3kl6
+1qng
+2a8g
+2j62
+3zi0
+1h6e
+3ap7
+3zj6
+4hp0
+1bmk
+1nj1
+5xp7
+4iwd
+1if7
+4ia0
+3ms7
+5yjk
+5j7f
+5aba
+5ake
+1h2t
+4mzk
+4ezj
+4an3
+5jfp
+5v24
+2xmy
+1yqy
+3aox
+4yjn
+2i0h
+4lwe
+4frs
+2hwg
+3o4l
+3ddg
+4cu8
+2zfs
+5wfd
+4lnb
+4j17
+4qyy
+2r3j
+4eoi
+3pdh
+1y3p
+4g5y
+3v4t
+4okp
+5meh
+1xpc
+3avl
+6gzl
+2pqc
+4g0y
+3wab
+5iop
+2ax9
+3bex
+3cdb
+6afc
+1v0m
+3ksl
+3fvl
+6c7d
+1t2v
+5bot
+5yto
+2wed
+1jn2
+4k4e
+1a94
+3ozj
+1j81
+3e93
+1q9d
+4idt
+4k2f
+2whp
+3snc
+4otg
+2lty
+5bnr
+6c7r
+3as3
+4ge9
+4xg7
+5dw1
+3ivq
+2qrg
+2xdk
+1gno
+2gg5
+1d4y
+5dqe
+3rwi
+4x8o
+3el9
+4kup
+3eks
+2kff
+1w5y
+2flh
+2vwv
+3rvg
+4oeu
+3u4o
+2pv3
+4i0t
+3aru
+4yv0
+5ura
+5uv1
+3vjc
+4bam
+5vc5
+2ito
+1l5s
+3adv
+3oe6
+5mnb
+5gvm
+1r1j
+2qe5
+2rjr
+3wiz
+4cfm
+1gz4
+3c1n
+6ma3
+1z9h
+6g98
+1qm4
+4j24
+3pju
+5cqx
+1wvj
+2yde
+4mqp
+2np9
+4zyu
+3ua9
+3fc2
+2xp5
+1umw
+5jer
+3dcv
+4aft
+4rhu
+3ahn
+4ytf
+5os2
+5el9
+3gjw
+4oth
+1lkl
+6ela
+5vkf
+2oz6
+4q1f
+5usz
+2xfk
+3n51
+4fxq
+4oc3
+3ggu
+1hiv
+4d2s
+2fkf
+4oz3
+6fkz
+3lhg
+1jyq
+3uvw
+4f6s
+2qk8
+5jat
+1lvu
+11gs
+4cwt
+4gz3
+5vkc
+1wu1
+5m17
+1rne
+1l2z
+3s0d
+4nxq
+5iu2
+1x78
+6bc9
+6drx
+5juz
+4qvn
+5i7x
+5o7e
+4ktc
+1fl3
+1enu
+2clf
+4zs0
+4llp
+4djy
+5nrf
+4z0k
+4k8a
+6cdc
+6bin
+4e1z
+1a61
+2q64
+3s0o
+4z0u
+1ii5
+4lk7
+5ubt
+5m1z
+1t4j
+1g7v
+4q1n
+3l08
+5d7r
+2emt
+4y2b
+4j0t
+2zof
+6ecz
+4gb9
+4jfl
+2ow3
+1ibg
+3fyz
+5g5z
+7lpr
+4g8n
+4bie
+2yir
+4yps
+5jdc
+3p4v
+2fum
+5c4o
+1kcs
+4r3s
+1cj1
+6f09
+4ce3
+4ft2
+2pcu
+4mo8
+2g2r
+4lxk
+4utn
+1stc
+5nxp
+5t52
+3iod
+1h01
+4deb
+4jjs
+5ts0
+3su0
+4yhf
+1y4z
+3heg
+2g70
+3p2h
+1sqq
+1swk
+2f4b
+1igj
+3fcq
+2pja
+4ase
+2o63
+2xyf
+4j0v
+5ka7
+5kzp
+3oob
+5w6t
+1h2u
+5a00
+4zt3
+2cer
+1w96
+2pzi
+4qf7
+3c88
+4y67
+4nrl
+4du8
+4qtn
+3sue
+6fhu
+1qs4
+5u6b
+5edb
+2p2h
+5gtr
+4hbp
+3tia
+3cbp
+2cgw
+4bio
+3gwt
+5t8r
+6fo7
+1g9t
+3kr0
+1gt3
+5c83
+2r9x
+3f8c
+5nk7
+4cfu
+5e2r
+2wf1
+3uil
+5fl6
+1ms7
+4csj
+4yat
+4f1q
+4lm1
+4oc0
+3hj0
+2xwe
+3ms4
+2jh0
+3oad
+2y2j
+4bd3
+6h7n
+5exm
+1ctu
+5l3f
+2ajd
+2c94
+3g3r
+2zo3
+4dro
+1uou
+6bsl
+1uy7
+3zyb
+3t0b
+4o43
+5q0l
+3wch
+5orb
+3kgp
+3zeb
+4pyy
+4b0j
+1jq8
+3u4w
+2xyt
+5jhd
+3hmp
+4dxj
+5hg7
+3zlx
+1zkk
+1gny
+4ju6
+1il3
+4daf
+4o0x
+5ygi
+6gla
+3mtw
+3mg0
+3d9l
+2vcj
+5edq
+4bci
+4e4x
+2qdt
+3wuu
+3udv
+4dkq
+4bfp
+5i2e
+3q1x
+1qaq
+3rm8
+4ahu
+1gx4
+3wi2
+4de2
+5g2g
+2wuu
+5nud
+4hcu
+3k37
+6prc
+2e7f
+4bb9
+1amk
+4l1a
+4lqy
+4a16
+5wkf
+4qsm
+3sxu
+3uqg
+5lif
+4azf
+5ntt
+3vsw
+5fos
+1qin
+3e7b
+1ll4
+3vjm
+3ny3
+6aqs
+1k6c
+4ngr
+6fc6
+2gnj
+4zy6
+3u8d
+2aou
+4b0b
+5t2p
+5efb
+5u0d
+5vjp
+5bui
+5e13
+5f5i
+4ivk
+7hvp
+3d7k
+4vgc
+5g1p
+6b1h
+2br8
+1cqp
+4w50
+4ehv
+5kqf
+2yfa
+3ws9
+4xkb
+5mrp
+3str
+5nkd
+5mpk
+5kqy
+4lw1
+1avp
+3qdd
+5xxk
+4oks
+4hxr
+3ur0
+4pns
+4i5m
+4hgl
+2vtd
+2gph
+3rme
+5fqp
+5hfu
+2iok
+4yux
+6g1u
+2hvx
+1n0s
+4l4v
+3s1y
+1g7p
+2rr4
+2x00
+2b55
+4wmv
+4kc1
+2x39
+5joh
+2w0p
+4az6
+1o0f
+3haw
+5w44
+2xhr
+1h8s
+4ge1
+3phe
+1evh
+6b30
+1g35
+2qfo
+5gjf
+6bt6
+2gg3
+5trr
+5ly3
+5eh0
+5ioy
+1xt8
+3c3o
+5e74
+3acl
+5abh
+5fol
+5ddf
+3qxv
+3iwy
+3ttm
+5n18
+6beb
+5nxw
+5lxb
+4ra1
+4muk
+4mm4
+4hs6
+5ncq
+3zmg
+4hcv
+4i72
+4pee
+1qtn
+4awg
+4rfy
+5dyy
+3ud8
+5itf
+5cuq
+5f1c
+5hey
+2ko7
+3oil
+5tks
+3t3u
+5o3q
+4k72
+4oma
+3hv6
+6df7
+5er4
+4no6
+5ugd
+1ciz
+1rev
+6hdq
+3c43
+1a7t
+6fky
+5n1r
+3rr4
+5bqg
+3b8r
+2pfy
+5c7f
+5l6j
+1etz
+3drf
+3k7f
+4b6e
+6asu
+1fd0
+4zjc
+1y3g
+4joh
+4c38
+4h5c
+5kej
+6aji
+4mvy
+3lpi
+4pms
+6fx1
+3ioi
+2xbj
+1bnm
+4tuh
+4fgt
+5mhp
+4hwb
+6eo9
+4aje
+1v2t
+5h85
+1x8d
+3uoj
+2aog
+3o5x
+5vr8
+2o4p
+1fh8
+4jz1
+5oq4
+3zdh
+5otz
+3lnz
+1n9m
+3rak
+6ba7
+6czb
+2wyj
+4qyo
+3mxd
+2ri9
+2zq1
+3ljo
+1kvo
+6g2a
+1gj6
+5oug
+1g4o
+3mvm
+1iyl
+2cgx
+3v2x
+5iv4
+4m2u
+4q1b
+3ggc
+2c68
+3ind
+1laf
+4kxn
+2ada
+1om1
+5eh7
+4jk5
+4q93
+1b8y
+1x11
+4mr6
+4c4j
+4yik
+1osv
+4y7r
+4zow
+2hdx
+4ui6
+4r6x
+1pxi
+5ni5
+5v4b
+1h0a
+4pf5
+5ggz
+1o4m
+1urc
+5mqt
+3ads
+5gn5
+4mrg
+5ylt
+5k6a
+4nld
+2xl3
+4auy
+1zoe
+4pum
+5lxd
+5csh
+3gbq
+5xs2
+4ggl
+5a14
+4au8
+5n49
+5x8i
+6c7f
+3fvk
+4kqp
+5y97
+6gub
+5mxx
+3lox
+2oc7
+3g1m
+6hsz
+1ilq
+3twj
+5vcx
+3svv
+4na4
+3v5g
+4p7e
+2vwl
+4umu
+3v4j
+6gfz
+6duf
+5aqf
+5ezh
+5nar
+4yti
+4zyq
+4lop
+4nus
+2a3a
+2o9i
+3ia6
+5nfh
+1e00
+5u8a
+3n8n
+4axd
+3kxz
+3owd
+4fxj
+1gx0
+4u90
+1d7x
+3eq8
+3q44
+4ago
+5e2p
+1pxj
+4p6c
+1zoh
+4i7j
+4hlg
+5j6n
+3s2o
+4ec0
+3wiy
+4k77
+5yf1
+1hsl
+4tjz
+6cbh
+6g38
+4efu
+5wep
+1jwm
+3wfg
+5lvl
+5kqd
+1fq5
+5nn6
+3b5r
+5xag
+3w69
+3udr
+4c4h
+5mgi
+5ad1
+5zh3
+4i7b
+6cwi
+1tkb
+3unj
+4u54
+2x0y
+3isw
+2xbw
+6fzu
+5wgq
+4kzc
+1yhs
+5btr
+1juq
+2idk
+2v25
+4prh
+5o5a
+3r0h
+1sle
+2ieh
+5ous
+2fqy
+1p7m
+3fuf
+1w10
+1vot
+1guw
+1gbq
+5thn
+6b41
+4xoc
+5ugc
+4h71
+5i43
+4oc1
+3t0d
+4kw6
+4jrg
+4ra5
+4azp
+5sz3
+5hbj
+1npa
+4wi1
+4b7p
+3biz
+4bpj
+3lxe
+5qav
+4ge6
+4k1e
+3vje
+4mic
+4qp9
+2xg5
+1w5x
+2rin
+3omc
+5ii2
+3pax
+5elq
+3bpc
+1ec0
+4x8g
+3cft
+4qd6
+5n1x
+2v96
+4ucc
+5ocj
+4mul
+3lok
+2e5y
+3eql
+5kyj
+1n5r
+3hfz
+2pg2
+3g3m
+4fs3
+1nnk
+4n9a
+4jp9
+2j77
+6cvd
+4tkb
+3tfp
+5a5o
+5nsx
+6bbv
+5myg
+3ws8
+6ex1
+2jkr
+5kcv
+2vw5
+4ezl
+3dp0
+5hyq
+5iyy
+2oc4
+5div
+1gag
+2r38
+3chp
+3ot8
+2wqb
+6bqh
+3gxl
+2pgj
+6gfx
+2i3v
+5yqw
+5n9t
+6cc9
+3p3s
+3oe9
+2y1o
+2o4r
+1dzm
+3vp2
+3djo
+4oz2
+1p6d
+5mih
+4xt2
+3tf7
+5v19
+6g9k
+4ipf
+3bim
+4nwm
+2aeb
+3blu
+3vtc
+3nww
+3mp6
+1nu8
+3wzk
+4zec
+3gvb
+4bt4
+2f6v
+5xwr
+5m53
+4yvc
+1ai4
+1z1r
+1ryh
+1rhu
+5d1s
+3g0i
+1gpk
+4clb
+5cte
+4kmu
+6f9r
+3pjg
+1uz1
+3tww
+2lbv
+5w1w
+1s26
+3w9r
+4yes
+3sl8
+5km0
+2j79
+2h03
+1qnh
+4cpx
+5enk
+4mnv
+3r5m
+4tk4
+5wg6
+5uwf
+4cjp
+3uu1
+4bxu
+3h30
+4qwr
+2gyi
+1cnw
+4fzc
+6gdg
+4e5i
+3oq5
+3wig
+5my8
+4aj1
+3b65
+2e95
+4xcb
+3wf5
+5h15
+5htb
+3t3e
+3h5b
+1ugp
+5anq
+6g4n
+1apb
+2ai7
+4xxs
+1lrh
+1ove
+3v7t
+1xk5
+4fiv
+4mcv
+1mrw
+4b14
+5isl
+5cj6
+5un9
+3nil
+1x07
+6fni
+4u2y
+4hbv
+6h2t
+4q6d
+3b8z
+2cnf
+4isu
+5u2f
+2cng
+4q90
+4l0b
+1z6s
+6g3q
+4ylu
+3w5t
+4dei
+2zjv
+2x6y
+3ikg
+5j7p
+4mcc
+3vqs
+3u9n
+2g5t
+3owj
+5bnm
+3hkn
+3qtx
+3qzt
+5bpa
+3p17
+6fag
+5nfb
+1o43
+1nok
+3vby
+6as6
+5vjn
+3skc
+3bg8
+1tc1
+5orh
+5bq0
+5qac
+3l16
+5kw2
+3uj9
+6byk
+5kgw
+4wvt
+1bv9
+4ruz
+1fzq
+1mu8
+1srg
+4y73
+3upv
+4jj8
+5xvf
+1xka
+4ewn
+4ewr
+4a7b
+2xab
+6g9b
+4u69
+1qb6
+3c6t
+3qar
+4k5o
+2gnf
+1u3q
+3g31
+3oev
+4oc2
+1eby
+3uef
+4k0o
+5ehi
+5h63
+3wcg
+6hvi
+1hii
+4jhq
+3sgt
+4aua
+4hnc
+3bxh
+5g5v
+4j08
+4anu
+3hac
+2vcq
+2f7p
+4ycl
+3s22
+3obq
+1o9k
+4xy2
+3k9x
+1xog
+5b0x
+4m5g
+4czs
+5eyr
+4lm5
+5xup
+4rao
+3qcl
+3lkj
+2xno
+1hfs
+1gjd
+4bzs
+4j7d
+1fw0
+1jfh
+2cgr
+4pd8
+3mz6
+5obj
+3gm0
+4j3u
+1uvt
+5ml5
+4fut
+4r3c
+4kfp
+4jck
+5xhs
+5mwz
+3n86
+5qab
+5zeq
+1fo3
+4eki
+5n8w
+2exm
+4klb
+5ikb
+2qhc
+5aqr
+4qn7
+5nbw
+4k2g
+3qck
+3zo1
+4knx
+3m6r
+6dh6
+3p8n
+5f3e
+2vie
+4uyh
+5uoy
+4gr3
+6cyd
+2f14
+5v2q
+3h2m
+3ao2
+1k1n
+3gfw
+4q19
+2b1g
+4pdk
+1oxn
+2pj1
+4o3a
+2y6s
+1inh
+4u5l
+4esg
+4mg6
+4de5
+3wke
+6dgy
+4mp7
+4de1
+4z6i
+2r4b
+5q0z
+4b33
+2koh
+4qyh
+5y94
+6fo8
+3p4r
+4mny
+2vjx
+5lj0
+5yqx
+5c26
+4eb8
+3lk1
+3mg6
+4oru
+2wpb
+4k7i
+5fsy
+3ava
+2z7r
+3ex6
+4b8p
+3l13
+5fbi
+3s3k
+3chc
+2wf0
+4hgt
+6b5i
+4ayw
+3gk4
+2lcs
+5fas
+1lyx
+6bvh
+6edr
+1ghz
+1dg9
+1egh
+2x6f
+5o48
+4avg
+3znc
+1czq
+5f39
+2bpy
+2vc9
+5op5
+4ory
+3hu3
+1xuc
+3hmo
+2ybt
+1v1k
+5o22
+3qt7
+4eo8
+4bco
+2p3i
+3hii
+2xdm
+6axq
+2xhs
+5cyv
+1sre
+2y8o
+9lpr
+4l3o
+1dzk
+3lnj
+3fsm
+4ghi
+2nqi
+1i48
+5mo2
+6fr2
+1bnn
+2jbl
+1r17
+4rrf
+5jah
+6apw
+4trc
+1lgt
+2wl0
+4i0z
+3oe5
+3fjz
+2uyn
+3hxd
+6g3o
+2nqg
+3fc8
+2zvj
+3mw1
+2c5n
+3poz
+5ax9
+3mhw
+5tdw
+5n7b
+4kij
+5axq
+3pxy
+2q96
+3kjd
+5cf5
+5id0
+5e0j
+4kjv
+4e1m
+3gy4
+5m34
+1xqc
+2hzy
+5brn
+2d0k
+3s8n
+2j7f
+5zc5
+3p5o
+1xh5
+2vj7
+4zt2
+2b7a
+1thl
+1jmq
+5h7h
+5mz3
+3dxj
+5q1g
+6ays
+3w2t
+1bkm
+3pb3
+3vjl
+2a8h
+4pax
+5tt8
+2y2p
+5lyx
+1zaf
+3mb6
+2fuu
+4mw6
+2j6m
+4rll
+3qai
+4kzq
+2uym
+5v84
+3bm9
+4egk
+2bqw
+2q8y
+4pji
+6ary
+4daw
+4i06
+2wxk
+1r5n

data/splits/timesplit_no_lig_overlap_val

@@ -0,0 +1,968 @@
+4lp9
+1me7
+2zv9
+2qo8
+1cw2
+3k5c
+2o65
+4kqq
+3rdv
+1d4w
+1q4l
+4b5w
+4bgg
+4mm5
+3iej
+3ftu
+830c
+2xye
+1olu
+2wk2
+4pxf
+5o0j
+1my2
+5czm
+4jit
+5mb1
+1sqp
+3zlw
+4xqu
+3hkq
+6fns
+5e0l
+2p8o
+4gzw
+3n87
+1lhc
+4itj
+4m7c
+4olh
+4q1e
+5l7e
+3faa
+5vqx
+3pka
+5x54
+5a9u
+4n9e
+4est
+1il9
+4igr
+3t2t
+6dar
+3gol
+3vbg
+2ydk
+4zpf
+5zo7
+4xnw
+1fpy
+2r1y
+6m8w
+2jds
+5icx
+1hwr
+6bj2
+4b4m
+1zsb
+4do3
+3t3i
+1f8a
+2ke1
+5ezx
+3p78
+4rvm
+3ovn
+5wzv
+4udb
+1okz
+1mpl
+5npc
+5ff6
+1hlf
+1nvq
+4bhf
+4y4g
+5mkz
+2o0u
+3bcs
+1wvc
+4fsl
+3oz1
+6dgt
+1me8
+2puy
+4odp
+1hpx
+4nrq
+1z2b
+3uik
+3mfv
+3vqh
+4w9g
+4xek
+4jok
+2wap
+1g50
+4j0p
+2o9a
+3m94
+4i1c
+5a82
+4i9h
+1k1i
+4uro
+2f7i
+5fpk
+2lgf
+4l7f
+1g3d
+4ir5
+3mta
+3jzg
+5f94
+4nrt
+4yax
+5nhv
+2xtk
+4qh7
+1tok
+4b6p
+3rg2
+3q8d
+3obu
+4awj
+3daj
+2j50
+5l2z
+5bml
+2bba
+5n34
+2xvn
+1dpu
+5fnt
+1jyc
+4zz1
+6hm7
+4rrv
+4rww
+5orv
+3qo2
+3uii
+6d1x
+3juq
+4qk4
+6mr5
+5hjc
+2p4s
+2hnc
+1k4g
+4g0c
+2y5g
+4u3f
+3tv5
+1i3z
+4mw7
+3n2c
+6cvw
+3v66
+3wzp
+3s7m
+5ujv
+1p06
+3ipy
+4wkt
+4ie0
+5fot
+5i59
+5za9
+4gii
+4h2o
+4yrs
+5a6h
+2xo8
+4e3n
+4m5k
+3dga
+6fse
+6ck6
+1sqc
+4x1r
+3dnj
+3rvi
+2a58
+4bf6
+3zlk
+4mbj
+4tpm
+4d8c
+1ejn
+4yt6
+2x7x
+4qp1
+4de3
+5yg4
+1x7b
+5n9s
+2fme
+1ydt
+2bdf
+6baw
+6fsd
+2xn3
+4tk0
+3q4j
+1u9l
+1oqp
+5htz
+4glr
+5kj0
+5ukl
+3fun
+4wk2
+4ht6
+5hv1
+1uze
+4bcc
+3ff6
+5if6
+1tsm
+2r59
+3iqh
+2v7a
+5d10
+5nvh
+3eqr
+1jq9
+1u1b
+6cer
+5uq9
+1u3s
+5icy
+3exh
+2oqs
+1pzp
+1d4i
+4x6p
+4mb9
+5emk
+1iky
+6b7f
+3chq
+3h5s
+5zmq
+4ib5
+2wej
+6fjm
+5ewa
+2igx
+2z78
+5lpm
+4wet
+3lxl
+2xba
+5wbl
+5zla
+2x6x
+4mw9
+5t2d
+4j3m
+4aqh
+3lbk
+4djp
+4odl
+4x6j
+1ero
+5f3t
+4k3q
+5ta4
+1caq
+2eg7
+1f73
+3rxg
+6ezq
+1qkt
+5l3e
+5c28
+4pp9
+4bgk
+3iaf
+5vrp
+5zz4
+5ur5
+3ft2
+5ech
+4jjq
+5iz6
+5dhr
+4l2g
+4r17
+3wk6
+4h1e
+2aq9
+5g1n
+3zm9
+5c4l
+5mfs
+1fzj
+2ltw
+4x7i
+4c94
+2cfg
+2va5
+3vb6
+2hob
+5ah2
+5syn
+3g6g
+3rwj
+5sz4
+4f9v
+5n2d
+3n9r
+5ldo
+3vb7
+1sqo
+3drg
+5j9y
+6b96
+4yz9
+1vcj
+5epr
+4tx6
+3dz6
+3czv
+5v49
+1ahy
+3wzq
+1bq4
+5u8c
+6bj3
+2qnb
+4a9m
+3d4f
+5oui
+5wmg
+6ma4
+4x5q
+5cbr
+6msy
+5avi
+1g3b
+2wi4
+3kjn
+4dhn
+4o7e
+5kit
+5y5t
+3hfj
+2qd8
+5vsj
+2y2i
+5m0m
+3tcp
+4bhz
+1jd6
+5idn
+4zzx
+4kn4
+2a5c
+6hly
+1au2
+4jbo
+5cgj
+3ske
+3lq2
+4pxm
+2wxg
+5tb6
+2vc7
+3iw4
+5hct
+3skf
+5lyy
+3fmz
+4p5z
+5ktw
+6e4w
+1cx9
+6em7
+4mjr
+4u7t
+3rde
+4ux4
+4i6f
+3l3x
+4ie6
+4j70
+1jd0
+4iaw
+1szm
+2afw
+3ess
+3sap
+1olx
+1bzh
+5hfb
+4x3h
+5we9
+3zsw
+5ny6
+1hn2
+3l3z
+4qp2
+1d4p
+4xkc
+2is0
+6c7e
+5zku
+4fai
+6g9a
+4xu3
+5dry
+4d8z
+3zcz
+3kbz
+2y59
+4nal
+4rpv
+4yje
+3vf8
+4bqx
+4z9l
+4ep2
+4ylk
+5mme
+4dht
+2uy4
+6mu3
+3kx1
+5o0s
+4bch
+5c4k
+2br1
+4ddh
+2f9k
+2w2i
+4ogn
+4up5
+5o4y
+5hjd
+2qw1
+5y8z
+4kqr
+1o2t
+6e05
+3u7l
+2mip
+3hvg
+2p59
+4d3h
+4pl5
+3tzd
+2vnp
+4e3m
+3vgc
+5bqi
+1b7h
+1lhu
+3rlr
+3h22
+2wnc
+2wot
+5d1t
+3mo0
+4wn5
+3p3u
+1nfs
+4e90
+5aqu
+1bmq
+3kwz
+6f6n
+4rj5
+4omd
+6min
+1ujj
+4ppa
+4uxl
+5y3n
+6df2
+4wvl
+1xt3
+5oaj
+4a9r
+5mli
+4p4e
+3juo
+1z9g
+2ykc
+5a0e
+3g0w
+5t9w
+1sqa
+3wci
+1fkw
+5u4g
+4mfe
+4kpx
+3nti
+3azb
+2xog
+3c3r
+2buc
+1hyz
+4dcd
+6azl
+3t3d
+3q4l
+4few
+1q95
+4u0b
+3b7u
+4bo4
+4o10
+5wmt
+5v9t
+5aok
+1jtq
+5uit
+2vgc
+2gfd
+3mna
+1aqc
+4xtt
+4z0d
+4ty9
+2yiv
+2hrp
+4zh2
+2z4o
+1qku
+2xdw
+4n7j
+4yp1
+3exf
+4c6z
+6ccu
+2wxn
+1bwb
+2gvf
+1hiy
+5c4t
+2za5
+2xkf
+4q18
+1o2p
+5th2
+4dj7
+3eyd
+4j0r
+2m3o
+2b53
+4m3b
+2izl
+2vtr
+2x6d
+2i0a
+5ehg
+6cw4
+4c37
+3cwj
+1azm
+2qci
+5sz0
+2gkl
+2z4z
+6awo
+1v11
+4l53
+3p55
+2ynn
+2vu3
+4dli
+2bcd
+4l0s
+4uda
+3m37
+5j5t
+2p16
+4gh6
+1mfg
+3s3i
+4j73
+2v5x
+2h4n
+4jsz
+4wk1
+4igt
+4k63
+3qqk
+16pk
+5aom
+1hyv
+5a3w
+3veh
+3g4l
+2ph8
+5mkx
+5c4u
+4gto
+3cj5
+4prj
+2vd7
+5duc
+3odi
+6bg5
+1qwu
+5jn8
+1v1m
+1qpe
+5v3r
+2wc4
+2vte
+1a52
+4dhq
+2qta
+6ccy
+4jog
+4bgy
+5u9i
+3az9
+1gt1
+2jew
+3pdc
+1n3i
+5fyx
+4f49
+4nzn
+6hm2
+4a4l
+5xij
+5vk0
+4xsx
+2aj8
+4odq
+2n7b
+4ygf
+2a4q
+2jc0
+4jsa
+1inq
+3dc3
+5tob
+4urn
+6bik
+4ju4
+5nya
+5oh2
+5znr
+5ct2
+3u4u
+4x7h
+3max
+3rbm
+3krj
+1aj6
+1pmv
+5n0e
+4nhy
+4oem
+6fi4
+4e3j
+1fq4
+5myr
+2hkf
+1os0
+3rqg
+4ivc
+5c7b
+3lq4
+1u6q
+1qxz
+1l5r
+4xxh
+3m40
+5or9
+4okg
+4d89
+2gm9
+5x33
+4de0
+4gr8
+5lz8
+1p93
+2brp
+2gg8
+6fdt
+5cxh
+1jvu
+3wp1
+1fzm
+5cxa
+2gbg
+2g78
+5aml
+2y34
+2qnp
+1v16
+1njj
+2a5u
+4z88
+4wmx
+5vo2
+4fod
+2pou
+3jsw
+2ow2
+5g3m
+3odl
+3o9e
+3eyh
+4ej2
+3c4e
+4b6f
+1pl0
+3pb8
+6fap
+4iax
+2bua
+6fgg
+2o4h
+4uwh
+5wbf
+2yxj
+1ff1
+2giu
+1qbt
+2ovq
+4bak
+2y3p
+2iwu
+3hvi
+2w0x
+3fcl
+1zpa
+5czb
+3t1l
+2cfd
+3k3g
+4cfw
+2e91
+5op8
+3hig
+6h7y
+3mtb
+4eb9
+4lkg
+5ehv
+5ier
+4ode
+1xoq
+5d6p
+3kwa
+5np8
+5v82
+6ma1
+3bz3
+3myq
+4j0s
+4f4p
+4lh6
+1uef
+4j3d
+4yx4
+4amx
+4ptg
+2c97
+4ec4
+4r1v
+1zc9
+4nuf
+3g2u
+6hlx
+5vij
+2x4o
+6hlz
+4lkj
+3s75
+2gz8
+1gvk
+2yhd
+3hqz
+3pb7
+1thr
+4ris
+5twh
+4gql
+3n3l
+3acx
+5yvx
+3gy2
+1xmu
+5l6p
+5l8n
+4msn
+4rz1
+3f66
+3ucj
+5hcl
+1t1r
+3kce
+3u15
+1wbg
+5khi
+3er5
+4qew
+5mft
+6eqp
+5gsw
+2qd7
+4cli
+3f9w
+3msc
+1jgl
+3kid
+1ymx
+1ui0
+3d1f
+1pxl
+5kos
+3vzd
+5fcz
+3ara
+4li6
+5ks7
+4wym
+5j7q
+4qsh
+2ce9
+5vqz
+3o2m
+4bcm
+5orx
+1i41
+3c5u
+4kai
+6gjy
+4tsz
+5o0e
+6drt
+1y57
+3kqb
+3jup
+5ork
+3ikc
+3gwu
+4wke
+4x7l
+3lp1
+5ivy
+3f16
+4c36
+1w2x
+2d06
+1hbj
+1ols
+1iup
+5aix
+1ydd
+5w4r
+3h23
+3rj7
+4ish
+1ebw
+1fcy
+1d09
+5hdv
+4x1n
+5boj
+2xn7
+4b6s
+3f82
+4clj
+4zzz
+5j5d
+2vts
+1k08
+3u3f
+4jk6
+4csy
+6hth
+2mnz
+2vpg
+2qd6
+4jkw
+3ml5
+1ih0
+4at5
+5dgu
+4g31
+5n0d
+5aa9
+4u4s
+5oa6
+2wzm
+4b4q
+6fi1
+6chn
+1z4u
+5aa8
+1lpk
+3cib
+5d75
+5x4o
+1ydb
+5dhq
+5t28
+4zz0
+3evf
+5vyy
+6eip
+1q63
+3ldw
+5tq4
+5uxf
+2j7x
+4kil
+1yda
+3bc4
+2ew5
+6ee3
+4yrr
+3wax
+3bzf
+5ody
+1k06
+4j84
+5l6h
+5eok
+5nne
+5m6m
+2a4r
+3p1d
+2ayp
+3iux
+4b0g
+1jr1
+4qo9
+4bh4
+4xt9
+2ok1
+2r7g
+4uib
+5mmn
+5akj
+3hs4
+5wpb
+6e5x
+5vnd
+5evd
+5wlg
+5l4m
+4kiu
+4own
+5oh9
+6arv
+1xr9
+4hv7

data/splits/timesplit_test

@@ -0,0 +1,363 @@
+6qqw
+6d08
+6jap
+6np2
+6uvp
+6oxq
+6jsn
+6hzb
+6qrc
+6oio
+6jag
+6moa
+6hld
+6i9a
+6e4c
+6g24
+6jb4
+6s55
+6seo
+6dyz
+5zk5
+6jid
+5ze6
+6qlu
+6a6k
+6qgf
+6e3z
+6te6
+6pka
+6g2o
+6jsf
+5zxk
+6qxd
+6n97
+6jt3
+6qtr
+6oy1
+6n96
+6qzh
+6qqz
+6qmt
+6ibx
+6hmt
+5zk7
+6k3l
+6cjs
+6n9l
+6ibz
+6ott
+6gge
+6hot
+6e3p
+6md6
+6hlb
+6fe5
+6uwp
+6npp
+6g2f
+6mo7
+6bqd
+6nsv
+6i76
+6n53
+6g2c
+6eeb
+6n0m
+6uvy
+6ovz
+6olx
+6v5l
+6hhg
+5zcu
+6dz2
+6mjq
+6efk
+6s9w
+6gdy
+6kqi
+6ueg
+6oxt
+6oy0
+6qr7
+6i41
+6cyg
+6qmr
+6g27
+6ggb
+6g3c
+6n4e
+6fcj
+6quv
+6iql
+6i74
+6qr4
+6rnu
+6jib
+6izq
+6qw8
+6qto
+6qrd
+6hza
+6e5s
+6dz3
+6e6w
+6cyh
+5zlf
+6om4
+6gga
+6pgp
+6qqv
+6qtq
+6gj6
+6os5
+6s07
+6i77
+6hhj
+6ahs
+6oxx
+6mjj
+6hor
+6jb0
+6i68
+6pz4
+6mhb
+6uim
+6jsg
+6i78
+6oxy
+6gbw
+6mo0
+6ggf
+6qge
+6cjr
+6oxp
+6d07
+6i63
+6ten
+6uii
+6qlr
+6sen
+6oxv
+6g2b
+5zr3
+6kjf
+6qr9
+6g9f
+6e6v
+5zk9
+6pnn
+6nri
+6uwv
+6ooz
+6npi
+6oip
+6miv
+6s57
+6p8x
+6hoq
+6qts
+6ggd
+6pnm
+6oy2
+6oi8
+6mhd
+6agt
+6i5p
+6hhr
+6p8z
+6c85
+6g5u
+6j06
+6qsz
+6jbb
+6hhp
+6np5
+6nlj
+6qlp
+6n94
+6e13
+6qls
+6uil
+6st3
+6n92
+6s56
+6hzd
+6uhv
+6k05
+6q36
+6ic0
+6hhi
+6e3m
+6qtx
+6jse
+5zjy
+6o3y
+6rpg
+6rr0
+6gzy
+6qlt
+6ufo
+6o0h
+6o3x
+5zjz
+6i8t
+6ooy
+6oiq
+6od6
+6nrh
+6qra
+6hhh
+6m7h
+6ufn
+6qr0
+6o5u
+6h14
+6jwa
+6ny0
+6jan
+6ftf
+6oxw
+6jon
+6cf7
+6rtn
+6jsz
+6o9c
+6mo8
+6qln
+6qqu
+6i66
+6mja
+6gwe
+6d3z
+6oxr
+6r4k
+6hle
+6h9v
+6hou
+6nv9
+6py0
+6qlq
+6nv7
+6n4b
+6jaq
+6i8m
+6dz0
+6oxs
+6k2n
+6cjj
+6ffg
+6a73
+6qqt
+6a1c
+6oxu
+6qre
+6qtw
+6np4
+6hv2
+6n55
+6e3o
+6kjd
+6sfc
+6qi7
+6hzc
+6k04
+6op0
+6q38
+6n8x
+6np3
+6uvv
+6pgo
+6jbe
+6i75
+6qqq
+6i62
+6j9y
+6g29
+6h7d
+6mo9
+6jao
+6jmf
+6hmy
+6qfe
+5zml
+6i65
+6e7m
+6i61
+6rz6
+6qtm
+6qlo
+6oie
+6miy
+6nrf
+6gj5
+6jad
+6mj4
+6h12
+6d3y
+6qr2
+6qxa
+6o9b
+6ckl
+6oir
+6d40
+6e6j
+6i7a
+6g25
+6oin
+6jam
+6oxz
+6hop
+6rot
+6uhu
+6mji
+6nrj
+6nt2
+6op9
+6pno
+6e4v
+6k1s
+6a87
+6oim
+6cjp
+6pyb
+6h13
+6qrf
+6mhc
+6j9w
+6nrg
+6fff
+6n93
+6jut
+6g2e
+6nd3
+6os6
+6dql
+6inz
+6i67
+6quw
+6qwi
+6npm
+6i64
+6e3n
+6qrg
+6nxz
+6iby
+6gj7
+6qr3
+6qr1
+6s9x
+6q4q
+6hbn
+6nw3
+6tel
+6p8y
+6d5w
+6t6a
+6o5g
+6r7d
+6pya
+6ffe
+6d3x
+6gj8
+6mo2

data/splits/timesplit_test_no_rec_overlap

@@ -0,0 +1,144 @@
+6qqw
+6jap
+6np2
+6qrc
+6oio
+6jag
+6i9a
+6jb4
+6seo
+6jid
+5ze6
+6pka
+6n97
+6qtr
+6n96
+6qzh
+6qqz
+6k3l
+6cjs
+6n9l
+6ott
+6npp
+6nsv
+6n53
+6eeb
+6n0m
+6ovz
+5zcu
+6mjq
+6efk
+6gdy
+6kqi
+6ueg
+6qr7
+6g3c
+6iql
+6qr4
+6jib
+6qto
+6qrd
+6e5s
+5zlf
+6om4
+6qqv
+6qtq
+6os5
+6s07
+6mjj
+6jb0
+6uim
+6mo0
+6cjr
+6uii
+6sen
+6kjf
+6qr9
+6g9f
+6npi
+6oip
+6miv
+6qts
+6oi8
+6c85
+6qsz
+6jbb
+6np5
+6nlj
+6n94
+6e13
+6uil
+6n92
+6uhv
+6q36
+6qtx
+6rr0
+6ufo
+6oiq
+6qra
+6m7h
+6ufn
+6qr0
+6o5u
+6ny0
+6jan
+6ftf
+6jon
+6cf7
+6o9c
+6qqu
+6mja
+6r4k
+6h9v
+6py0
+6jaq
+6k2n
+6cjj
+6a73
+6qqt
+6qre
+6qtw
+6np4
+6n55
+6kjd
+6np3
+6jbe
+6qqq
+6j9y
+6h7d
+6jao
+6e7m
+6rz6
+6qtm
+6miy
+6jad
+6mj4
+6qr2
+6qxa
+6o9b
+6ckl
+6oir
+6oin
+6jam
+6uhu
+6mji
+6nt2
+6op9
+6e4v
+6a87
+6cjp
+6qrf
+6j9w
+6n93
+6nd3
+6os6
+6dql
+6qwi
+6npm
+6qrg
+6nxz
+6qr3
+6qr1
+6o5g
+6r7d
+6mo2

datasets/conformer_matching.py

@@ -0,0 +1,196 @@
+import copy, time
+import numpy as np
+from collections import defaultdict
+from rdkit import Chem, RDLogger
+from rdkit.Chem import AllChem, rdMolTransforms
+from rdkit import Geometry
+import networkx as nx
+from scipy.optimize import differential_evolution
+
+RDLogger.DisableLog('rdApp.*')
+
+"""
+    Conformer matching routines from Torsional Diffusion
+"""
+
+def GetDihedral(conf, atom_idx):
+    return rdMolTransforms.GetDihedralRad(conf, atom_idx[0], atom_idx[1], atom_idx[2], atom_idx[3])
+
+
+def SetDihedral(conf, atom_idx, new_vale):
+    rdMolTransforms.SetDihedralRad(conf, atom_idx[0], atom_idx[1], atom_idx[2], atom_idx[3], new_vale)
+
+
+def apply_changes(mol, values, rotable_bonds, conf_id):
+    opt_mol = copy.copy(mol)
+    [SetDihedral(opt_mol.GetConformer(conf_id), rotable_bonds[r], values[r]) for r in range(len(rotable_bonds))]
+    return opt_mol
+
+
+def optimize_rotatable_bonds(mol, true_mol, rotable_bonds, probe_id=-1, ref_id=-1, seed=0, popsize=15, maxiter=500,
+                             mutation=(0.5, 1), recombination=0.8):
+    opt = OptimizeConformer(mol, true_mol, rotable_bonds, seed=seed, probe_id=probe_id, ref_id=ref_id)
+    max_bound = [np.pi] * len(opt.rotable_bonds)
+    min_bound = [-np.pi] * len(opt.rotable_bonds)
+    bounds = (min_bound, max_bound)
+    bounds = list(zip(bounds[0], bounds[1]))
+
+    # Optimize conformations
+    result = differential_evolution(opt.score_conformation, bounds,
+                                    maxiter=maxiter, popsize=popsize,
+                                    mutation=mutation, recombination=recombination, disp=False, seed=seed)
+    opt_mol = apply_changes(opt.mol, result['x'], opt.rotable_bonds, conf_id=probe_id)
+
+    return opt_mol
+
+
+class OptimizeConformer:
+    def __init__(self, mol, true_mol, rotable_bonds, probe_id=-1, ref_id=-1, seed=None):
+        super(OptimizeConformer, self).__init__()
+        if seed:
+            np.random.seed(seed)
+        self.rotable_bonds = rotable_bonds
+        self.mol = mol
+        self.true_mol = true_mol
+        self.probe_id = probe_id
+        self.ref_id = ref_id
+
+    def score_conformation(self, values):
+        for i, r in enumerate(self.rotable_bonds):
+            SetDihedral(self.mol.GetConformer(self.probe_id), r, values[i])
+        return RMSD(self.mol, self.true_mol, self.probe_id, self.ref_id)
+
+
+def get_torsion_angles(mol):
+    torsions_list = []
+    G = nx.Graph()
+    for i, atom in enumerate(mol.GetAtoms()):
+        G.add_node(i)
+    nodes = set(G.nodes())
+    for bond in mol.GetBonds():
+        start, end = bond.GetBeginAtomIdx(), bond.GetEndAtomIdx()
+        G.add_edge(start, end)
+    for e in G.edges():
+        G2 = copy.deepcopy(G)
+        G2.remove_edge(*e)
+        if nx.is_connected(G2): continue
+        l = list(sorted(nx.connected_components(G2), key=len)[0])
+        if len(l) < 2: continue
+        n0 = list(G2.neighbors(e[0]))
+        n1 = list(G2.neighbors(e[1]))
+        torsions_list.append(
+            (n0[0], e[0], e[1], n1[0])
+        )
+    return torsions_list
+
+
+# GeoMol
+def get_torsions(mol_list):
+    print('USING GEOMOL GET TORSIONS FUNCTION')
+    atom_counter = 0
+    torsionList = []
+    for m in mol_list:
+        torsionSmarts = '[!$(*#*)&!D1]-&!@[!$(*#*)&!D1]'
+        torsionQuery = Chem.MolFromSmarts(torsionSmarts)
+        matches = m.GetSubstructMatches(torsionQuery)
+        for match in matches:
+            idx2 = match[0]
+            idx3 = match[1]
+            bond = m.GetBondBetweenAtoms(idx2, idx3)
+            jAtom = m.GetAtomWithIdx(idx2)
+            kAtom = m.GetAtomWithIdx(idx3)
+            for b1 in jAtom.GetBonds():
+                if (b1.GetIdx() == bond.GetIdx()):
+                    continue
+                idx1 = b1.GetOtherAtomIdx(idx2)
+                for b2 in kAtom.GetBonds():
+                    if ((b2.GetIdx() == bond.GetIdx())
+                            or (b2.GetIdx() == b1.GetIdx())):
+                        continue
+                    idx4 = b2.GetOtherAtomIdx(idx3)
+                    # skip 3-membered rings
+                    if (idx4 == idx1):
+                        continue
+                    if m.GetAtomWithIdx(idx4).IsInRing():
+                        torsionList.append(
+                            (idx4 + atom_counter, idx3 + atom_counter, idx2 + atom_counter, idx1 + atom_counter))
+                        break
+                    else:
+                        torsionList.append(
+                            (idx1 + atom_counter, idx2 + atom_counter, idx3 + atom_counter, idx4 + atom_counter))
+                        break
+                break
+
+        atom_counter += m.GetNumAtoms()
+    return torsionList
+
+
+def A_transpose_matrix(alpha):
+    return np.array([[np.cos(alpha), np.sin(alpha)], [-np.sin(alpha), np.cos(alpha)]], dtype=np.double)
+
+
+def S_vec(alpha):
+    return np.array([[np.cos(alpha)], [np.sin(alpha)]], dtype=np.double)
+
+
+def GetDihedralFromPointCloud(Z, atom_idx):
+    p = Z[list(atom_idx)]
+    b = p[:-1] - p[1:]
+    b[0] *= -1
+    v = np.array([v - (v.dot(b[1]) / b[1].dot(b[1])) * b[1] for v in [b[0], b[2]]])
+    # Normalize vectors
+    v /= np.sqrt(np.einsum('...i,...i', v, v)).reshape(-1, 1)
+    b1 = b[1] / np.linalg.norm(b[1])
+    x = np.dot(v[0], v[1])
+    m = np.cross(v[0], b1)
+    y = np.dot(m, v[1])
+    return np.arctan2(y, x)
+
+
+def get_dihedral_vonMises(mol, conf, atom_idx, Z):
+    Z = np.array(Z)
+    v = np.zeros((2, 1))
+    iAtom = mol.GetAtomWithIdx(atom_idx[1])
+    jAtom = mol.GetAtomWithIdx(atom_idx[2])
+    k_0 = atom_idx[0]
+    i = atom_idx[1]
+    j = atom_idx[2]
+    l_0 = atom_idx[3]
+    for b1 in iAtom.GetBonds():
+        k = b1.GetOtherAtomIdx(i)
+        if k == j:
+            continue
+        for b2 in jAtom.GetBonds():
+            l = b2.GetOtherAtomIdx(j)
+            if l == i:
+                continue
+            assert k != l
+            s_star = S_vec(GetDihedralFromPointCloud(Z, (k, i, j, l)))
+            a_mat = A_transpose_matrix(GetDihedral(conf, (k, i, j, k_0)) + GetDihedral(conf, (l_0, i, j, l)))
+            v = v + np.matmul(a_mat, s_star)
+    v = v / np.linalg.norm(v)
+    v = v.reshape(-1)
+    return np.arctan2(v[1], v[0])
+
+
+def get_von_mises_rms(mol, mol_rdkit, rotable_bonds, conf_id):
+    new_dihedrals = np.zeros(len(rotable_bonds))
+    for idx, r in enumerate(rotable_bonds):
+        new_dihedrals[idx] = get_dihedral_vonMises(mol_rdkit,
+                                                   mol_rdkit.GetConformer(conf_id), r,
+                                                   mol.GetConformer().GetPositions())
+    mol_rdkit = apply_changes(mol_rdkit, new_dihedrals, rotable_bonds, conf_id)
+    return RMSD(mol_rdkit, mol, conf_id)
+
+
+def mmff_func(mol):
+    mol_mmff = copy.deepcopy(mol)
+    AllChem.MMFFOptimizeMoleculeConfs(mol_mmff, mmffVariant='MMFF94s')
+    for i in range(mol.GetNumConformers()):
+        coords = mol_mmff.GetConformers()[i].GetPositions()
+        for j in range(coords.shape[0]):
+            mol.GetConformer(i).SetAtomPosition(j,
+                                                Geometry.Point3D(*coords[j]))
+
+
+RMSD = AllChem.AlignMol

datasets/esm_embedding_preparation.py

@@ -0,0 +1,88 @@
+import os
+from argparse import FileType, ArgumentParser
+
+import numpy as np
+import pandas as pd
+from Bio.PDB import PDBParser
+from Bio.Seq import Seq
+from Bio.SeqRecord import SeqRecord
+from tqdm import tqdm
+from Bio import SeqIO
+
+
+
+def esm_embedding_prep(out_file, protein_path):
+    biopython_parser = PDBParser()
+
+    three_to_one = {
+        "ALA": "A",
+        "ARG": "R",
+        "ASN": "N",
+        "ASP": "D",
+        "CYS": "C",
+        "GLN": "Q",
+        "GLU": "E",
+        "GLY": "G",
+        "HIS": "H",
+        "ILE": "I",
+        "LEU": "L",
+        "LYS": "K",
+        "MET": "M",
+        "MSE": "M",  # MSE this is almost the same AA as MET. The sulfur is just replaced by Selen
+        "PHE": "F",
+        "PRO": "P",
+        "PYL": "O",
+        "SER": "S",
+        "SEC": "U",
+        "THR": "T",
+        "TRP": "W",
+        "TYR": "Y",
+        "VAL": "V",
+        "ASX": "B",
+        "GLX": "Z",
+        "XAA": "X",
+        "XLE": "J",
+    }
+
+    file_paths = [protein_path]
+    sequences = []
+    ids = []
+    for file_path in tqdm(file_paths):
+        structure = biopython_parser.get_structure("random_id", file_path)
+        structure = structure[0]
+        for i, chain in enumerate(structure):
+            seq = ""
+            for res_idx, residue in enumerate(chain):
+                if residue.get_resname() == "HOH":
+                    continue
+                residue_coords = []
+                c_alpha, n, c = None, None, None
+                for atom in residue:
+                    if atom.name == "CA":
+                        c_alpha = list(atom.get_vector())
+                    if atom.name == "N":
+                        n = list(atom.get_vector())
+                    if atom.name == "C":
+                        c = list(atom.get_vector())
+                if (
+                    c_alpha != None and n != None and c != None
+                ):  # only append residue if it is an amino acid
+                    try:
+                        seq += three_to_one[residue.get_resname()]
+                    except Exception as e:
+                        seq += "-"
+                        print(
+                            "encountered unknown AA: ",
+                            residue.get_resname(),
+                            " in the complex ",
+                            file_path,
+                            ". Replacing it with a dash - .",
+                        )
+            sequences.append(seq)
+            ids.append(f"{os.path.basename(file_path)}_chain_{i}")
+    records = []
+    for (index, seq) in zip(ids, sequences):
+        record = SeqRecord(Seq(seq), str(index))
+        record.description = ""
+        records.append(record)
+    SeqIO.write(records, out_file, "fasta")

datasets/esm_embeddings_to_pt.py

@@ -0,0 +1,17 @@
+
+import os
+from argparse import ArgumentParser
+
+import torch
+from tqdm import tqdm
+
+
+parser = ArgumentParser()
+parser.add_argument('--esm_embeddings_path', type=str, default='data/embeddings_output', help='')
+parser.add_argument('--output_path', type=str, default='data/esm2_3billion_embeddings.pt', help='')
+args = parser.parse_args()
+
+dict = {}
+for filename in tqdm(os.listdir(args.esm_embeddings_path)):
+    dict[filename.split('.')[0]] = torch.load(os.path.join(args.esm_embeddings_path,filename))['representations'][33]
+torch.save(dict,args.output_path)

datasets/pdbbind.py

@@ -0,0 +1,705 @@
+import binascii
+import glob
+import hashlib
+import os
+import pickle
+from collections import defaultdict
+from multiprocessing import Pool
+import random
+import copy
+
+import numpy as np
+import torch
+from rdkit.Chem import MolToSmiles, MolFromSmiles, AddHs
+from torch_geometric.data import Dataset, HeteroData
+from torch_geometric.loader import DataLoader, DataListLoader
+from torch_geometric.transforms import BaseTransform
+from tqdm import tqdm
+
+from datasets.process_mols import (
+    read_molecule,
+    get_rec_graph,
+    generate_conformer,
+    get_lig_graph_with_matching,
+    extract_receptor_structure,
+    parse_receptor,
+    parse_pdb_from_path,
+)
+from utils.diffusion_utils import modify_conformer, set_time
+from utils.utils import read_strings_from_txt
+from utils import so3, torus
+
+
+class NoiseTransform(BaseTransform):
+    def __init__(self, t_to_sigma, no_torsion, all_atom):
+        self.t_to_sigma = t_to_sigma
+        self.no_torsion = no_torsion
+        self.all_atom = all_atom
+
+    def __call__(self, data):
+        t = np.random.uniform()
+        t_tr, t_rot, t_tor = t, t, t
+        return self.apply_noise(data, t_tr, t_rot, t_tor)
+
+    def apply_noise(
+        self,
+        data,
+        t_tr,
+        t_rot,
+        t_tor,
+        tr_update=None,
+        rot_update=None,
+        torsion_updates=None,
+    ):
+        if not torch.is_tensor(data["ligand"].pos):
+            data["ligand"].pos = random.choice(data["ligand"].pos)
+
+        tr_sigma, rot_sigma, tor_sigma = self.t_to_sigma(t_tr, t_rot, t_tor)
+        set_time(data, t_tr, t_rot, t_tor, 1, self.all_atom, device=None)
+
+        tr_update = (
+            torch.normal(mean=0, std=tr_sigma, size=(1, 3))
+            if tr_update is None
+            else tr_update
+        )
+        rot_update = so3.sample_vec(eps=rot_sigma) if rot_update is None else rot_update
+        torsion_updates = (
+            np.random.normal(
+                loc=0.0, scale=tor_sigma, size=data["ligand"].edge_mask.sum()
+            )
+            if torsion_updates is None
+            else torsion_updates
+        )
+        torsion_updates = None if self.no_torsion else torsion_updates
+        modify_conformer(
+            data, tr_update, torch.from_numpy(rot_update).float(), torsion_updates
+        )
+
+        data.tr_score = -tr_update / tr_sigma**2
+        data.rot_score = (
+            torch.from_numpy(so3.score_vec(vec=rot_update, eps=rot_sigma))
+            .float()
+            .unsqueeze(0)
+        )
+        data.tor_score = (
+            None
+            if self.no_torsion
+            else torch.from_numpy(torus.score(torsion_updates, tor_sigma)).float()
+        )
+        data.tor_sigma_edge = (
+            None
+            if self.no_torsion
+            else np.ones(data["ligand"].edge_mask.sum()) * tor_sigma
+        )
+        return data
+
+
+class PDBBind(Dataset):
+    def __init__(
+        self,
+        root,
+        transform=None,
+        cache_path="data/cache",
+        split_path="data/",
+        limit_complexes=0,
+        receptor_radius=30,
+        num_workers=1,
+        c_alpha_max_neighbors=None,
+        popsize=15,
+        maxiter=15,
+        matching=True,
+        keep_original=False,
+        max_lig_size=None,
+        remove_hs=False,
+        num_conformers=1,
+        all_atoms=False,
+        atom_radius=5,
+        atom_max_neighbors=None,
+        esm_embeddings_path=None,
+        require_ligand=False,
+        ligands_list=None,
+        protein_path_list=None,
+        ligand_descriptions=None,
+        keep_local_structures=False,
+    ):
+
+        super(PDBBind, self).__init__(root, transform)
+        self.pdbbind_dir = root
+        self.max_lig_size = max_lig_size
+        self.split_path = split_path
+        self.limit_complexes = limit_complexes
+        self.receptor_radius = receptor_radius
+        self.num_workers = num_workers
+        self.c_alpha_max_neighbors = c_alpha_max_neighbors
+        self.remove_hs = remove_hs
+        self.esm_embeddings_path = esm_embeddings_path
+        self.require_ligand = require_ligand
+        self.protein_path_list = protein_path_list
+        self.ligand_descriptions = ligand_descriptions
+        self.keep_local_structures = keep_local_structures
+        if (
+            matching
+            or protein_path_list is not None
+            and ligand_descriptions is not None
+        ):
+            cache_path += "_torsion"
+        if all_atoms:
+            cache_path += "_allatoms"
+        self.full_cache_path = os.path.join(
+            cache_path,
+            f"limit{self.limit_complexes}"
+            f"_INDEX{os.path.splitext(os.path.basename(self.split_path))[0]}"
+            f"_maxLigSize{self.max_lig_size}_H{int(not self.remove_hs)}"
+            f"_recRad{self.receptor_radius}_recMax{self.c_alpha_max_neighbors}"
+            + (
+                ""
+                if not all_atoms
+                else f"_atomRad{atom_radius}_atomMax{atom_max_neighbors}"
+            )
+            + ("" if not matching or num_conformers == 1 else f"_confs{num_conformers}")
+            + ("" if self.esm_embeddings_path is None else f"_esmEmbeddings")
+            + ("" if not keep_local_structures else f"_keptLocalStruct")
+            + (
+                ""
+                if protein_path_list is None or ligand_descriptions is None
+                else str(
+                    binascii.crc32(
+                        "".join(ligand_descriptions + protein_path_list).encode()
+                    )
+                )
+            ),
+        )
+        self.popsize, self.maxiter = popsize, maxiter
+        self.matching, self.keep_original = matching, keep_original
+        self.num_conformers = num_conformers
+        self.all_atoms = all_atoms
+        self.atom_radius, self.atom_max_neighbors = atom_radius, atom_max_neighbors
+        if not os.path.exists(
+            os.path.join(self.full_cache_path, "heterographs.pkl")
+        ) or (
+            require_ligand
+            and not os.path.exists(
+                os.path.join(self.full_cache_path, "rdkit_ligands.pkl")
+            )
+        ):
+            os.makedirs(self.full_cache_path, exist_ok=True)
+            if protein_path_list is None or ligand_descriptions is None:
+                self.preprocessing()
+            else:
+                self.inference_preprocessing()
+
+        print(
+            "loading data from memory: ",
+            os.path.join(self.full_cache_path, "heterographs.pkl"),
+        )
+        with open(os.path.join(self.full_cache_path, "heterographs.pkl"), "rb") as f:
+            self.complex_graphs = pickle.load(f)
+        if require_ligand:
+            with open(
+                os.path.join(self.full_cache_path, "rdkit_ligands.pkl"), "rb"
+            ) as f:
+                self.rdkit_ligands = pickle.load(f)
+
+        print_statistics(self.complex_graphs)
+
+    def len(self):
+        return len(self.complex_graphs)
+
+    def get(self, idx):
+        if self.require_ligand:
+            complex_graph = copy.deepcopy(self.complex_graphs[idx])
+            complex_graph.mol = copy.deepcopy(self.rdkit_ligands[idx])
+            return complex_graph
+        else:
+            return copy.deepcopy(self.complex_graphs[idx])
+
+    def preprocessing(self):
+        print(
+            f"Processing complexes from [{self.split_path}] and saving it to [{self.full_cache_path}]"
+        )
+
+        complex_names_all = read_strings_from_txt(self.split_path)
+        if self.limit_complexes is not None and self.limit_complexes != 0:
+            complex_names_all = complex_names_all[: self.limit_complexes]
+        print(f"Loading {len(complex_names_all)} complexes.")
+
+        if self.esm_embeddings_path is not None:
+            id_to_embeddings = torch.load(self.esm_embeddings_path)
+            chain_embeddings_dictlist = defaultdict(list)
+            for key, embedding in id_to_embeddings.items():
+                key_name = key.split("_")[0]
+                if key_name in complex_names_all:
+                    chain_embeddings_dictlist[key_name].append(embedding)
+            lm_embeddings_chains_all = []
+            for name in complex_names_all:
+                lm_embeddings_chains_all.append(chain_embeddings_dictlist[name])
+        else:
+            lm_embeddings_chains_all = [None] * len(complex_names_all)
+
+        if self.num_workers > 1:
+            # running preprocessing in parallel on multiple workers and saving the progress every 1000 complexes
+            for i in range(len(complex_names_all) // 1000 + 1):
+                if os.path.exists(
+                    os.path.join(self.full_cache_path, f"heterographs{i}.pkl")
+                ):
+                    continue
+                complex_names = complex_names_all[1000 * i : 1000 * (i + 1)]
+                lm_embeddings_chains = lm_embeddings_chains_all[
+                    1000 * i : 1000 * (i + 1)
+                ]
+                complex_graphs, rdkit_ligands = [], []
+                if self.num_workers > 1:
+                    p = Pool(self.num_workers, maxtasksperchild=1)
+                    p.__enter__()
+                with tqdm(
+                    total=len(complex_names),
+                    desc=f"loading complexes {i}/{len(complex_names_all)//1000+1}",
+                ) as pbar:
+                    map_fn = p.imap_unordered if self.num_workers > 1 else map
+                    for t in map_fn(
+                        self.get_complex,
+                        zip(
+                            complex_names,
+                            lm_embeddings_chains,
+                            [None] * len(complex_names),
+                            [None] * len(complex_names),
+                        ),
+                    ):
+                        complex_graphs.extend(t[0])
+                        rdkit_ligands.extend(t[1])
+                        pbar.update()
+                if self.num_workers > 1:
+                    p.__exit__(None, None, None)
+
+                with open(
+                    os.path.join(self.full_cache_path, f"heterographs{i}.pkl"), "wb"
+                ) as f:
+                    pickle.dump((complex_graphs), f)
+                with open(
+                    os.path.join(self.full_cache_path, f"rdkit_ligands{i}.pkl"), "wb"
+                ) as f:
+                    pickle.dump((rdkit_ligands), f)
+
+            complex_graphs_all = []
+            for i in range(len(complex_names_all) // 1000 + 1):
+                with open(
+                    os.path.join(self.full_cache_path, f"heterographs{i}.pkl"), "rb"
+                ) as f:
+                    l = pickle.load(f)
+                    complex_graphs_all.extend(l)
+            with open(
+                os.path.join(self.full_cache_path, f"heterographs.pkl"), "wb"
+            ) as f:
+                pickle.dump((complex_graphs_all), f)
+
+            rdkit_ligands_all = []
+            for i in range(len(complex_names_all) // 1000 + 1):
+                with open(
+                    os.path.join(self.full_cache_path, f"rdkit_ligands{i}.pkl"), "rb"
+                ) as f:
+                    l = pickle.load(f)
+                    rdkit_ligands_all.extend(l)
+            with open(
+                os.path.join(self.full_cache_path, f"rdkit_ligands.pkl"), "wb"
+            ) as f:
+                pickle.dump((rdkit_ligands_all), f)
+        else:
+            complex_graphs, rdkit_ligands = [], []
+            with tqdm(total=len(complex_names_all), desc="loading complexes") as pbar:
+                for t in map(
+                    self.get_complex,
+                    zip(
+                        complex_names_all,
+                        lm_embeddings_chains_all,
+                        [None] * len(complex_names_all),
+                        [None] * len(complex_names_all),
+                    ),
+                ):
+                    complex_graphs.extend(t[0])
+                    rdkit_ligands.extend(t[1])
+                    pbar.update()
+            with open(
+                os.path.join(self.full_cache_path, "heterographs.pkl"), "wb"
+            ) as f:
+                pickle.dump((complex_graphs), f)
+            with open(
+                os.path.join(self.full_cache_path, "rdkit_ligands.pkl"), "wb"
+            ) as f:
+                pickle.dump((rdkit_ligands), f)
+
+    def inference_preprocessing(self):
+        ligands_list = []
+        print("Reading molecules and generating local structures with RDKit")
+        for ligand_description in tqdm(self.ligand_descriptions):
+            mol = MolFromSmiles(ligand_description)  # check if it is a smiles or a path
+            if mol is not None:
+                mol = AddHs(mol)
+                generate_conformer(mol)
+                ligands_list.append(mol)
+            else:
+                mol = read_molecule(ligand_description, remove_hs=False, sanitize=True)
+                if not self.keep_local_structures:
+                    mol.RemoveAllConformers()
+                    mol = AddHs(mol)
+                    generate_conformer(mol)
+                ligands_list.append(mol)
+
+        if self.esm_embeddings_path is not None:
+            print("Reading language model embeddings.")
+            lm_embeddings_chains_all = []
+            if not os.path.exists(self.esm_embeddings_path):
+                raise Exception(
+                    "ESM embeddings path does not exist: ", self.esm_embeddings_path
+                )
+            for protein_path in self.protein_path_list:
+                embeddings_paths = sorted(
+                    glob.glob(
+                        os.path.join(
+                            self.esm_embeddings_path, os.path.basename(protein_path)
+                        )
+                        + "*"
+                    )
+                )
+                lm_embeddings_chains = []
+                for embeddings_path in embeddings_paths:
+                    lm_embeddings_chains.append(
+                        torch.load(embeddings_path)["representations"][33]
+                    )
+                lm_embeddings_chains_all.append(lm_embeddings_chains)
+        else:
+            lm_embeddings_chains_all = [None] * len(self.protein_path_list)
+
+        print("Generating graphs for ligands and proteins")
+        if self.num_workers > 1:
+            # running preprocessing in parallel on multiple workers and saving the progress every 1000 complexes
+            for i in range(len(self.protein_path_list) // 1000 + 1):
+                if os.path.exists(
+                    os.path.join(self.full_cache_path, f"heterographs{i}.pkl")
+                ):
+                    continue
+                protein_paths_chunk = self.protein_path_list[1000 * i : 1000 * (i + 1)]
+                ligand_description_chunk = self.ligand_descriptions[
+                    1000 * i : 1000 * (i + 1)
+                ]
+                ligands_chunk = ligands_list[1000 * i : 1000 * (i + 1)]
+                lm_embeddings_chains = lm_embeddings_chains_all[
+                    1000 * i : 1000 * (i + 1)
+                ]
+                complex_graphs, rdkit_ligands = [], []
+                if self.num_workers > 1:
+                    p = Pool(self.num_workers, maxtasksperchild=1)
+                    p.__enter__()
+                with tqdm(
+                    total=len(protein_paths_chunk),
+                    desc=f"loading complexes {i}/{len(protein_paths_chunk)//1000+1}",
+                ) as pbar:
+                    map_fn = p.imap_unordered if self.num_workers > 1 else map
+                    for t in map_fn(
+                        self.get_complex,
+                        zip(
+                            protein_paths_chunk,
+                            lm_embeddings_chains,
+                            ligands_chunk,
+                            ligand_description_chunk,
+                        ),
+                    ):
+                        complex_graphs.extend(t[0])
+                        rdkit_ligands.extend(t[1])
+                        pbar.update()
+                if self.num_workers > 1:
+                    p.__exit__(None, None, None)
+
+                with open(
+                    os.path.join(self.full_cache_path, f"heterographs{i}.pkl"), "wb"
+                ) as f:
+                    pickle.dump((complex_graphs), f)
+                with open(
+                    os.path.join(self.full_cache_path, f"rdkit_ligands{i}.pkl"), "wb"
+                ) as f:
+                    pickle.dump((rdkit_ligands), f)
+
+            complex_graphs_all = []
+            for i in range(len(self.protein_path_list) // 1000 + 1):
+                with open(
+                    os.path.join(self.full_cache_path, f"heterographs{i}.pkl"), "rb"
+                ) as f:
+                    l = pickle.load(f)
+                    complex_graphs_all.extend(l)
+            with open(
+                os.path.join(self.full_cache_path, f"heterographs.pkl"), "wb"
+            ) as f:
+                pickle.dump((complex_graphs_all), f)
+
+            rdkit_ligands_all = []
+            for i in range(len(self.protein_path_list) // 1000 + 1):
+                with open(
+                    os.path.join(self.full_cache_path, f"rdkit_ligands{i}.pkl"), "rb"
+                ) as f:
+                    l = pickle.load(f)
+                    rdkit_ligands_all.extend(l)
+            with open(
+                os.path.join(self.full_cache_path, f"rdkit_ligands.pkl"), "wb"
+            ) as f:
+                pickle.dump((rdkit_ligands_all), f)
+        else:
+            complex_graphs, rdkit_ligands = [], []
+            with tqdm(
+                total=len(self.protein_path_list), desc="loading complexes"
+            ) as pbar:
+                for t in map(
+                    self.get_complex,
+                    zip(
+                        self.protein_path_list,
+                        lm_embeddings_chains_all,
+                        ligands_list,
+                        self.ligand_descriptions,
+                    ),
+                ):
+                    complex_graphs.extend(t[0])
+                    rdkit_ligands.extend(t[1])
+                    pbar.update()
+            with open(
+                os.path.join(self.full_cache_path, "heterographs.pkl"), "wb"
+            ) as f:
+                pickle.dump((complex_graphs), f)
+            with open(
+                os.path.join(self.full_cache_path, "rdkit_ligands.pkl"), "wb"
+            ) as f:
+                pickle.dump((rdkit_ligands), f)
+
+    def get_complex(self, par):
+        name, lm_embedding_chains, ligand, ligand_description = par
+        if not os.path.exists(os.path.join(self.pdbbind_dir, name)) and ligand is None:
+            print("Folder not found", name)
+            return [], []
+
+        if ligand is not None:
+            rec_model = parse_pdb_from_path(name)
+            name = f"{name}____{ligand_description}"
+            ligs = [ligand]
+        else:
+            try:
+                rec_model = parse_receptor(name, self.pdbbind_dir)
+            except Exception as e:
+                print(f"Skipping {name} because of the error:")
+                print(e)
+                return [], []
+
+            ligs = read_mols(self.pdbbind_dir, name, remove_hs=False)
+        complex_graphs = []
+        for i, lig in enumerate(ligs):
+            if (
+                self.max_lig_size is not None
+                and lig.GetNumHeavyAtoms() > self.max_lig_size
+            ):
+                print(
+                    f"Ligand with {lig.GetNumHeavyAtoms()} heavy atoms is larger than max_lig_size {self.max_lig_size}. Not including {name} in preprocessed data."
+                )
+                continue
+            complex_graph = HeteroData()
+            complex_graph["name"] = name
+            try:
+                get_lig_graph_with_matching(
+                    lig,
+                    complex_graph,
+                    self.popsize,
+                    self.maxiter,
+                    self.matching,
+                    self.keep_original,
+                    self.num_conformers,
+                    remove_hs=self.remove_hs,
+                )
+                print(lm_embedding_chains)
+                (
+                    rec,
+                    rec_coords,
+                    c_alpha_coords,
+                    n_coords,
+                    c_coords,
+                    lm_embeddings,
+                ) = extract_receptor_structure(
+                    copy.deepcopy(rec_model),
+                    lig,
+                    lm_embedding_chains=lm_embedding_chains,
+                )
+                if lm_embeddings is not None and len(c_alpha_coords) != len(
+                    lm_embeddings
+                ):
+                    print(
+                        f"LM embeddings for complex {name} did not have the right length for the protein. Skipping {name}."
+                    )
+                    continue
+
+                get_rec_graph(
+                    rec,
+                    rec_coords,
+                    c_alpha_coords,
+                    n_coords,
+                    c_coords,
+                    complex_graph,
+                    rec_radius=self.receptor_radius,
+                    c_alpha_max_neighbors=self.c_alpha_max_neighbors,
+                    all_atoms=self.all_atoms,
+                    atom_radius=self.atom_radius,
+                    atom_max_neighbors=self.atom_max_neighbors,
+                    remove_hs=self.remove_hs,
+                    lm_embeddings=lm_embeddings,
+                )
+
+            except Exception as e:
+                print(f"Skipping {name} because of the error:")
+                print(e)
+                raise e
+                continue
+
+            protein_center = torch.mean(
+                complex_graph["receptor"].pos, dim=0, keepdim=True
+            )
+            complex_graph["receptor"].pos -= protein_center
+            if self.all_atoms:
+                complex_graph["atom"].pos -= protein_center
+
+            if (not self.matching) or self.num_conformers == 1:
+                complex_graph["ligand"].pos -= protein_center
+            else:
+                for p in complex_graph["ligand"].pos:
+                    p -= protein_center
+
+            complex_graph.original_center = protein_center
+            complex_graphs.append(complex_graph)
+        return complex_graphs, ligs
+
+
+def print_statistics(complex_graphs):
+    statistics = ([], [], [], [])
+
+    for complex_graph in complex_graphs:
+        lig_pos = (
+            complex_graph["ligand"].pos
+            if torch.is_tensor(complex_graph["ligand"].pos)
+            else complex_graph["ligand"].pos[0]
+        )
+        radius_protein = torch.max(
+            torch.linalg.vector_norm(complex_graph["receptor"].pos, dim=1)
+        )
+        molecule_center = torch.mean(lig_pos, dim=0)
+        radius_molecule = torch.max(
+            torch.linalg.vector_norm(lig_pos - molecule_center.unsqueeze(0), dim=1)
+        )
+        distance_center = torch.linalg.vector_norm(molecule_center)
+        statistics[0].append(radius_protein)
+        statistics[1].append(radius_molecule)
+        statistics[2].append(distance_center)
+        if "rmsd_matching" in complex_graph:
+            statistics[3].append(complex_graph.rmsd_matching)
+        else:
+            statistics[3].append(0)
+
+    name = [
+        "radius protein",
+        "radius molecule",
+        "distance protein-mol",
+        "rmsd matching",
+    ]
+    print("Number of complexes: ", len(complex_graphs))
+    for i in range(4):
+        array = np.asarray(statistics[i])
+        print(
+            f"{name[i]}: mean {np.mean(array)}, std {np.std(array)}, max {np.max(array)}"
+        )
+
+
+def construct_loader(args, t_to_sigma):
+    transform = NoiseTransform(
+        t_to_sigma=t_to_sigma, no_torsion=args.no_torsion, all_atom=args.all_atoms
+    )
+
+    common_args = {
+        "transform": transform,
+        "root": args.data_dir,
+        "limit_complexes": args.limit_complexes,
+        "receptor_radius": args.receptor_radius,
+        "c_alpha_max_neighbors": args.c_alpha_max_neighbors,
+        "remove_hs": args.remove_hs,
+        "max_lig_size": args.max_lig_size,
+        "matching": not args.no_torsion,
+        "popsize": args.matching_popsize,
+        "maxiter": args.matching_maxiter,
+        "num_workers": args.num_workers,
+        "all_atoms": args.all_atoms,
+        "atom_radius": args.atom_radius,
+        "atom_max_neighbors": args.atom_max_neighbors,
+        "esm_embeddings_path": args.esm_embeddings_path,
+    }
+
+    train_dataset = PDBBind(
+        cache_path=args.cache_path,
+        split_path=args.split_train,
+        keep_original=True,
+        num_conformers=args.num_conformers,
+        **common_args,
+    )
+    val_dataset = PDBBind(
+        cache_path=args.cache_path,
+        split_path=args.split_val,
+        keep_original=True,
+        **common_args,
+    )
+
+    loader_class = DataListLoader if torch.cuda.is_available() else DataLoader
+    train_loader = loader_class(
+        dataset=train_dataset,
+        batch_size=args.batch_size,
+        num_workers=args.num_dataloader_workers,
+        shuffle=True,
+        pin_memory=args.pin_memory,
+    )
+    val_loader = loader_class(
+        dataset=val_dataset,
+        batch_size=args.batch_size,
+        num_workers=args.num_dataloader_workers,
+        shuffle=True,
+        pin_memory=args.pin_memory,
+    )
+
+    return train_loader, val_loader
+
+
+def read_mol(pdbbind_dir, name, remove_hs=False):
+    lig = read_molecule(
+        os.path.join(pdbbind_dir, name, f"{name}_ligand.sdf"),
+        remove_hs=remove_hs,
+        sanitize=True,
+    )
+    if lig is None:  # read mol2 file if sdf file cannot be sanitized
+        lig = read_molecule(
+            os.path.join(pdbbind_dir, name, f"{name}_ligand.mol2"),
+            remove_hs=remove_hs,
+            sanitize=True,
+        )
+    return lig
+
+
+def read_mols(pdbbind_dir, name, remove_hs=False):
+    ligs = []
+    for file in os.listdir(os.path.join(pdbbind_dir, name)):
+        if file.endswith(".sdf") and "rdkit" not in file:
+            lig = read_molecule(
+                os.path.join(pdbbind_dir, name, file),
+                remove_hs=remove_hs,
+                sanitize=True,
+            )
+            if lig is None and os.path.exists(
+                os.path.join(pdbbind_dir, name, file[:-4] + ".mol2")
+            ):  # read mol2 file if sdf file cannot be sanitized
+                print(
+                    "Using the .sdf file failed. We found a .mol2 file instead and are trying to use that."
+                )
+                lig = read_molecule(
+                    os.path.join(pdbbind_dir, name, file[:-4] + ".mol2"),
+                    remove_hs=remove_hs,
+                    sanitize=True,
+                )
+            if lig is not None:
+                ligs.append(lig)
+    return ligs

datasets/pdbbind_lm_embedding_preparation.py

@@ -0,0 +1,94 @@
+import os
+from argparse import FileType, ArgumentParser
+
+import numpy as np
+from Bio.PDB import PDBParser
+from Bio.Seq import Seq
+from Bio.SeqRecord import SeqRecord
+from tqdm import tqdm
+
+parser = ArgumentParser()
+parser.add_argument('--data_dir', type=str, default='data/PDBBind_processed', help='')
+parser.add_argument('--chain_cutoff', type=int, default=10, help='')
+parser.add_argument('--out_file', type=str, default="data/pdbbind_sequences.fasta")
+args = parser.parse_args()
+
+cutoff = args.chain_cutoff
+data_dir = args.data_dir
+names = os.listdir(data_dir)
+#%%
+from Bio import SeqIO
+biopython_parser = PDBParser()
+
+three_to_one = {'ALA':	'A',
+'ARG':	'R',
+'ASN':	'N',
+'ASP':	'D',
+'CYS':	'C',
+'GLN':	'Q',
+'GLU':	'E',
+'GLY':	'G',
+'HIS':	'H',
+'ILE':	'I',
+'LEU':	'L',
+'LYS':	'K',
+'MET':	'M',
+'MSE':  'M', # this is almost the same AA as MET. The sulfur is just replaced by Selen
+'PHE':	'F',
+'PRO':	'P',
+'PYL':	'O',
+'SER':	'S',
+'SEC':	'U',
+'THR':	'T',
+'TRP':	'W',
+'TYR':	'Y',
+'VAL':	'V',
+'ASX':	'B',
+'GLX':	'Z',
+'XAA':	'X',
+'XLE':	'J'}
+
+sequences = []
+ids = []
+for name in tqdm(names):
+    if name == '.DS_Store': continue
+    if os.path.exists(os.path.join(data_dir, name, f'{name}_protein_processed.pdb')):
+        rec_path = os.path.join(data_dir, name, f'{name}_protein_processed.pdb')
+    else:
+        rec_path = os.path.join(data_dir, name, f'{name}_protein.pdb')
+    if cutoff > 10:
+        rec_path = os.path.join(data_dir, name, f'{name}_protein_obabel_reduce.pdb')
+        if not os.path.exists(rec_path):
+            rec_path = os.path.join(data_dir, name, f'{name}_protein.pdb')
+    structure = biopython_parser.get_structure('random_id', rec_path)
+    structure = structure[0]
+    for i, chain in enumerate(structure):
+        seq = ''
+        for res_idx, residue in enumerate(chain):
+            if residue.get_resname() == 'HOH':
+                continue
+            residue_coords = []
+            c_alpha, n, c = None, None, None
+            for atom in residue:
+                if atom.name == 'CA':
+                    c_alpha = list(atom.get_vector())
+                if atom.name == 'N':
+                    n = list(atom.get_vector())
+                if atom.name == 'C':
+                    c = list(atom.get_vector())
+            if c_alpha != None and n != None and c != None:  # only append residue if it is an amino acid and not
+                try:
+                    seq += three_to_one[residue.get_resname()]
+                except Exception as e:
+                    seq += '-'
+                    print("encountered unknown AA: ", residue.get_resname(), ' in the complex ', name, '. Replacing it with a dash - .')
+        sequences.append(seq)
+        ids.append(f'{name}_chain_{i}')
+records = []
+for (index, seq) in zip(ids,sequences):
+    record = SeqRecord(Seq(seq), str(index))
+    record.description = ''
+    records.append(record)
+SeqIO.write(records, args.out_file, "fasta")
+
+

datasets/process_mols.py

@@ -0,0 +1,550 @@
+import copy
+import os
+import warnings
+
+import numpy as np
+import scipy.spatial as spa
+import torch
+from Bio.PDB import PDBParser
+from Bio.PDB.PDBExceptions import PDBConstructionWarning
+from rdkit import Chem
+from rdkit.Chem.rdchem import BondType as BT
+from rdkit.Chem import AllChem, GetPeriodicTable, RemoveHs
+from rdkit.Geometry import Point3D
+from scipy import spatial
+from scipy.special import softmax
+from torch_cluster import radius_graph
+
+
+import torch.nn.functional as F
+
+from datasets.conformer_matching import get_torsion_angles, optimize_rotatable_bonds
+from utils.torsion import get_transformation_mask
+
+
+biopython_parser = PDBParser()
+periodic_table = GetPeriodicTable()
+allowable_features = {
+    'possible_atomic_num_list': list(range(1, 119)) + ['misc'],
+    'possible_chirality_list': [
+        'CHI_UNSPECIFIED',
+        'CHI_TETRAHEDRAL_CW',
+        'CHI_TETRAHEDRAL_CCW',
+        'CHI_OTHER'
+    ],
+    'possible_degree_list': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 'misc'],
+    'possible_numring_list': [0, 1, 2, 3, 4, 5, 6, 'misc'],
+    'possible_implicit_valence_list': [0, 1, 2, 3, 4, 5, 6, 'misc'],
+    'possible_formal_charge_list': [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 'misc'],
+    'possible_numH_list': [0, 1, 2, 3, 4, 5, 6, 7, 8, 'misc'],
+    'possible_number_radical_e_list': [0, 1, 2, 3, 4, 'misc'],
+    'possible_hybridization_list': [
+        'SP', 'SP2', 'SP3', 'SP3D', 'SP3D2', 'misc'
+    ],
+    'possible_is_aromatic_list': [False, True],
+    'possible_is_in_ring3_list': [False, True],
+    'possible_is_in_ring4_list': [False, True],
+    'possible_is_in_ring5_list': [False, True],
+    'possible_is_in_ring6_list': [False, True],
+    'possible_is_in_ring7_list': [False, True],
+    'possible_is_in_ring8_list': [False, True],
+    'possible_amino_acids': ['ALA', 'ARG', 'ASN', 'ASP', 'CYS', 'GLN', 'GLU', 'GLY', 'HIS', 'ILE', 'LEU', 'LYS', 'MET',
+                             'PHE', 'PRO', 'SER', 'THR', 'TRP', 'TYR', 'VAL', 'HIP', 'HIE', 'TPO', 'HID', 'LEV', 'MEU',
+                             'PTR', 'GLV', 'CYT', 'SEP', 'HIZ', 'CYM', 'GLM', 'ASQ', 'TYS', 'CYX', 'GLZ', 'misc'],
+    'possible_atom_type_2': ['C*', 'CA', 'CB', 'CD', 'CE', 'CG', 'CH', 'CZ', 'N*', 'ND', 'NE', 'NH', 'NZ', 'O*', 'OD',
+                             'OE', 'OG', 'OH', 'OX', 'S*', 'SD', 'SG', 'misc'],
+    'possible_atom_type_3': ['C', 'CA', 'CB', 'CD', 'CD1', 'CD2', 'CE', 'CE1', 'CE2', 'CE3', 'CG', 'CG1', 'CG2', 'CH2',
+                             'CZ', 'CZ2', 'CZ3', 'N', 'ND1', 'ND2', 'NE', 'NE1', 'NE2', 'NH1', 'NH2', 'NZ', 'O', 'OD1',
+                             'OD2', 'OE1', 'OE2', 'OG', 'OG1', 'OH', 'OXT', 'SD', 'SG', 'misc'],
+}
+bonds = {BT.SINGLE: 0, BT.DOUBLE: 1, BT.TRIPLE: 2, BT.AROMATIC: 3}
+
+lig_feature_dims = (list(map(len, [
+    allowable_features['possible_atomic_num_list'],
+    allowable_features['possible_chirality_list'],
+    allowable_features['possible_degree_list'],
+    allowable_features['possible_formal_charge_list'],
+    allowable_features['possible_implicit_valence_list'],
+    allowable_features['possible_numH_list'],
+    allowable_features['possible_number_radical_e_list'],
+    allowable_features['possible_hybridization_list'],
+    allowable_features['possible_is_aromatic_list'],
+    allowable_features['possible_numring_list'],
+    allowable_features['possible_is_in_ring3_list'],
+    allowable_features['possible_is_in_ring4_list'],
+    allowable_features['possible_is_in_ring5_list'],
+    allowable_features['possible_is_in_ring6_list'],
+    allowable_features['possible_is_in_ring7_list'],
+    allowable_features['possible_is_in_ring8_list'],
+])), 0)  # number of scalar features
+
+rec_atom_feature_dims = (list(map(len, [
+    allowable_features['possible_amino_acids'],
+    allowable_features['possible_atomic_num_list'],
+    allowable_features['possible_atom_type_2'],
+    allowable_features['possible_atom_type_3'],
+])), 0)
+
+rec_residue_feature_dims = (list(map(len, [
+    allowable_features['possible_amino_acids']
+])), 0)
+
+
+def lig_atom_featurizer(mol):
+    ringinfo = mol.GetRingInfo()
+    atom_features_list = []
+    for idx, atom in enumerate(mol.GetAtoms()):
+        atom_features_list.append([
+            safe_index(allowable_features['possible_atomic_num_list'], atom.GetAtomicNum()),
+            allowable_features['possible_chirality_list'].index(str(atom.GetChiralTag())),
+            safe_index(allowable_features['possible_degree_list'], atom.GetTotalDegree()),
+            safe_index(allowable_features['possible_formal_charge_list'], atom.GetFormalCharge()),
+            safe_index(allowable_features['possible_implicit_valence_list'], atom.GetImplicitValence()),
+            safe_index(allowable_features['possible_numH_list'], atom.GetTotalNumHs()),
+            safe_index(allowable_features['possible_number_radical_e_list'], atom.GetNumRadicalElectrons()),
+            safe_index(allowable_features['possible_hybridization_list'], str(atom.GetHybridization())),
+            allowable_features['possible_is_aromatic_list'].index(atom.GetIsAromatic()),
+            safe_index(allowable_features['possible_numring_list'], ringinfo.NumAtomRings(idx)),
+            allowable_features['possible_is_in_ring3_list'].index(ringinfo.IsAtomInRingOfSize(idx, 3)),
+            allowable_features['possible_is_in_ring4_list'].index(ringinfo.IsAtomInRingOfSize(idx, 4)),
+            allowable_features['possible_is_in_ring5_list'].index(ringinfo.IsAtomInRingOfSize(idx, 5)),
+            allowable_features['possible_is_in_ring6_list'].index(ringinfo.IsAtomInRingOfSize(idx, 6)),
+            allowable_features['possible_is_in_ring7_list'].index(ringinfo.IsAtomInRingOfSize(idx, 7)),
+            allowable_features['possible_is_in_ring8_list'].index(ringinfo.IsAtomInRingOfSize(idx, 8)),
+        ])
+
+    return torch.tensor(atom_features_list)
+
+
+def rec_residue_featurizer(rec):
+    feature_list = []
+    for residue in rec.get_residues():
+        feature_list.append([safe_index(allowable_features['possible_amino_acids'], residue.get_resname())])
+    return torch.tensor(feature_list, dtype=torch.float32)  # (N_res, 1)
+
+
+def safe_index(l, e):
+    """ Return index of element e in list l. If e is not present, return the last index """
+    try:
+        return l.index(e)
+    except:
+        return len(l) - 1
+
+
+
+def parse_receptor(pdbid, pdbbind_dir):
+    rec = parsePDB(pdbid, pdbbind_dir)
+    return rec
+
+
+def parsePDB(pdbid, pdbbind_dir):
+    rec_path = os.path.join(pdbbind_dir, pdbid, f'{pdbid}_protein_processed.pdb')
+    return parse_pdb_from_path(rec_path)
+
+def parse_pdb_from_path(path):
+    with warnings.catch_warnings():
+        warnings.filterwarnings("ignore", category=PDBConstructionWarning)
+        structure = biopython_parser.get_structure('random_id', path)
+        rec = structure[0]
+    return rec
+
+
+def extract_receptor_structure(rec, lig, lm_embedding_chains=None):
+    conf = lig.GetConformer()
+    lig_coords = conf.GetPositions()
+    min_distances = []
+    coords = []
+    c_alpha_coords = []
+    n_coords = []
+    c_coords = []
+    valid_chain_ids = []
+    lengths = []
+    for i, chain in enumerate(rec):
+        chain_coords = []  # num_residues, num_atoms, 3
+        chain_c_alpha_coords = []
+        chain_n_coords = []
+        chain_c_coords = []
+        count = 0
+        invalid_res_ids = []
+        for res_idx, residue in enumerate(chain):
+            if residue.get_resname() == 'HOH':
+                invalid_res_ids.append(residue.get_id())
+                continue
+            residue_coords = []
+            c_alpha, n, c = None, None, None
+            for atom in residue:
+                if atom.name == 'CA':
+                    c_alpha = list(atom.get_vector())
+                if atom.name == 'N':
+                    n = list(atom.get_vector())
+                if atom.name == 'C':
+                    c = list(atom.get_vector())
+                residue_coords.append(list(atom.get_vector()))
+
+            if c_alpha != None and n != None and c != None:
+                # only append residue if it is an amino acid and not some weird molecule that is part of the complex
+                chain_c_alpha_coords.append(c_alpha)
+                chain_n_coords.append(n)
+                chain_c_coords.append(c)
+                chain_coords.append(np.array(residue_coords))
+                count += 1
+            else:
+                invalid_res_ids.append(residue.get_id())
+        for res_id in invalid_res_ids:
+            chain.detach_child(res_id)
+        if len(chain_coords) > 0:
+            all_chain_coords = np.concatenate(chain_coords, axis=0)
+            distances = spatial.distance.cdist(lig_coords, all_chain_coords)
+            min_distance = distances.min()
+        else:
+            min_distance = np.inf
+
+        min_distances.append(min_distance)
+        lengths.append(count)
+        coords.append(chain_coords)
+        c_alpha_coords.append(np.array(chain_c_alpha_coords))
+        n_coords.append(np.array(chain_n_coords))
+        c_coords.append(np.array(chain_c_coords))
+        if not count == 0: valid_chain_ids.append(chain.get_id())
+
+    min_distances = np.array(min_distances)
+    if len(valid_chain_ids) == 0:
+        valid_chain_ids.append(np.argmin(min_distances))
+    valid_coords = []
+    valid_c_alpha_coords = []
+    valid_n_coords = []
+    valid_c_coords = []
+    valid_lengths = []
+    invalid_chain_ids = []
+    valid_lm_embeddings = []
+    for i, chain in enumerate(rec):
+        if chain.get_id() in valid_chain_ids:
+            valid_coords.append(coords[i])
+            valid_c_alpha_coords.append(c_alpha_coords[i])
+            if lm_embedding_chains is not None:
+                if i >= len(lm_embedding_chains):
+                    raise ValueError('Encountered valid chain id that was not present in the LM embeddings')
+                valid_lm_embeddings.append(lm_embedding_chains[i])
+            valid_n_coords.append(n_coords[i])
+            valid_c_coords.append(c_coords[i])
+            valid_lengths.append(lengths[i])
+        else:
+            invalid_chain_ids.append(chain.get_id())
+    coords = [item for sublist in valid_coords for item in sublist]  # list with n_residues arrays: [n_atoms, 3]
+
+    c_alpha_coords = np.concatenate(valid_c_alpha_coords, axis=0)  # [n_residues, 3]
+    n_coords = np.concatenate(valid_n_coords, axis=0)  # [n_residues, 3]
+    c_coords = np.concatenate(valid_c_coords, axis=0)  # [n_residues, 3]
+    lm_embeddings = np.concatenate(valid_lm_embeddings, axis=0) if lm_embedding_chains is not None else None
+    for invalid_id in invalid_chain_ids:
+        rec.detach_child(invalid_id)
+
+    assert len(c_alpha_coords) == len(n_coords)
+    assert len(c_alpha_coords) == len(c_coords)
+    assert sum(valid_lengths) == len(c_alpha_coords)
+    return rec, coords, c_alpha_coords, n_coords, c_coords, lm_embeddings
+
+
+def get_lig_graph(mol, complex_graph):
+    lig_coords = torch.from_numpy(mol.GetConformer().GetPositions()).float()
+    atom_feats = lig_atom_featurizer(mol)
+
+    row, col, edge_type = [], [], []
+    for bond in mol.GetBonds():
+        start, end = bond.GetBeginAtomIdx(), bond.GetEndAtomIdx()
+        row += [start, end]
+        col += [end, start]
+        edge_type += 2 * [bonds[bond.GetBondType()]] if bond.GetBondType() != BT.UNSPECIFIED else [0, 0]
+
+    edge_index = torch.tensor([row, col], dtype=torch.long)
+    edge_type = torch.tensor(edge_type, dtype=torch.long)
+    edge_attr = F.one_hot(edge_type, num_classes=len(bonds)).to(torch.float)
+
+    complex_graph['ligand'].x = atom_feats
+    complex_graph['ligand'].pos = lig_coords
+    complex_graph['ligand', 'lig_bond', 'ligand'].edge_index = edge_index
+    complex_graph['ligand', 'lig_bond', 'ligand'].edge_attr = edge_attr
+    return
+
+def generate_conformer(mol):
+    ps = AllChem.ETKDGv2()
+    id = AllChem.EmbedMolecule(mol, ps)
+    if id == -1:
+        print('rdkit coords could not be generated without using random coords. using random coords now.')
+        ps.useRandomCoords = True
+        AllChem.EmbedMolecule(mol, ps)
+        AllChem.MMFFOptimizeMolecule(mol, confId=0)
+    # else:
+    #    AllChem.MMFFOptimizeMolecule(mol_rdkit, confId=0)
+
+def get_lig_graph_with_matching(mol_, complex_graph, popsize, maxiter, matching, keep_original, num_conformers, remove_hs):
+    if matching:
+        mol_maybe_noh = copy.deepcopy(mol_)
+        if remove_hs:
+            mol_maybe_noh = RemoveHs(mol_maybe_noh, sanitize=True)
+        if keep_original:
+            complex_graph['ligand'].orig_pos = mol_maybe_noh.GetConformer().GetPositions()
+
+        rotable_bonds = get_torsion_angles(mol_maybe_noh)
+        if not rotable_bonds: print("no_rotable_bonds but still using it")
+
+        for i in range(num_conformers):
+            mol_rdkit = copy.deepcopy(mol_)
+
+            mol_rdkit.RemoveAllConformers()
+            mol_rdkit = AllChem.AddHs(mol_rdkit)
+            generate_conformer(mol_rdkit)
+            if remove_hs:
+                mol_rdkit = RemoveHs(mol_rdkit, sanitize=True)
+            mol = copy.deepcopy(mol_maybe_noh)
+            if rotable_bonds:
+                optimize_rotatable_bonds(mol_rdkit, mol, rotable_bonds, popsize=popsize, maxiter=maxiter)
+            mol.AddConformer(mol_rdkit.GetConformer())
+            rms_list = []
+            AllChem.AlignMolConformers(mol, RMSlist=rms_list)
+            mol_rdkit.RemoveAllConformers()
+            mol_rdkit.AddConformer(mol.GetConformers()[1])
+
+            if i == 0:
+                complex_graph.rmsd_matching = rms_list[0]
+                get_lig_graph(mol_rdkit, complex_graph)
+            else:
+                if torch.is_tensor(complex_graph['ligand'].pos):
+                    complex_graph['ligand'].pos = [complex_graph['ligand'].pos]
+                complex_graph['ligand'].pos.append(torch.from_numpy(mol_rdkit.GetConformer().GetPositions()).float())
+
+    else:  # no matching
+        complex_graph.rmsd_matching = 0
+        if remove_hs: mol_ = RemoveHs(mol_)
+        get_lig_graph(mol_, complex_graph)
+
+    edge_mask, mask_rotate = get_transformation_mask(complex_graph)
+    complex_graph['ligand'].edge_mask = torch.tensor(edge_mask)
+    complex_graph['ligand'].mask_rotate = mask_rotate
+
+    return
+
+
+def get_calpha_graph(rec, c_alpha_coords, n_coords, c_coords, complex_graph, cutoff=20, max_neighbor=None, lm_embeddings=None):
+    n_rel_pos = n_coords - c_alpha_coords
+    c_rel_pos = c_coords - c_alpha_coords
+    num_residues = len(c_alpha_coords)
+    if num_residues <= 1:
+        raise ValueError(f"rec contains only 1 residue!")
+
+    # Build the k-NN graph
+    distances = spa.distance.cdist(c_alpha_coords, c_alpha_coords)
+    src_list = []
+    dst_list = []
+    mean_norm_list = []
+    for i in range(num_residues):
+        dst = list(np.where(distances[i, :] < cutoff)[0])
+        dst.remove(i)
+        if max_neighbor != None and len(dst) > max_neighbor:
+            dst = list(np.argsort(distances[i, :]))[1: max_neighbor + 1]
+        if len(dst) == 0:
+            dst = list(np.argsort(distances[i, :]))[1:2]  # choose second because first is i itself
+            print(f'The c_alpha_cutoff {cutoff} was too small for one c_alpha such that it had no neighbors. '
+                  f'So we connected it to the closest other c_alpha')
+        assert i not in dst
+        src = [i] * len(dst)
+        src_list.extend(src)
+        dst_list.extend(dst)
+        valid_dist = list(distances[i, dst])
+        valid_dist_np = distances[i, dst]
+        sigma = np.array([1., 2., 5., 10., 30.]).reshape((-1, 1))
+        weights = softmax(- valid_dist_np.reshape((1, -1)) ** 2 / sigma, axis=1)  # (sigma_num, neigh_num)
+        assert weights[0].sum() > 1 - 1e-2 and weights[0].sum() < 1.01
+        diff_vecs = c_alpha_coords[src, :] - c_alpha_coords[dst, :]  # (neigh_num, 3)
+        mean_vec = weights.dot(diff_vecs)  # (sigma_num, 3)
+        denominator = weights.dot(np.linalg.norm(diff_vecs, axis=1))  # (sigma_num,)
+        mean_vec_ratio_norm = np.linalg.norm(mean_vec, axis=1) / denominator  # (sigma_num,)
+        mean_norm_list.append(mean_vec_ratio_norm)
+    assert len(src_list) == len(dst_list)
+
+    node_feat = rec_residue_featurizer(rec)
+    mu_r_norm = torch.from_numpy(np.array(mean_norm_list).astype(np.float32))
+    side_chain_vecs = torch.from_numpy(
+        np.concatenate([np.expand_dims(n_rel_pos, axis=1), np.expand_dims(c_rel_pos, axis=1)], axis=1))
+
+    complex_graph['receptor'].x = torch.cat([node_feat, torch.tensor(lm_embeddings)], axis=1) if lm_embeddings is not None else node_feat
+    complex_graph['receptor'].pos = torch.from_numpy(c_alpha_coords).float()
+    complex_graph['receptor'].mu_r_norm = mu_r_norm
+    complex_graph['receptor'].side_chain_vecs = side_chain_vecs.float()
+    complex_graph['receptor', 'rec_contact', 'receptor'].edge_index = torch.from_numpy(np.asarray([src_list, dst_list]))
+
+    return
+
+
+def rec_atom_featurizer(rec):
+    atom_feats = []
+    for i, atom in enumerate(rec.get_atoms()):
+        atom_name, element = atom.name, atom.element
+        if element == 'CD':
+            element = 'C'
+        assert not element == ''
+        try:
+            atomic_num = periodic_table.GetAtomicNumber(element)
+        except:
+            atomic_num = -1
+        atom_feat = [safe_index(allowable_features['possible_amino_acids'], atom.get_parent().get_resname()),
+                     safe_index(allowable_features['possible_atomic_num_list'], atomic_num),
+                     safe_index(allowable_features['possible_atom_type_2'], (atom_name + '*')[:2]),
+                     safe_index(allowable_features['possible_atom_type_3'], atom_name)]
+        atom_feats.append(atom_feat)
+
+    return atom_feats
+
+
+def get_rec_graph(rec, rec_coords, c_alpha_coords, n_coords, c_coords, complex_graph, rec_radius, c_alpha_max_neighbors=None, all_atoms=False,
+                  atom_radius=5, atom_max_neighbors=None, remove_hs=False, lm_embeddings=None):
+    if all_atoms:
+        return get_fullrec_graph(rec, rec_coords, c_alpha_coords, n_coords, c_coords, complex_graph,
+                                 c_alpha_cutoff=rec_radius, c_alpha_max_neighbors=c_alpha_max_neighbors,
+                                 atom_cutoff=atom_radius, atom_max_neighbors=atom_max_neighbors, remove_hs=remove_hs,lm_embeddings=lm_embeddings)
+    else:
+        return get_calpha_graph(rec, c_alpha_coords, n_coords, c_coords, complex_graph, rec_radius, c_alpha_max_neighbors,lm_embeddings=lm_embeddings)
+
+
+def get_fullrec_graph(rec, rec_coords, c_alpha_coords, n_coords, c_coords, complex_graph, c_alpha_cutoff=20,
+                      c_alpha_max_neighbors=None, atom_cutoff=5, atom_max_neighbors=None, remove_hs=False, lm_embeddings=None):
+    # builds the receptor graph with both residues and atoms
+
+    n_rel_pos = n_coords - c_alpha_coords
+    c_rel_pos = c_coords - c_alpha_coords
+    num_residues = len(c_alpha_coords)
+    if num_residues <= 1:
+        raise ValueError(f"rec contains only 1 residue!")
+
+    # Build the k-NN graph of residues
+    distances = spa.distance.cdist(c_alpha_coords, c_alpha_coords)
+    src_list = []
+    dst_list = []
+    mean_norm_list = []
+    for i in range(num_residues):
+        dst = list(np.where(distances[i, :] < c_alpha_cutoff)[0])
+        dst.remove(i)
+        if c_alpha_max_neighbors != None and len(dst) > c_alpha_max_neighbors:
+            dst = list(np.argsort(distances[i, :]))[1: c_alpha_max_neighbors + 1]
+        if len(dst) == 0:
+            dst = list(np.argsort(distances[i, :]))[1:2]  # choose second because first is i itself
+            print(f'The c_alpha_cutoff {c_alpha_cutoff} was too small for one c_alpha such that it had no neighbors. '
+                  f'So we connected it to the closest other c_alpha')
+        assert i not in dst
+        src = [i] * len(dst)
+        src_list.extend(src)
+        dst_list.extend(dst)
+        valid_dist = list(distances[i, dst])
+        valid_dist_np = distances[i, dst]
+        sigma = np.array([1., 2., 5., 10., 30.]).reshape((-1, 1))
+        weights = softmax(- valid_dist_np.reshape((1, -1)) ** 2 / sigma, axis=1)  # (sigma_num, neigh_num)
+        assert 1 - 1e-2 < weights[0].sum() < 1.01
+        diff_vecs = c_alpha_coords[src, :] - c_alpha_coords[dst, :]  # (neigh_num, 3)
+        mean_vec = weights.dot(diff_vecs)  # (sigma_num, 3)
+        denominator = weights.dot(np.linalg.norm(diff_vecs, axis=1))  # (sigma_num,)
+        mean_vec_ratio_norm = np.linalg.norm(mean_vec, axis=1) / denominator  # (sigma_num,)
+        mean_norm_list.append(mean_vec_ratio_norm)
+    assert len(src_list) == len(dst_list)
+
+    node_feat = rec_residue_featurizer(rec)
+    mu_r_norm = torch.from_numpy(np.array(mean_norm_list).astype(np.float32))
+    side_chain_vecs = torch.from_numpy(
+        np.concatenate([np.expand_dims(n_rel_pos, axis=1), np.expand_dims(c_rel_pos, axis=1)], axis=1))
+
+    complex_graph['receptor'].x = torch.cat([node_feat, torch.tensor(lm_embeddings)], axis=1) if lm_embeddings is not None else node_feat
+    complex_graph['receptor'].pos = torch.from_numpy(c_alpha_coords).float()
+    complex_graph['receptor'].mu_r_norm = mu_r_norm
+    complex_graph['receptor'].side_chain_vecs = side_chain_vecs.float()
+    complex_graph['receptor', 'rec_contact', 'receptor'].edge_index = torch.from_numpy(np.asarray([src_list, dst_list]))
+
+    src_c_alpha_idx = np.concatenate([np.asarray([i]*len(l)) for i, l in enumerate(rec_coords)])
+    atom_feat = torch.from_numpy(np.asarray(rec_atom_featurizer(rec)))
+    atom_coords = torch.from_numpy(np.concatenate(rec_coords, axis=0)).float()
+
+    if remove_hs:
+        not_hs = (atom_feat[:, 1] != 0)
+        src_c_alpha_idx = src_c_alpha_idx[not_hs]
+        atom_feat = atom_feat[not_hs]
+        atom_coords = atom_coords[not_hs]
+
+    atoms_edge_index = radius_graph(atom_coords, atom_cutoff, max_num_neighbors=atom_max_neighbors if atom_max_neighbors else 1000)
+    atom_res_edge_index = torch.from_numpy(np.asarray([np.arange(len(atom_feat)), src_c_alpha_idx])).long()
+
+    complex_graph['atom'].x = atom_feat
+    complex_graph['atom'].pos = atom_coords
+    complex_graph['atom', 'atom_contact', 'atom'].edge_index = atoms_edge_index
+    complex_graph['atom', 'atom_rec_contact', 'receptor'].edge_index = atom_res_edge_index
+
+    return
+
+def write_mol_with_coords(mol, new_coords, path):
+    w = Chem.SDWriter(path)
+    conf = mol.GetConformer()
+    for i in range(mol.GetNumAtoms()):
+        x,y,z = new_coords.astype(np.double)[i]
+        conf.SetAtomPosition(i,Point3D(x,y,z))
+    w.write(mol)
+    w.close()
+
+def read_molecule(molecule_file, sanitize=False, calc_charges=False, remove_hs=False):
+    if molecule_file.endswith('.mol2'):
+        mol = Chem.MolFromMol2File(molecule_file, sanitize=False, removeHs=False)
+    elif molecule_file.endswith('.sdf'):
+        print(molecule_file)
+        supplier = Chem.SDMolSupplier(molecule_file, sanitize=False, removeHs=False)
+        mol = supplier[0]
+        print(mol)
+    elif molecule_file.endswith('.pdbqt'):
+        with open(molecule_file) as file:
+            pdbqt_data = file.readlines()
+        pdb_block = ''
+        for line in pdbqt_data:
+            pdb_block += '{}\n'.format(line[:66])
+        mol = Chem.MolFromPDBBlock(pdb_block, sanitize=False, removeHs=False)
+    elif molecule_file.endswith('.pdb'):
+        mol = Chem.MolFromPDBFile(molecule_file, sanitize=False, removeHs=False)
+    else:
+        return ValueError('Expect the format of the molecule_file to be '
+                          'one of .mol2, .sdf, .pdbqt and .pdb, got {}'.format(molecule_file))
+
+    print(sanitize, calc_charges, remove_hs)
+
+    try:
+        if sanitize or calc_charges:
+            Chem.SanitizeMol(mol)
+
+        if calc_charges:
+            # Compute Gasteiger charges on the molecule.
+            try:
+                AllChem.ComputeGasteigerCharges(mol)
+            except:
+                warnings.warn('Unable to compute charges for the molecule.')
+
+        if remove_hs:
+            mol = Chem.RemoveHs(mol, sanitize=sanitize)
+    except Exception as e:
+        print(e)
+        return None
+
+    return mol
+
+
+def read_sdf_or_mol2(sdf_fileName, mol2_fileName):
+
+    mol = Chem.MolFromMolFile(sdf_fileName, sanitize=False)
+    problem = False
+    try:
+        Chem.SanitizeMol(mol)
+        mol = Chem.RemoveHs(mol)
+    except Exception as e:
+        problem = True
+    if problem:
+        mol = Chem.MolFromMol2File(mol2_fileName, sanitize=False)
+        try:
+            Chem.SanitizeMol(mol)
+            mol = Chem.RemoveHs(mol)
+            problem = False
+        except Exception as e:
+            problem = True
+
+    return mol, problem

environment.yml

@@ -0,0 +1,102 @@
+name: diffdock
+channels:
+  - pytorch
+  - defaults
+dependencies:
+  - blas=1.0
+  - brotlipy=0.7.0
+  - bzip2=1.0.8
+  - ca-certificates=2022.07.19
+  - certifi=2022.9.14
+  - cffi=1.15.1
+  - charset-normalizer=2.0.4
+  - cryptography=37.0.1
+  - ffmpeg=4.3
+  - freetype=2.11.0
+  - gettext=0.21.0
+  - giflib=5.2.1
+  - gmp=6.2.1
+  - gnutls=3.6.15
+  - icu=58.2
+  - idna=3.3
+  - intel-openmp=2021.4.0
+  - jpeg=9e
+  - lame=3.100
+  - lcms2=2.12
+  - lerc=3.0
+  - libcxx=14.0.6
+  - libdeflate=1.8
+  - libffi=3.3
+  - libiconv=1.16
+  - libidn2=2.3.2
+  - libpng=1.6.37
+  - libtasn1=4.16.0
+  - libtiff=4.4.0
+  - libunistring=0.9.10
+  - libwebp=1.2.2
+  - libwebp-base=1.2.2
+  - libxml2=2.9.14
+  - llvm-openmp=14.0.6
+  - lz4-c=1.9.3
+  - mkl=2021.4.0
+  - mkl-service=2.4.0
+  - mkl_fft=1.3.1
+  - mkl_random=1.2.2
+  - ncurses=6.3
+  - nettle=3.7.3
+  - numpy=1.23.1
+  - numpy-base=1.23.1
+  - openh264=2.1.1
+  - openssl=1.1.1q
+  - pillow=9.2.0
+  - pip=22.2.2
+  - pycparser=2.21
+  - pyopenssl=22.0.0
+  - pysocks=1.7.1
+  - python=3.9.13
+  - pytorch=1.12.1
+  - readline=8.1.2
+  - requests=2.28.1
+  - setuptools=63.4.1
+  - six=1.16.0
+  - sqlite=3.39.3
+  - tk=8.6.12
+  - torchaudio=0.12.1
+  - torchvision=0.13.1
+  - typing_extensions=4.3.0
+  - tzdata=2022c
+  - urllib3=1.26.11
+  - wheel=0.37.1
+  - xz=5.2.6
+  - zlib=1.2.12
+  - zstd=1.5.2
+  - pip:
+    - biopandas==0.4.1
+    - biopython==1.79
+    - e3nn==0.5.0
+    - jinja2==3.1.2
+    - joblib==1.2.0
+    - markupsafe==2.1.1
+    - mpmath==1.2.1
+    - networkx==2.8.7
+    - opt-einsum==3.3.0
+    - opt-einsum-fx==0.1.4
+    - packaging==21.3
+    - pandas==1.5.0
+    - pyaml==21.10.1
+    - pyparsing==3.0.9
+    - python-dateutil==2.8.2
+    - pytz==2022.4
+    - pyyaml==6.0
+    - rdkit-pypi==2022.3.5
+    - scikit-learn==1.1.2
+    - scipy==1.9.1
+    - spyrmsd==0.5.2
+    - sympy==1.11.1
+    - threadpoolctl==3.1.0
+    - torch-cluster==1.6.0
+    - torch-geometric==2.1.0.post1
+    - torch-scatter==2.0.9
+    - torch-sparse==0.6.15
+    - torch-spline-conv==1.2.1
+    - tqdm==4.64.1

esm/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Facebook, Inc. and its affiliates.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

esm/esm/__init__.py

@@ -0,0 +1,12 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from .version import version as __version__  # noqa
+
+from .data import Alphabet, BatchConverter, FastaBatchedDataset  # noqa
+from .model.esm1 import ProteinBertModel  # noqa
+from .model.esm2 import ESM2  # noqa
+from .model.msa_transformer import MSATransformer  #noqa
+from . import pretrained  # noqa

esm/esm/axial_attention.py

@@ -0,0 +1,239 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import math
+import torch
+import torch.nn as nn
+
+
+class RowSelfAttention(nn.Module):
+    """Compute self-attention over rows of a 2D input."""
+
+    def __init__(
+        self,
+        embed_dim,
+        num_heads,
+        dropout=0.0,
+        max_tokens_per_msa: int = 2 ** 16,
+    ):
+        super().__init__()
+        self.num_heads = num_heads
+        self.dropout = dropout
+        self.head_dim = embed_dim // num_heads
+        self.scaling = self.head_dim ** -0.5
+        self.max_tokens_per_msa = max_tokens_per_msa
+        self.attn_shape = "hnij"
+
+        self.k_proj = nn.Linear(embed_dim, embed_dim)
+        self.v_proj = nn.Linear(embed_dim, embed_dim)
+        self.q_proj = nn.Linear(embed_dim, embed_dim)
+
+        self.out_proj = nn.Linear(embed_dim, embed_dim)
+        self.dropout_module = nn.Dropout(dropout)
+
+    def align_scaling(self, q):
+        num_rows = q.size(0)
+        return self.scaling / math.sqrt(num_rows)
+
+    def _batched_forward(
+        self,
+        x,
+        self_attn_mask=None,
+        self_attn_padding_mask=None,
+    ):
+        num_rows, num_cols, batch_size, embed_dim = x.size()
+        max_rows = max(1, self.max_tokens_per_msa // num_cols)
+        attns = 0
+        scaling = self.align_scaling(x)
+        for start in range(0, num_rows, max_rows):
+            attn_weights = self.compute_attention_weights(
+                x[start : start + max_rows],
+                scaling,
+                self_attn_mask=self_attn_mask,
+                self_attn_padding_mask=self_attn_padding_mask[:, start : start + max_rows]
+                if self_attn_padding_mask is not None
+                else None,
+            )
+            attns += attn_weights
+        attn_probs = attns.softmax(-1)
+        attn_probs = self.dropout_module(attn_probs)
+
+        outputs = []
+        for start in range(0, num_rows, max_rows):
+            output = self.compute_attention_update(x[start : start + max_rows], attn_probs)
+            outputs.append(output)
+
+        output = torch.cat(outputs, 0)
+        return output, attn_probs
+
+    def compute_attention_weights(
+        self,
+        x,
+        scaling: float,
+        self_attn_mask=None,
+        self_attn_padding_mask=None,
+    ):
+        num_rows, num_cols, batch_size, embed_dim = x.size()
+        q = self.q_proj(x).view(num_rows, num_cols, batch_size, self.num_heads, self.head_dim)
+        k = self.k_proj(x).view(num_rows, num_cols, batch_size, self.num_heads, self.head_dim)
+        q *= scaling
+        if self_attn_padding_mask is not None:
+            # Zero out any padded aligned positions - this is important since
+            # we take a sum across the alignment axis.
+            q *= 1 - self_attn_padding_mask.permute(1, 2, 0).unsqueeze(3).unsqueeze(4).to(q)
+
+        attn_weights = torch.einsum(f"rinhd,rjnhd->{self.attn_shape}", q, k)
+
+        if self_attn_mask is not None:
+            raise NotImplementedError
+            # Mask Size: [B x R x C], Weights Size: [H x B x C x C]
+
+        if self_attn_padding_mask is not None:
+            attn_weights = attn_weights.masked_fill(
+                self_attn_padding_mask[:, 0].unsqueeze(0).unsqueeze(2),
+                -10000,
+            )
+
+        return attn_weights
+
+    def compute_attention_update(
+        self,
+        x,
+        attn_probs,
+    ):
+        num_rows, num_cols, batch_size, embed_dim = x.size()
+        v = self.v_proj(x).view(num_rows, num_cols, batch_size, self.num_heads, self.head_dim)
+        context = torch.einsum(f"{self.attn_shape},rjnhd->rinhd", attn_probs, v)
+        context = context.contiguous().view(num_rows, num_cols, batch_size, embed_dim)
+        output = self.out_proj(context)
+        return output
+
+    def forward(
+        self,
+        x,
+        self_attn_mask=None,
+        self_attn_padding_mask=None,
+    ):
+        num_rows, num_cols, batch_size, embed_dim = x.size()
+        if (num_rows * num_cols > self.max_tokens_per_msa) and not torch.is_grad_enabled():
+            return self._batched_forward(x, self_attn_mask, self_attn_padding_mask)
+        else:
+            scaling = self.align_scaling(x)
+            attn_weights = self.compute_attention_weights(
+                x, scaling, self_attn_mask, self_attn_padding_mask
+            )
+            attn_probs = attn_weights.softmax(-1)
+            attn_probs = self.dropout_module(attn_probs)
+            output = self.compute_attention_update(x, attn_probs)
+            return output, attn_probs
+
+
+class ColumnSelfAttention(nn.Module):
+    """Compute self-attention over columns of a 2D input."""
+
+    def __init__(
+        self,
+        embed_dim,
+        num_heads,
+        dropout=0.0,
+        max_tokens_per_msa: int = 2 ** 16,
+    ):
+        super().__init__()
+
+        self.num_heads = num_heads
+        self.dropout = dropout
+        self.head_dim = embed_dim // num_heads
+        self.scaling = self.head_dim ** -0.5
+        self.max_tokens_per_msa = max_tokens_per_msa
+
+        self.k_proj = nn.Linear(embed_dim, embed_dim)
+        self.v_proj = nn.Linear(embed_dim, embed_dim)
+        self.q_proj = nn.Linear(embed_dim, embed_dim)
+
+        self.out_proj = nn.Linear(embed_dim, embed_dim)
+        self.dropout_module = nn.Dropout(dropout)
+
+    def _batched_forward(
+        self,
+        x,
+        self_attn_mask=None,
+        self_attn_padding_mask=None,
+    ):
+        num_rows, num_cols, batch_size, embed_dim = x.size()
+        max_cols = max(1, self.max_tokens_per_msa // num_rows)
+        outputs = []
+        attns = []
+        for start in range(0, num_cols, max_cols):
+            output, attn = self(
+                x[:, start : start + max_cols],
+                self_attn_mask=self_attn_mask,
+                self_attn_padding_mask=self_attn_padding_mask[:, :, start : start + max_cols]
+                if self_attn_padding_mask is not None
+                else None,
+            )
+            outputs.append(output)
+            attns.append(attn)
+        output = torch.cat(outputs, 1)
+        attns = torch.cat(attns, 1)
+        return output, attns
+
+    def compute_attention_update(
+        self,
+        x,
+        self_attn_mask=None,
+        self_attn_padding_mask=None,
+    ):
+        num_rows, num_cols, batch_size, embed_dim = x.size()
+        if num_rows == 1:
+            # if there is only 1 position, this is equivalent and doesn't break with padding
+            attn_probs = torch.ones(
+                self.num_heads,
+                num_cols,
+                batch_size,
+                num_rows,
+                num_rows,
+                device=x.device,
+                dtype=x.dtype,
+            )
+            output = self.out_proj(self.v_proj(x))
+        else:
+            q = self.q_proj(x).view(num_rows, num_cols, batch_size, self.num_heads, self.head_dim)
+            k = self.k_proj(x).view(num_rows, num_cols, batch_size, self.num_heads, self.head_dim)
+            v = self.v_proj(x).view(num_rows, num_cols, batch_size, self.num_heads, self.head_dim)
+            q *= self.scaling
+
+            attn_weights = torch.einsum("icnhd,jcnhd->hcnij", q, k)
+
+            if self_attn_mask is not None:
+                raise NotImplementedError
+            if self_attn_padding_mask is not None:
+                attn_weights = attn_weights.masked_fill(
+                    self_attn_padding_mask.permute(2, 0, 1).unsqueeze(0).unsqueeze(3),
+                    -10000,
+                )
+
+            attn_probs = attn_weights.softmax(-1)
+            attn_probs = self.dropout_module(attn_probs)
+            context = torch.einsum("hcnij,jcnhd->icnhd", attn_probs, v)
+            context = context.contiguous().view(num_rows, num_cols, batch_size, embed_dim)
+            output = self.out_proj(context)
+        return output, attn_probs
+
+    def forward(
+        self,
+        x,
+        self_attn_mask=None,
+        self_attn_padding_mask=None,
+    ):
+        num_rows, num_cols, batch_size, embed_dim = x.size()
+        # if False and num_rows * num_cols > 2 ** 14 and not torch.is_grad_enabled():
+        if (num_rows * num_cols) > self.max_tokens_per_msa and not torch.is_grad_enabled():
+            return self._batched_forward(
+                x,
+                self_attn_mask,
+                self_attn_padding_mask,
+            )
+        else:
+            return self.compute_attention_update(x, self_attn_mask, self_attn_padding_mask)

esm/esm/constants.py

@@ -0,0 +1,10 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+# fmt: off
+proteinseq_toks = {
+    'toks': ['L', 'A', 'G', 'V', 'S', 'E', 'R', 'T', 'I', 'D', 'P', 'K', 'Q', 'N', 'F', 'Y', 'M', 'H', 'W', 'C', 'X', 'B', 'U', 'Z', 'O', '.', '-']
+}
+# fmt: on

esm/esm/data.py

@@ -0,0 +1,493 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import itertools
+import os
+from typing import Sequence, Tuple, List, Union
+import pickle
+import re
+import shutil
+import torch
+from pathlib import Path
+from esm.constants import proteinseq_toks
+
+RawMSA = Sequence[Tuple[str, str]]
+
+
+class FastaBatchedDataset(object):
+    def __init__(self, sequence_labels, sequence_strs):
+        self.sequence_labels = list(sequence_labels)
+        self.sequence_strs = list(sequence_strs)
+
+    @classmethod
+    def from_file(cls, fasta_file):
+        sequence_labels, sequence_strs = [], []
+        cur_seq_label = None
+        buf = []
+
+        def _flush_current_seq():
+            nonlocal cur_seq_label, buf
+            if cur_seq_label is None:
+                return
+            sequence_labels.append(cur_seq_label)
+            sequence_strs.append("".join(buf))
+            cur_seq_label = None
+            buf = []
+
+        with open(fasta_file, "r") as infile:
+            for line_idx, line in enumerate(infile):
+                if line.startswith(">"):  # label line
+                    _flush_current_seq()
+                    line = line[1:].strip()
+                    if len(line) > 0:
+                        cur_seq_label = line
+                    else:
+                        cur_seq_label = f"seqnum{line_idx:09d}"
+                else:  # sequence line
+                    buf.append(line.strip())
+
+        _flush_current_seq()
+
+        assert len(set(sequence_labels)) == len(
+            sequence_labels
+        ), "Found duplicate sequence labels"
+
+        return cls(sequence_labels, sequence_strs)
+
+    def __len__(self):
+        return len(self.sequence_labels)
+
+    def __getitem__(self, idx):
+        return self.sequence_labels[idx], self.sequence_strs[idx]
+
+    def get_batch_indices(self, toks_per_batch, extra_toks_per_seq=0):
+        sizes = [(len(s), i) for i, s in enumerate(self.sequence_strs)]
+        sizes.sort()
+        batches = []
+        buf = []
+        max_len = 0
+
+        def _flush_current_buf():
+            nonlocal max_len, buf
+            if len(buf) == 0:
+                return
+            batches.append(buf)
+            buf = []
+            max_len = 0
+
+        for sz, i in sizes:
+            sz += extra_toks_per_seq
+            if max(sz, max_len) * (len(buf) + 1) > toks_per_batch:
+                _flush_current_buf()
+            max_len = max(max_len, sz)
+            buf.append(i)
+
+        _flush_current_buf()
+        return batches
+
+
+class Alphabet(object):
+    def __init__(
+        self,
+        standard_toks: Sequence[str],
+        prepend_toks: Sequence[str] = ("<null_0>", "<pad>", "<eos>", "<unk>"),
+        append_toks: Sequence[str] = ("<cls>", "<mask>", "<sep>"),
+        prepend_bos: bool = True,
+        append_eos: bool = False,
+        use_msa: bool = False,
+    ):
+        self.standard_toks = list(standard_toks)
+        self.prepend_toks = list(prepend_toks)
+        self.append_toks = list(append_toks)
+        self.prepend_bos = prepend_bos
+        self.append_eos = append_eos
+        self.use_msa = use_msa
+
+        self.all_toks = list(self.prepend_toks)
+        self.all_toks.extend(self.standard_toks)
+        for i in range((8 - (len(self.all_toks) % 8)) % 8):
+            self.all_toks.append(f"<null_{i  + 1}>")
+        self.all_toks.extend(self.append_toks)
+
+        self.tok_to_idx = {tok: i for i, tok in enumerate(self.all_toks)}
+
+        self.unk_idx = self.tok_to_idx["<unk>"]
+        self.padding_idx = self.get_idx("<pad>")
+        self.cls_idx = self.get_idx("<cls>")
+        self.mask_idx = self.get_idx("<mask>")
+        self.eos_idx = self.get_idx("<eos>")
+        self.all_special_tokens = ['<eos>', '<unk>', '<pad>', '<cls>', '<mask>']
+        self.unique_no_split_tokens = self.all_toks
+
+    def __len__(self):
+        return len(self.all_toks)
+
+    def get_idx(self, tok):
+        return self.tok_to_idx.get(tok, self.unk_idx)
+
+    def get_tok(self, ind):
+        return self.all_toks[ind]
+
+    def to_dict(self):
+        return self.tok_to_idx.copy()
+
+    def get_batch_converter(self, truncation_seq_length: int = None):
+        if self.use_msa:
+            return MSABatchConverter(self, truncation_seq_length)
+        else:
+            return BatchConverter(self, truncation_seq_length)
+
+    @classmethod
+    def from_architecture(cls, name: str) -> "Alphabet":
+        if name in ("ESM-1", "protein_bert_base"):
+            standard_toks = proteinseq_toks["toks"]
+            prepend_toks: Tuple[str, ...] = ("<null_0>", "<pad>", "<eos>", "<unk>")
+            append_toks: Tuple[str, ...] = ("<cls>", "<mask>", "<sep>")
+            prepend_bos = True
+            append_eos = False
+            use_msa = False
+        elif name in ("ESM-1b", "roberta_large"):
+            standard_toks = proteinseq_toks["toks"]
+            prepend_toks = ("<cls>", "<pad>", "<eos>", "<unk>")
+            append_toks = ("<mask>",)
+            prepend_bos = True
+            append_eos = True
+            use_msa = False
+        elif name in ("MSA Transformer", "msa_transformer"):
+            standard_toks = proteinseq_toks["toks"]
+            prepend_toks = ("<cls>", "<pad>", "<eos>", "<unk>")
+            append_toks = ("<mask>",)
+            prepend_bos = True
+            append_eos = False
+            use_msa = True
+        elif "invariant_gvp" in name.lower():
+            standard_toks = proteinseq_toks["toks"]
+            prepend_toks = ("<null_0>", "<pad>", "<eos>", "<unk>")
+            append_toks = ("<mask>", "<cath>", "<af2>")
+            prepend_bos = True
+            append_eos = False
+            use_msa = False
+        else:
+            raise ValueError("Unknown architecture selected")
+        return cls(standard_toks, prepend_toks, append_toks, prepend_bos, append_eos, use_msa)
+
+    def _tokenize(self, text) -> str:
+        return text.split()
+
+    def tokenize(self, text, **kwargs) -> List[str]:
+        """
+        Inspired by https://github.com/huggingface/transformers/blob/master/src/transformers/tokenization_utils.py
+        Converts a string in a sequence of tokens, using the tokenizer.
+
+        Args:
+            text (:obj:`str`):
+                The sequence to be encoded.
+
+        Returns:
+            :obj:`List[str]`: The list of tokens.
+        """
+
+        def split_on_token(tok, text):
+            result = []
+            split_text = text.split(tok)
+            for i, sub_text in enumerate(split_text):
+                # AddedToken can control whitespace stripping around them.
+                # We use them for GPT2 and Roberta to have different behavior depending on the special token
+                # Cf. https://github.com/huggingface/transformers/pull/2778
+                # and https://github.com/huggingface/transformers/issues/3788
+                # We strip left and right by default
+                if i < len(split_text) - 1:
+                    sub_text = sub_text.rstrip()
+                if i > 0:
+                    sub_text = sub_text.lstrip()
+
+                if i == 0 and not sub_text:
+                    result.append(tok)
+                elif i == len(split_text) - 1:
+                    if sub_text:
+                        result.append(sub_text)
+                    else:
+                        pass
+                else:
+                    if sub_text:
+                        result.append(sub_text)
+                    result.append(tok)
+            return result
+
+        def split_on_tokens(tok_list, text):
+            if not text.strip():
+                return []
+
+            tokenized_text = []
+            text_list = [text]
+            for tok in tok_list:
+                tokenized_text = []
+                for sub_text in text_list:
+                    if sub_text not in self.unique_no_split_tokens:
+                        tokenized_text.extend(split_on_token(tok, sub_text))
+                    else:
+                        tokenized_text.append(sub_text)
+                text_list = tokenized_text
+
+            return list(
+                itertools.chain.from_iterable(
+                    (
+                        self._tokenize(token)
+                        if token not in self.unique_no_split_tokens
+                        else [token]
+                        for token in tokenized_text
+                    )
+                )
+            )
+
+        no_split_token = self.unique_no_split_tokens
+        tokenized_text = split_on_tokens(no_split_token, text)
+        return tokenized_text
+
+    def encode(self, text):
+        return [self.tok_to_idx[tok] for tok in self.tokenize(text)]
+
+
+class BatchConverter(object):
+    """Callable to convert an unprocessed (labels + strings) batch to a
+    processed (labels + tensor) batch.
+    """
+
+    def __init__(self, alphabet, truncation_seq_length: int = None):
+        self.alphabet = alphabet
+        self.truncation_seq_length = truncation_seq_length
+
+    def __call__(self, raw_batch: Sequence[Tuple[str, str]]):
+        # RoBERTa uses an eos token, while ESM-1 does not.
+        batch_size = len(raw_batch)
+        batch_labels, seq_str_list = zip(*raw_batch)
+        seq_encoded_list = [self.alphabet.encode(seq_str) for seq_str in seq_str_list]
+        if self.truncation_seq_length:
+            seq_encoded_list = [seq_str[:self.truncation_seq_length] for seq_str in seq_encoded_list]
+        max_len = max(len(seq_encoded) for seq_encoded in seq_encoded_list)
+        tokens = torch.empty(
+            (
+                batch_size,
+                max_len + int(self.alphabet.prepend_bos) + int(self.alphabet.append_eos),
+            ),
+            dtype=torch.int64,
+        )
+        tokens.fill_(self.alphabet.padding_idx)
+        labels = []
+        strs = []
+
+        for i, (label, seq_str, seq_encoded) in enumerate(
+            zip(batch_labels, seq_str_list, seq_encoded_list)
+        ):
+            labels.append(label)
+            strs.append(seq_str)
+            if self.alphabet.prepend_bos:
+                tokens[i, 0] = self.alphabet.cls_idx
+            seq = torch.tensor(seq_encoded, dtype=torch.int64)
+            tokens[
+                i,
+                int(self.alphabet.prepend_bos) : len(seq_encoded)
+                + int(self.alphabet.prepend_bos),
+            ] = seq
+            if self.alphabet.append_eos:
+                tokens[i, len(seq_encoded) + int(self.alphabet.prepend_bos)] = self.alphabet.eos_idx
+
+        return labels, strs, tokens
+
+
+class MSABatchConverter(BatchConverter):
+    def __call__(self, inputs: Union[Sequence[RawMSA], RawMSA]):
+        if isinstance(inputs[0][0], str):
+            # Input is a single MSA
+            raw_batch: Sequence[RawMSA] = [inputs]  # type: ignore
+        else:
+            raw_batch = inputs  # type: ignore
+
+        batch_size = len(raw_batch)
+        max_alignments = max(len(msa) for msa in raw_batch)
+        max_seqlen = max(len(msa[0][1]) for msa in raw_batch)
+
+        tokens = torch.empty(
+            (
+                batch_size,
+                max_alignments,
+                max_seqlen + int(self.alphabet.prepend_bos) + int(self.alphabet.append_eos),
+            ),
+            dtype=torch.int64,
+        )
+        tokens.fill_(self.alphabet.padding_idx)
+        labels = []
+        strs = []
+
+        for i, msa in enumerate(raw_batch):
+            msa_seqlens = set(len(seq) for _, seq in msa)
+            if not len(msa_seqlens) == 1:
+                raise RuntimeError(
+                    "Received unaligned sequences for input to MSA, all sequence "
+                    "lengths must be equal."
+                )
+            msa_labels, msa_strs, msa_tokens = super().__call__(msa)
+            labels.append(msa_labels)
+            strs.append(msa_strs)
+            tokens[i, : msa_tokens.size(0), : msa_tokens.size(1)] = msa_tokens
+
+        return labels, strs, tokens
+
+
+def read_fasta(
+    path,
+    keep_gaps=True,
+    keep_insertions=True,
+    to_upper=False,
+):
+    with open(path, "r") as f:
+        for result in read_alignment_lines(
+            f, keep_gaps=keep_gaps, keep_insertions=keep_insertions, to_upper=to_upper
+        ):
+            yield result
+
+
+def read_alignment_lines(
+    lines,
+    keep_gaps=True,
+    keep_insertions=True,
+    to_upper=False,
+):
+    seq = desc = None
+
+    def parse(s):
+        if not keep_gaps:
+            s = re.sub("-", "", s)
+        if not keep_insertions:
+            s = re.sub("[a-z]", "", s)
+        return s.upper() if to_upper else s
+
+    for line in lines:
+        # Line may be empty if seq % file_line_width == 0
+        if len(line) > 0 and line[0] == ">":
+            if seq is not None:
+                yield desc, parse(seq)
+            desc = line.strip()
+            seq = ""
+        else:
+            assert isinstance(seq, str)
+            seq += line.strip()
+    assert isinstance(seq, str) and isinstance(desc, str)
+    yield desc, parse(seq)
+
+
+class ESMStructuralSplitDataset(torch.utils.data.Dataset):
+    """
+    Structural Split Dataset as described in section A.10 of the supplement of our paper.
+    https://doi.org/10.1101/622803
+
+    We use the full version of SCOPe 2.07, clustered at 90% sequence identity,
+    generated on January 23, 2020.
+
+    For each SCOPe domain:
+        - We extract the sequence from the corresponding PDB file
+        - We extract the 3D coordinates of the Carbon beta atoms, aligning them
+          to the sequence. We put NaN where Cb atoms are missing.
+        - From the 3D coordinates, we calculate a pairwise distance map, based
+          on L2 distance
+        - We use DSSP to generate secondary structure labels for the corresponding
+          PDB file. This is also aligned to the sequence. We put - where SSP
+          labels are missing.
+
+    For each SCOPe classification level of family/superfamily/fold (in order of difficulty),
+    we have split the data into 5 partitions for cross validation. These are provided
+    in a downloaded splits folder, in the format:
+            splits/{split_level}/{cv_partition}/{train|valid}.txt
+    where train is the partition and valid is the concatentation of the remaining 4.
+
+    For each SCOPe domain, we provide a pkl dump that contains:
+        - seq    : The domain sequence, stored as an L-length string
+        - ssp    : The secondary structure labels, stored as an L-length string
+        - dist   : The distance map, stored as an LxL numpy array
+        - coords : The 3D coordinates, stored as an Lx3 numpy array
+
+    """
+
+    base_folder = "structural-data"
+    file_list = [
+        #  url  tar filename   filename      MD5 Hash
+        (
+            "https://dl.fbaipublicfiles.com/fair-esm/structural-data/splits.tar.gz",
+            "splits.tar.gz",
+            "splits",
+            "456fe1c7f22c9d3d8dfe9735da52411d",
+        ),
+        (
+            "https://dl.fbaipublicfiles.com/fair-esm/structural-data/pkl.tar.gz",
+            "pkl.tar.gz",
+            "pkl",
+            "644ea91e56066c750cd50101d390f5db",
+        ),
+    ]
+
+    def __init__(
+        self,
+        split_level,
+        cv_partition,
+        split,
+        root_path=os.path.expanduser("~/.cache/torch/data/esm"),
+        download=False,
+    ):
+        super().__init__()
+        assert split in [
+            "train",
+            "valid",
+        ], "train_valid must be 'train' or 'valid'"
+        self.root_path = root_path
+        self.base_path = os.path.join(self.root_path, self.base_folder)
+
+        # check if root path has what you need or else download it
+        if download:
+            self.download()
+
+        self.split_file = os.path.join(
+            self.base_path, "splits", split_level, cv_partition, f"{split}.txt"
+        )
+        self.pkl_dir = os.path.join(self.base_path, "pkl")
+        self.names = []
+        with open(self.split_file) as f:
+            self.names = f.read().splitlines()
+
+    def __len__(self):
+        return len(self.names)
+
+    def _check_exists(self) -> bool:
+        for (_, _, filename, _) in self.file_list:
+            fpath = os.path.join(self.base_path, filename)
+            if not os.path.exists(fpath) or not os.path.isdir(fpath):
+                return False
+        return True
+
+    def download(self):
+
+        if self._check_exists():
+            print("Files already downloaded and verified")
+            return
+
+        from torchvision.datasets.utils import download_url
+
+        for url, tar_filename, filename, md5_hash in self.file_list:
+            download_path = os.path.join(self.base_path, tar_filename)
+            download_url(url=url, root=self.base_path, filename=tar_filename, md5=md5_hash)
+            shutil.unpack_archive(download_path, self.base_path)
+
+    def __getitem__(self, idx):
+        """
+        Returns a dict with the following entires
+         - seq : Str (domain sequence)
+         - ssp : Str (SSP labels)
+         - dist : np.array (distance map)
+         - coords : np.array (3D coordinates)
+        """
+        name = self.names[idx]
+        pkl_fname = os.path.join(self.pkl_dir, name[1:3], f"{name}.pkl")
+        with open(pkl_fname, "rb") as f:
+            obj = pickle.load(f)
+        return obj

esm/esm/inverse_folding/__init__.py

@@ -0,0 +1,8 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from . import gvp_transformer
+from . import util
+from . import multichain_util

esm/esm/inverse_folding/features.py

@@ -0,0 +1,352 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+# 
+# Portions of this file were adapted from the open source code for the following
+# two papers:
+#
+#   Ingraham, J., Garg, V., Barzilay, R., & Jaakkola, T. (2019). Generative
+#   models for graph-based protein design. Advances in Neural Information
+#   Processing Systems, 32.
+#
+#   Jing, B., Eismann, S., Suriana, P., Townshend, R. J. L., & Dror, R. (2020).
+#   Learning from Protein Structure with Geometric Vector Perceptrons. In
+#   International Conference on Learning Representations.
+#
+# MIT License
+# 
+# Copyright (c) 2020 Bowen Jing, Stephan Eismann, Patricia Suriana, Raphael Townshend, Ron Dror
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+# 
+# ================================================================
+# The below license applies to the portions of the code (parts of 
+# src/datasets.py and src/models.py) adapted from Ingraham, et al.
+# ================================================================
+# 
+# MIT License
+# 
+# Copyright (c) 2019 John Ingraham, Vikas Garg, Regina Barzilay, Tommi Jaakkola
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import math
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from .gvp_utils import flatten_graph
+from .gvp_modules import GVP, LayerNorm
+from .util import normalize, norm, nan_to_num, rbf
+
+
+class GVPInputFeaturizer(nn.Module):
+
+    @staticmethod
+    def get_node_features(coords, coord_mask, with_coord_mask=True):
+        # scalar features
+        node_scalar_features = GVPInputFeaturizer._dihedrals(coords)
+        if with_coord_mask:
+            node_scalar_features = torch.cat([
+                node_scalar_features,
+                coord_mask.float().unsqueeze(-1)
+            ], dim=-1) 
+        # vector features
+        X_ca = coords[:, :, 1]
+        orientations = GVPInputFeaturizer._orientations(X_ca)
+        sidechains = GVPInputFeaturizer._sidechains(coords)
+        node_vector_features = torch.cat([orientations, sidechains.unsqueeze(-2)], dim=-2)
+        return node_scalar_features, node_vector_features
+
+    @staticmethod
+    def _orientations(X):
+        forward = normalize(X[:, 1:] - X[:, :-1])
+        backward = normalize(X[:, :-1] - X[:, 1:])
+        forward = F.pad(forward, [0, 0, 0, 1])
+        backward = F.pad(backward, [0, 0, 1, 0])
+        return torch.cat([forward.unsqueeze(-2), backward.unsqueeze(-2)], -2)
+    
+    @staticmethod
+    def _sidechains(X):
+        n, origin, c = X[:, :, 0], X[:, :, 1], X[:, :, 2]
+        c, n = normalize(c - origin), normalize(n - origin)
+        bisector = normalize(c + n)
+        perp = normalize(torch.cross(c, n, dim=-1))
+        vec = -bisector * math.sqrt(1 / 3) - perp * math.sqrt(2 / 3)
+        return vec 
+
+    @staticmethod
+    def _dihedrals(X, eps=1e-7):
+        X = torch.flatten(X[:, :, :3], 1, 2)
+        bsz = X.shape[0]
+        dX = X[:, 1:] - X[:, :-1]
+        U = normalize(dX, dim=-1)
+        u_2 = U[:, :-2]
+        u_1 = U[:, 1:-1]
+        u_0 = U[:, 2:]
+    
+        # Backbone normals
+        n_2 = normalize(torch.cross(u_2, u_1, dim=-1), dim=-1)
+        n_1 = normalize(torch.cross(u_1, u_0, dim=-1), dim=-1)
+    
+        # Angle between normals
+        cosD = torch.sum(n_2 * n_1, -1)
+        cosD = torch.clamp(cosD, -1 + eps, 1 - eps)
+        D = torch.sign(torch.sum(u_2 * n_1, -1)) * torch.acos(cosD)
+    
+        # This scheme will remove phi[0], psi[-1], omega[-1]
+        D = F.pad(D, [1, 2]) 
+        D = torch.reshape(D, [bsz, -1, 3])
+        # Lift angle representations to the circle
+        D_features = torch.cat([torch.cos(D), torch.sin(D)], -1)
+        return D_features
+
+    @staticmethod
+    def _positional_embeddings(edge_index, 
+                               num_embeddings=None,
+                               num_positional_embeddings=16,
+                               period_range=[2, 1000]):
+        # From https://github.com/jingraham/neurips19-graph-protein-design
+        num_embeddings = num_embeddings or num_positional_embeddings
+        d = edge_index[0] - edge_index[1]
+     
+        frequency = torch.exp(
+            torch.arange(0, num_embeddings, 2, dtype=torch.float32,
+                device=edge_index.device)
+            * -(np.log(10000.0) / num_embeddings)
+        )
+        angles = d.unsqueeze(-1) * frequency
+        E = torch.cat((torch.cos(angles), torch.sin(angles)), -1)
+        return E
+
+    @staticmethod
+    def _dist(X, coord_mask, padding_mask, top_k_neighbors, eps=1e-8):
+        """ Pairwise euclidean distances """
+        bsz, maxlen = X.size(0), X.size(1)
+        coord_mask_2D = torch.unsqueeze(coord_mask,1) * torch.unsqueeze(coord_mask,2)
+        residue_mask = ~padding_mask
+        residue_mask_2D = torch.unsqueeze(residue_mask,1) * torch.unsqueeze(residue_mask,2)
+        dX = torch.unsqueeze(X,1) - torch.unsqueeze(X,2)
+        D = coord_mask_2D * norm(dX, dim=-1)
+    
+        # sorting preference: first those with coords, then among the residues that
+        # exist but are masked use distance in sequence as tie breaker, and then the
+        # residues that came from padding are last
+        seqpos = torch.arange(maxlen, device=X.device)
+        Dseq = torch.abs(seqpos.unsqueeze(1) - seqpos.unsqueeze(0)).repeat(bsz, 1, 1)
+        D_adjust = nan_to_num(D) + (~coord_mask_2D) * (1e8 + Dseq*1e6) + (
+            ~residue_mask_2D) * (1e10)
+    
+        if top_k_neighbors == -1:
+            D_neighbors = D_adjust
+            E_idx = seqpos.repeat(
+                    *D_neighbors.shape[:-1], 1)
+        else:
+            # Identify k nearest neighbors (including self)
+            k = min(top_k_neighbors, X.size(1))
+            D_neighbors, E_idx = torch.topk(D_adjust, k, dim=-1, largest=False)
+    
+        coord_mask_neighbors = (D_neighbors < 5e7)
+        residue_mask_neighbors = (D_neighbors < 5e9)
+        return D_neighbors, E_idx, coord_mask_neighbors, residue_mask_neighbors
+
+
+class Normalize(nn.Module):
+    def __init__(self, features, epsilon=1e-6):
+        super(Normalize, self).__init__()
+        self.gain = nn.Parameter(torch.ones(features))
+        self.bias = nn.Parameter(torch.zeros(features))
+        self.epsilon = epsilon
+
+    def forward(self, x, dim=-1):
+        mu = x.mean(dim, keepdim=True)
+        sigma = torch.sqrt(x.var(dim, keepdim=True) + self.epsilon)
+        gain = self.gain
+        bias = self.bias
+        # Reshape
+        if dim != -1:
+            shape = [1] * len(mu.size())
+            shape[dim] = self.gain.size()[0]
+            gain = gain.view(shape)
+            bias = bias.view(shape)
+        return gain * (x - mu) / (sigma + self.epsilon) + bias
+
+
+class DihedralFeatures(nn.Module):
+    def __init__(self, node_embed_dim):
+        """ Embed dihedral angle features. """
+        super(DihedralFeatures, self).__init__()
+        # 3 dihedral angles; sin and cos of each angle
+        node_in = 6
+        # Normalization and embedding
+        self.node_embedding = nn.Linear(node_in,  node_embed_dim, bias=True)
+        self.norm_nodes = Normalize(node_embed_dim)
+
+    def forward(self, X):
+        """ Featurize coordinates as an attributed graph """
+        V = self._dihedrals(X)
+        V = self.node_embedding(V)
+        V = self.norm_nodes(V)
+        return V
+
+    @staticmethod
+    def _dihedrals(X, eps=1e-7, return_angles=False):
+        # First 3 coordinates are N, CA, C
+        X = X[:,:,:3,:].reshape(X.shape[0], 3*X.shape[1], 3)
+
+        # Shifted slices of unit vectors
+        dX = X[:,1:,:] - X[:,:-1,:]
+        U = F.normalize(dX, dim=-1)
+        u_2 = U[:,:-2,:]
+        u_1 = U[:,1:-1,:]
+        u_0 = U[:,2:,:]
+        # Backbone normals
+        n_2 = F.normalize(torch.cross(u_2, u_1, dim=-1), dim=-1)
+        n_1 = F.normalize(torch.cross(u_1, u_0, dim=-1), dim=-1)
+
+        # Angle between normals
+        cosD = (n_2 * n_1).sum(-1)
+        cosD = torch.clamp(cosD, -1+eps, 1-eps)
+        D = torch.sign((u_2 * n_1).sum(-1)) * torch.acos(cosD)
+
+        # This scheme will remove phi[0], psi[-1], omega[-1]
+        D = F.pad(D, (1,2), 'constant', 0)
+        D = D.view((D.size(0), int(D.size(1)/3), 3))
+        phi, psi, omega = torch.unbind(D,-1)
+
+        if return_angles:
+            return phi, psi, omega
+
+        # Lift angle representations to the circle
+        D_features = torch.cat((torch.cos(D), torch.sin(D)), 2)
+        return D_features
+
+
+class GVPGraphEmbedding(GVPInputFeaturizer):
+
+    def __init__(self, args):
+        super().__init__()
+        self.top_k_neighbors = args.top_k_neighbors
+        self.num_positional_embeddings = 16
+        self.remove_edges_without_coords = True
+        node_input_dim = (7, 3)
+        edge_input_dim = (34, 1)
+        node_hidden_dim = (args.node_hidden_dim_scalar,
+                args.node_hidden_dim_vector)
+        edge_hidden_dim = (args.edge_hidden_dim_scalar,
+                args.edge_hidden_dim_vector)
+        self.embed_node = nn.Sequential(
+            GVP(node_input_dim, node_hidden_dim, activations=(None, None)),
+            LayerNorm(node_hidden_dim, eps=1e-4)
+        )
+        self.embed_edge = nn.Sequential(
+            GVP(edge_input_dim, edge_hidden_dim, activations=(None, None)),
+            LayerNorm(edge_hidden_dim, eps=1e-4)
+        )
+        self.embed_confidence = nn.Linear(16, args.node_hidden_dim_scalar)
+
+    def forward(self, coords, coord_mask, padding_mask, confidence):
+        with torch.no_grad():
+            node_features = self.get_node_features(coords, coord_mask)
+            edge_features, edge_index = self.get_edge_features(
+                coords, coord_mask, padding_mask)
+        node_embeddings_scalar, node_embeddings_vector = self.embed_node(node_features)
+        edge_embeddings = self.embed_edge(edge_features)
+
+        rbf_rep = rbf(confidence, 0., 1.)
+        node_embeddings = (
+            node_embeddings_scalar + self.embed_confidence(rbf_rep),
+            node_embeddings_vector
+        )
+
+        node_embeddings, edge_embeddings, edge_index = flatten_graph(
+            node_embeddings, edge_embeddings, edge_index)
+        return node_embeddings, edge_embeddings, edge_index
+
+    def get_edge_features(self, coords, coord_mask, padding_mask):
+        X_ca = coords[:, :, 1]
+        # Get distances to the top k neighbors
+        E_dist, E_idx, E_coord_mask, E_residue_mask = GVPInputFeaturizer._dist(
+                X_ca, coord_mask, padding_mask, self.top_k_neighbors)
+        # Flatten the graph to be batch size 1 for torch_geometric package 
+        dest = E_idx
+        B, L, k = E_idx.shape[:3]
+        src = torch.arange(L, device=E_idx.device).view([1, L, 1]).expand(B, L, k)
+        # After flattening, [2, B, E]
+        edge_index = torch.stack([src, dest], dim=0).flatten(2, 3)
+        # After flattening, [B, E]
+        E_dist = E_dist.flatten(1, 2)
+        E_coord_mask = E_coord_mask.flatten(1, 2).unsqueeze(-1)
+        E_residue_mask = E_residue_mask.flatten(1, 2)
+        # Calculate relative positional embeddings and distance RBF 
+        pos_embeddings = GVPInputFeaturizer._positional_embeddings(
+            edge_index,
+            num_positional_embeddings=self.num_positional_embeddings,
+        )
+        D_rbf = rbf(E_dist, 0., 20.)
+        # Calculate relative orientation 
+        X_src = X_ca.unsqueeze(2).expand(-1, -1, k, -1).flatten(1, 2)
+        X_dest = torch.gather(
+            X_ca,
+            1,
+            edge_index[1, :, :].unsqueeze(-1).expand([B, L*k, 3])
+        )
+        coord_mask_src = coord_mask.unsqueeze(2).expand(-1, -1, k).flatten(1, 2)
+        coord_mask_dest = torch.gather(
+            coord_mask,
+            1,
+            edge_index[1, :, :].expand([B, L*k])
+        )
+        E_vectors = X_src - X_dest
+        # For the ones without coordinates, substitute in the average vector
+        E_vector_mean = torch.sum(E_vectors * E_coord_mask, dim=1,
+                keepdims=True) / torch.sum(E_coord_mask, dim=1, keepdims=True)
+        E_vectors = E_vectors * E_coord_mask + E_vector_mean * ~(E_coord_mask)
+        # Normalize and remove nans 
+        edge_s = torch.cat([D_rbf, pos_embeddings], dim=-1)
+        edge_v = normalize(E_vectors).unsqueeze(-2)
+        edge_s, edge_v = map(nan_to_num, (edge_s, edge_v))
+        # Also add indications of whether the coordinates are present 
+        edge_s = torch.cat([
+            edge_s,
+            (~coord_mask_src).float().unsqueeze(-1),
+            (~coord_mask_dest).float().unsqueeze(-1),
+        ], dim=-1)
+        edge_index[:, ~E_residue_mask] = -1
+        if self.remove_edges_without_coords:
+            edge_index[:, ~E_coord_mask.squeeze(-1)] = -1
+        return (edge_s, edge_v), edge_index.transpose(0, 1) 

esm/esm/inverse_folding/gvp_encoder.py

@@ -0,0 +1,56 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from argparse import Namespace
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from .features import GVPGraphEmbedding
+from .gvp_modules import GVPConvLayer, LayerNorm 
+from .gvp_utils import unflatten_graph
+
+
+
+class GVPEncoder(nn.Module):
+
+    def __init__(self, args):
+        super().__init__()
+        self.args = args
+        self.embed_graph = GVPGraphEmbedding(args)
+
+        node_hidden_dim = (args.node_hidden_dim_scalar,
+                args.node_hidden_dim_vector)
+        edge_hidden_dim = (args.edge_hidden_dim_scalar,
+                args.edge_hidden_dim_vector)
+        
+        conv_activations = (F.relu, torch.sigmoid)
+        self.encoder_layers = nn.ModuleList(
+                GVPConvLayer(
+                    node_hidden_dim,
+                    edge_hidden_dim,
+                    drop_rate=args.dropout,
+                    vector_gate=True,
+                    attention_heads=0,
+                    n_message=3,
+                    conv_activations=conv_activations,
+                    n_edge_gvps=0,
+                    eps=1e-4,
+                    layernorm=True,
+                ) 
+            for i in range(args.num_encoder_layers)
+        )
+
+    def forward(self, coords, coord_mask, padding_mask, confidence):
+        node_embeddings, edge_embeddings, edge_index = self.embed_graph(
+                coords, coord_mask, padding_mask, confidence)
+        
+        for i, layer in enumerate(self.encoder_layers):
+            node_embeddings, edge_embeddings = layer(node_embeddings,
+                    edge_index, edge_embeddings)
+
+        node_embeddings = unflatten_graph(node_embeddings, coords.shape[0])
+        return node_embeddings

esm/esm/inverse_folding/gvp_modules.py

@@ -0,0 +1,473 @@
+# Contents of this file are from the open source code for
+#
+#   Jing, B., Eismann, S., Suriana, P., Townshend, R. J. L., & Dror, R. (2020).
+#   Learning from Protein Structure with Geometric Vector Perceptrons. In
+#   International Conference on Learning Representations.
+#
+# MIT License
+# 
+# Copyright (c) 2020 Bowen Jing, Stephan Eismann, Patricia Suriana, Raphael Townshend, Ron Dror
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import typing as T
+import torch
+from torch import nn
+import torch.nn.functional as F
+from torch_geometric.nn import MessagePassing
+from torch_scatter import scatter_add, scatter
+
+def tuple_size(tp):
+    return tuple([0 if a is None else a.size() for a in tp])
+
+def tuple_sum(tp1, tp2):
+    s1, v1 = tp1
+    s2, v2 = tp2
+    if v2 is None and v2 is None:
+        return (s1 + s2, None)
+    return (s1 + s2, v1 + v2)
+
+def tuple_cat(*args, dim=-1):
+    '''
+    Concatenates any number of tuples (s, V) elementwise.
+    
+    :param dim: dimension along which to concatenate when viewed
+                as the `dim` index for the scalar-channel tensors.
+                This means that `dim=-1` will be applied as
+                `dim=-2` for the vector-channel tensors.
+    '''
+    dim %= len(args[0][0].shape)
+    s_args, v_args = list(zip(*args))
+    return torch.cat(s_args, dim=dim), torch.cat(v_args, dim=dim)
+
+def tuple_index(x, idx):
+    '''
+    Indexes into a tuple (s, V) along the first dimension.
+    
+    :param idx: any object which can be used to index into a `torch.Tensor`
+    '''
+    return x[0][idx], x[1][idx]
+
+def randn(n, dims, device="cpu"):
+    '''
+    Returns random tuples (s, V) drawn elementwise from a normal distribution.
+    
+    :param n: number of data points
+    :param dims: tuple of dimensions (n_scalar, n_vector)
+    
+    :return: (s, V) with s.shape = (n, n_scalar) and
+             V.shape = (n, n_vector, 3)
+    '''
+    return torch.randn(n, dims[0], device=device), \
+            torch.randn(n, dims[1], 3, device=device)
+
+def _norm_no_nan(x, axis=-1, keepdims=False, eps=1e-8, sqrt=True):
+    '''
+    L2 norm of tensor clamped above a minimum value `eps`.
+    
+    :param sqrt: if `False`, returns the square of the L2 norm
+    '''
+    # clamp is slow
+    # out = torch.clamp(torch.sum(torch.square(x), axis, keepdims), min=eps)
+    out = torch.sum(torch.square(x), axis, keepdims) + eps
+    return torch.sqrt(out) if sqrt else out
+
+def _split(x, nv):
+    '''
+    Splits a merged representation of (s, V) back into a tuple. 
+    Should be used only with `_merge(s, V)` and only if the tuple 
+    representation cannot be used.
+    
+    :param x: the `torch.Tensor` returned from `_merge`
+    :param nv: the number of vector channels in the input to `_merge`
+    '''
+    v = torch.reshape(x[..., -3*nv:], x.shape[:-1] + (nv, 3))
+    s = x[..., :-3*nv]
+    return s, v
+
+def _merge(s, v):
+    '''
+    Merges a tuple (s, V) into a single `torch.Tensor`, where the
+    vector channels are flattened and appended to the scalar channels.
+    Should be used only if the tuple representation cannot be used.
+    Use `_split(x, nv)` to reverse.
+    '''
+    v = torch.reshape(v, v.shape[:-2] + (3*v.shape[-2],))
+    return torch.cat([s, v], -1)
+
+class GVP(nn.Module):
+    '''
+    Geometric Vector Perceptron. See manuscript and README.md
+    for more details.
+    
+    :param in_dims: tuple (n_scalar, n_vector)
+    :param out_dims: tuple (n_scalar, n_vector)
+    :param h_dim: intermediate number of vector channels, optional
+    :param activations: tuple of functions (scalar_act, vector_act)
+    :param tuple_io: whether to keep accepting tuple inputs and outputs when vi
+    or vo = 0
+    '''
+    def __init__(self, in_dims, out_dims, h_dim=None, vector_gate=False,
+                 activations=(F.relu, torch.sigmoid), tuple_io=True,
+                 eps=1e-8):
+        super(GVP, self).__init__()
+        self.si, self.vi = in_dims
+        self.so, self.vo = out_dims
+        self.tuple_io = tuple_io
+        if self.vi: 
+            self.h_dim = h_dim or max(self.vi, self.vo) 
+            self.wh = nn.Linear(self.vi, self.h_dim, bias=False)
+            self.ws = nn.Linear(self.h_dim + self.si, self.so)
+            if self.vo:
+                self.wv = nn.Linear(self.h_dim, self.vo, bias=False)
+                if vector_gate:
+                    self.wg = nn.Linear(self.so, self.vo)
+        else:
+            self.ws = nn.Linear(self.si, self.so)
+        
+        self.vector_gate = vector_gate
+        self.scalar_act, self.vector_act = activations
+        self.eps = eps
+        
+    def forward(self, x):
+        '''
+        :param x: tuple (s, V) of `torch.Tensor`, 
+                  or (if vectors_in is 0), a single `torch.Tensor`
+        :return: tuple (s, V) of `torch.Tensor`,
+                 or (if vectors_out is 0), a single `torch.Tensor`
+        '''
+        if self.vi:
+            s, v = x
+            v = torch.transpose(v, -1, -2)
+            vh = self.wh(v)    
+            vn = _norm_no_nan(vh, axis=-2, eps=self.eps)
+            s = self.ws(torch.cat([s, vn], -1))
+            if self.scalar_act:
+                s = self.scalar_act(s)
+            if self.vo: 
+                v = self.wv(vh) 
+                v = torch.transpose(v, -1, -2)
+                if self.vector_gate:
+                    g = self.wg(s).unsqueeze(-1)
+                else:
+                    g = _norm_no_nan(v, axis=-1, keepdims=True, eps=self.eps)
+                if self.vector_act:
+                    g = self.vector_act(g)
+                    v = v * g
+        else:
+            if self.tuple_io:
+                assert x[1] is None
+                x = x[0]
+            s = self.ws(x)
+            if self.scalar_act:
+                s = self.scalar_act(s)
+            if self.vo:
+                v = torch.zeros(list(s.shape)[:-1] + [self.vo, 3],
+                        device=s.device)
+        
+        if self.vo:
+            return (s, v)
+        elif self.tuple_io:
+            return (s, None)
+        else:
+            return s
+
+
+class _VDropout(nn.Module):
+    '''
+    Vector channel dropout where the elements of each
+    vector channel are dropped together.
+    '''
+    def __init__(self, drop_rate):
+        super(_VDropout, self).__init__()
+        self.drop_rate = drop_rate
+
+    def forward(self, x):
+        '''
+        :param x: `torch.Tensor` corresponding to vector channels
+        '''
+        if x is None:
+            return None
+        device = x.device
+        if not self.training:
+            return x
+        mask = torch.bernoulli(
+            (1 - self.drop_rate) * torch.ones(x.shape[:-1], device=device)
+        ).unsqueeze(-1)
+        x = mask * x / (1 - self.drop_rate)
+        return x
+
+class Dropout(nn.Module):
+    '''
+    Combined dropout for tuples (s, V).
+    Takes tuples (s, V) as input and as output.
+    '''
+    def __init__(self, drop_rate):
+        super(Dropout, self).__init__()
+        self.sdropout = nn.Dropout(drop_rate)
+        self.vdropout = _VDropout(drop_rate)
+
+    def forward(self, x):
+        '''
+        :param x: tuple (s, V) of `torch.Tensor`,
+                  or single `torch.Tensor` 
+                  (will be assumed to be scalar channels)
+        '''
+        if type(x) is torch.Tensor:
+            return self.sdropout(x)
+        s, v = x
+        return self.sdropout(s), self.vdropout(v)
+
+class LayerNorm(nn.Module):
+    '''
+    Combined LayerNorm for tuples (s, V).
+    Takes tuples (s, V) as input and as output.
+    '''
+    def __init__(self, dims, tuple_io=True, eps=1e-8):
+        super(LayerNorm, self).__init__()
+        self.tuple_io = tuple_io
+        self.s, self.v = dims
+        self.scalar_norm = nn.LayerNorm(self.s)
+        self.eps = eps
+        
+    def forward(self, x):
+        '''
+        :param x: tuple (s, V) of `torch.Tensor`,
+                  or single `torch.Tensor` 
+                  (will be assumed to be scalar channels)
+        '''
+        if not self.v:
+            if self.tuple_io:
+                return self.scalar_norm(x[0]), None
+            return self.scalar_norm(x)
+        s, v = x
+        vn = _norm_no_nan(v, axis=-1, keepdims=True, sqrt=False, eps=self.eps)
+        nonzero_mask = (vn > 2 * self.eps)
+        vn = torch.sum(vn * nonzero_mask, dim=-2, keepdim=True
+            ) / (self.eps + torch.sum(nonzero_mask, dim=-2, keepdim=True))
+        vn = torch.sqrt(vn + self.eps)
+        v = nonzero_mask * (v / vn)
+        return self.scalar_norm(s), v
+
+class GVPConv(MessagePassing):
+    '''
+    Graph convolution / message passing with Geometric Vector Perceptrons.
+    Takes in a graph with node and edge embeddings,
+    and returns new node embeddings.
+    
+    This does NOT do residual updates and pointwise feedforward layers
+    ---see `GVPConvLayer`.
+    
+    :param in_dims: input node embedding dimensions (n_scalar, n_vector)
+    :param out_dims: output node embedding dimensions (n_scalar, n_vector)
+    :param edge_dims: input edge embedding dimensions (n_scalar, n_vector)
+    :param n_layers: number of GVPs in the message function
+    :param module_list: preconstructed message function, overrides n_layers
+    :param aggr: should be "add" if some incoming edges are masked, as in
+                 a masked autoregressive decoder architecture
+    '''
+    def __init__(self, in_dims, out_dims, edge_dims, n_layers=3,
+            vector_gate=False, module_list=None, aggr="mean", eps=1e-8,
+            activations=(F.relu, torch.sigmoid)):
+        super(GVPConv, self).__init__(aggr=aggr)
+        self.eps = eps
+        self.si, self.vi = in_dims
+        self.so, self.vo = out_dims
+        self.se, self.ve = edge_dims
+        
+        module_list = module_list or []
+        if not module_list:
+            if n_layers == 1:
+                module_list.append(
+                    GVP((2*self.si + self.se, 2*self.vi + self.ve), 
+                        (self.so, self.vo), activations=(None, None)))
+            else:
+                module_list.append(
+                    GVP((2*self.si + self.se, 2*self.vi + self.ve), out_dims,
+                        vector_gate=vector_gate, activations=activations)
+                )
+                for i in range(n_layers - 2):
+                    module_list.append(GVP(out_dims, out_dims,
+                        vector_gate=vector_gate))
+                module_list.append(GVP(out_dims, out_dims,
+                                       activations=(None, None)))
+        self.message_func = nn.Sequential(*module_list)
+
+    def forward(self, x, edge_index, edge_attr):
+        '''
+        :param x: tuple (s, V) of `torch.Tensor`
+        :param edge_index: array of shape [2, n_edges]
+        :param edge_attr: tuple (s, V) of `torch.Tensor`
+        '''
+        x_s, x_v = x
+        message = self.propagate(edge_index, 
+                    s=x_s, v=x_v.reshape(x_v.shape[0], 3*x_v.shape[1]),
+                    edge_attr=edge_attr)
+        return _split(message, self.vo) 
+
+    def message(self, s_i, v_i, s_j, v_j, edge_attr):
+        v_j = v_j.view(v_j.shape[0], v_j.shape[1]//3, 3)
+        v_i = v_i.view(v_i.shape[0], v_i.shape[1]//3, 3)
+        message = tuple_cat((s_j, v_j), edge_attr, (s_i, v_i))
+        message = self.message_func(message)
+        return _merge(*message)
+
+
+class GVPConvLayer(nn.Module):
+    '''
+    Full graph convolution / message passing layer with 
+    Geometric Vector Perceptrons. Residually updates node embeddings with
+    aggregated incoming messages, applies a pointwise feedforward 
+    network to node embeddings, and returns updated node embeddings.
+    
+    To only compute the aggregated messages, see `GVPConv`.
+    
+    :param node_dims: node embedding dimensions (n_scalar, n_vector)
+    :param edge_dims: input edge embedding dimensions (n_scalar, n_vector)
+    :param n_message: number of GVPs to use in message function
+    :param n_feedforward: number of GVPs to use in feedforward function
+    :param drop_rate: drop probability in all dropout layers
+    :param autoregressive: if `True`, this `GVPConvLayer` will be used
+           with a different set of input node embeddings for messages
+           where src >= dst
+    '''
+    def __init__(self, node_dims, edge_dims, vector_gate=False,
+                 n_message=3, n_feedforward=2, drop_rate=.1,
+                 autoregressive=False, attention_heads=0,
+                 conv_activations=(F.relu, torch.sigmoid),
+                 n_edge_gvps=0, layernorm=True, eps=1e-8):
+        
+        super(GVPConvLayer, self).__init__()
+        if attention_heads == 0:
+            self.conv = GVPConv(
+                    node_dims, node_dims, edge_dims, n_layers=n_message,
+                    vector_gate=vector_gate,
+                    aggr="add" if autoregressive else "mean",
+                    activations=conv_activations, 
+                    eps=eps,
+            )
+        else:
+            raise NotImplementedError
+        if layernorm:
+            self.norm = nn.ModuleList([LayerNorm(node_dims, eps=eps) for _ in range(2)])
+        else:
+            self.norm = nn.ModuleList([nn.Identity() for _ in range(2)])
+        self.dropout = nn.ModuleList([Dropout(drop_rate) for _ in range(2)])
+
+        ff_func = []
+        if n_feedforward == 1:
+            ff_func.append(GVP(node_dims, node_dims, activations=(None, None)))
+        else:
+            hid_dims = 4*node_dims[0], 2*node_dims[1]
+            ff_func.append(GVP(node_dims, hid_dims, vector_gate=vector_gate))
+            for i in range(n_feedforward-2):
+                ff_func.append(GVP(hid_dims, hid_dims, vector_gate=vector_gate))
+            ff_func.append(GVP(hid_dims, node_dims, activations=(None, None)))
+        self.ff_func = nn.Sequential(*ff_func)
+
+        self.edge_message_func = None
+        if n_edge_gvps > 0:
+            si, vi = node_dims
+            se, ve = edge_dims
+            module_list = [
+                GVP((2*si + se, 2*vi + ve), edge_dims, vector_gate=vector_gate)
+            ]
+            for i in range(n_edge_gvps - 2):
+                module_list.append(GVP(edge_dims, edge_dims,
+                    vector_gate=vector_gate))
+            if n_edge_gvps > 1:
+                module_list.append(GVP(edge_dims, edge_dims,
+                    activations=(None, None)))
+            self.edge_message_func = nn.Sequential(*module_list)
+            if layernorm:
+                self.edge_norm = LayerNorm(edge_dims, eps=eps)
+            else:
+                self.edge_norm = nn.Identity()
+            self.edge_dropout = Dropout(drop_rate)
+
+    def forward(self, x, edge_index, edge_attr,
+                autoregressive_x=None, node_mask=None):
+        '''
+        :param x: tuple (s, V) of `torch.Tensor`
+        :param edge_index: array of shape [2, n_edges]
+        :param edge_attr: tuple (s, V) of `torch.Tensor`
+        :param autoregressive_x: tuple (s, V) of `torch.Tensor`. 
+                If not `None`, will be used as srcqq node embeddings
+                for forming messages where src >= dst. The corrent node 
+                embeddings `x` will still be the base of the update and the 
+                pointwise feedforward.
+        :param node_mask: array of type `bool` to index into the first
+                dim of node embeddings (s, V). If not `None`, only
+                these nodes will be updated.
+        '''
+        if self.edge_message_func:
+            src, dst = edge_index
+            if autoregressive_x is None:
+                x_src = x[0][src], x[1][src]
+            else: 
+                mask = (src < dst).unsqueeze(-1)
+                x_src = (
+                    torch.where(mask, x[0][src], autoregressive_x[0][src]),
+                    torch.where(mask.unsqueeze(-1), x[1][src],
+                        autoregressive_x[1][src])
+                )
+            x_dst = x[0][dst], x[1][dst]
+            x_edge = (
+                torch.cat([x_src[0], edge_attr[0], x_dst[0]], dim=-1),
+                torch.cat([x_src[1], edge_attr[1], x_dst[1]], dim=-2)
+            )
+            edge_attr_dh = self.edge_message_func(x_edge)
+            edge_attr = self.edge_norm(tuple_sum(edge_attr,
+                self.edge_dropout(edge_attr_dh)))
+        
+        if autoregressive_x is not None:
+            src, dst = edge_index
+            mask = src < dst
+            edge_index_forward = edge_index[:, mask]
+            edge_index_backward = edge_index[:, ~mask]
+            edge_attr_forward = tuple_index(edge_attr, mask)
+            edge_attr_backward = tuple_index(edge_attr, ~mask)
+            
+            dh = tuple_sum(
+                self.conv(x, edge_index_forward, edge_attr_forward),
+                self.conv(autoregressive_x, edge_index_backward, edge_attr_backward)
+            )
+            
+            count = scatter_add(torch.ones_like(dst), dst,
+                        dim_size=dh[0].size(0)).clamp(min=1).unsqueeze(-1)
+            
+            dh = dh[0] / count, dh[1] / count.unsqueeze(-1)
+
+        else:
+            dh = self.conv(x, edge_index, edge_attr)
+        
+        if node_mask is not None:
+            x_ = x
+            x, dh = tuple_index(x, node_mask), tuple_index(dh, node_mask)
+            
+        x = self.norm[0](tuple_sum(x, self.dropout[0](dh)))
+        
+        dh = self.ff_func(x)
+        x = self.norm[1](tuple_sum(x, self.dropout[1](dh)))
+        
+        if node_mask is not None:
+            x_[0][node_mask], x_[1][node_mask] = x[0], x[1]
+            x = x_
+
+        return x, edge_attr

esm/esm/inverse_folding/gvp_transformer.py

@@ -0,0 +1,137 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+from typing import Any, Dict, List, Optional, Tuple, NamedTuple
+import torch
+from torch import nn
+from torch import Tensor
+import torch.nn.functional as F
+from scipy.spatial import transform
+
+from esm.data import Alphabet
+
+from .features import DihedralFeatures
+from .gvp_encoder import GVPEncoder
+from .gvp_utils import unflatten_graph
+from .gvp_transformer_encoder import GVPTransformerEncoder
+from .transformer_decoder import TransformerDecoder
+from .util import rotate, CoordBatchConverter 
+
+
+class GVPTransformerModel(nn.Module):
+    """
+    GVP-Transformer inverse folding model.
+
+    Architecture: Geometric GVP-GNN as initial layers, followed by
+    sequence-to-sequence Transformer encoder and decoder.
+    """
+
+    def __init__(self, args, alphabet):
+        super().__init__()
+        encoder_embed_tokens = self.build_embedding(
+            args, alphabet, args.encoder_embed_dim,
+        )
+        decoder_embed_tokens = self.build_embedding(
+            args, alphabet, args.decoder_embed_dim, 
+        )
+        encoder = self.build_encoder(args, alphabet, encoder_embed_tokens)
+        decoder = self.build_decoder(args, alphabet, decoder_embed_tokens)
+        self.args = args
+        self.encoder = encoder
+        self.decoder = decoder
+
+    @classmethod
+    def build_encoder(cls, args, src_dict, embed_tokens):
+        encoder = GVPTransformerEncoder(args, src_dict, embed_tokens)
+        return encoder
+
+    @classmethod
+    def build_decoder(cls, args, tgt_dict, embed_tokens):
+        decoder = TransformerDecoder(
+            args,
+            tgt_dict,
+            embed_tokens,
+        )
+        return decoder
+
+    @classmethod
+    def build_embedding(cls, args, dictionary, embed_dim):
+        num_embeddings = len(dictionary)
+        padding_idx = dictionary.padding_idx
+        emb = nn.Embedding(num_embeddings, embed_dim, padding_idx)
+        nn.init.normal_(emb.weight, mean=0, std=embed_dim ** -0.5)
+        nn.init.constant_(emb.weight[padding_idx], 0)
+        return emb
+
+    def forward(
+        self,
+        coords,
+        padding_mask,
+        confidence,
+        prev_output_tokens,
+        return_all_hiddens: bool = False,
+        features_only: bool = False,
+    ):
+        encoder_out = self.encoder(coords, padding_mask, confidence,
+            return_all_hiddens=return_all_hiddens)
+        logits, extra = self.decoder(
+            prev_output_tokens,
+            encoder_out=encoder_out,
+            features_only=features_only,
+            return_all_hiddens=return_all_hiddens,
+        )
+        return logits, extra
+    
+    def sample(self, coords, partial_seq=None, temperature=1.0, confidence=None):
+        """
+        Samples sequences based on multinomial sampling (no beam search).
+
+        Args:
+            coords: L x 3 x 3 list representing one backbone
+            partial_seq: Optional, partial sequence with mask tokens if part of
+                the sequence is known
+            temperature: sampling temperature, use low temperature for higher
+                sequence recovery and high temperature for higher diversity
+            confidence: optional length L list of confidence scores for coordinates
+        """
+        L = len(coords)
+        # Convert to batch format
+        batch_converter = CoordBatchConverter(self.decoder.dictionary)
+        batch_coords, confidence, _, _, padding_mask = (
+            batch_converter([(coords, confidence, None)])
+        )
+        
+        # Start with prepend token
+        mask_idx = self.decoder.dictionary.get_idx('<mask>')
+        sampled_tokens = torch.full((1, 1+L), mask_idx, dtype=int)
+        sampled_tokens[0, 0] = self.decoder.dictionary.get_idx('<cath>')
+        if partial_seq is not None:
+            for i, c in enumerate(partial_seq):
+                sampled_tokens[0, i+1] = self.decoder.dictionary.get_idx(c)
+            
+        # Save incremental states for faster sampling
+        incremental_state = dict()
+        
+        # Run encoder only once
+        encoder_out = self.encoder(batch_coords, padding_mask, confidence)
+        
+        # Decode one token at a time
+        for i in range(1, L+1):
+            if sampled_tokens[0, i] != mask_idx:
+                continue
+            logits, _ = self.decoder(
+                sampled_tokens[:, :i], 
+                encoder_out,
+                incremental_state=incremental_state,
+            )
+            logits = logits[0].transpose(0, 1)
+            logits /= temperature
+            probs = F.softmax(logits, dim=-1)
+            sampled_tokens[:, i] = torch.multinomial(probs, 1).squeeze(-1)
+        sampled_seq = sampled_tokens[0, 1:]
+        
+        # Convert back to string via lookup
+        return ''.join([self.decoder.dictionary.get_tok(a) for a in sampled_seq])

esm/esm/inverse_folding/gvp_transformer_encoder.py

@@ -0,0 +1,184 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Contents of this file were adapted from the open source fairseq repository.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+import math
+from typing import Dict, List, Optional
+
+import torch
+import torch.nn as nn
+from torch import Tensor
+
+from esm.modules import SinusoidalPositionalEmbedding
+from .features import GVPInputFeaturizer, DihedralFeatures
+from .gvp_encoder import GVPEncoder
+from .transformer_layer import TransformerEncoderLayer
+from .util import nan_to_num, get_rotation_frames, rotate, rbf
+
+
+class GVPTransformerEncoder(nn.Module):
+    """
+    Transformer encoder consisting of *args.encoder.layers* layers. Each layer
+    is a :class:`TransformerEncoderLayer`.
+
+    Args:
+        args (argparse.Namespace): parsed command-line arguments
+        dictionary (~fairseq.data.Dictionary): encoding dictionary
+        embed_tokens (torch.nn.Embedding): input embedding
+    """
+
+    def __init__(self, args, dictionary, embed_tokens):
+        super().__init__()
+        self.args = args
+        self.dictionary = dictionary
+
+        self.dropout_module = nn.Dropout(args.dropout)
+
+        embed_dim = embed_tokens.embedding_dim
+        self.padding_idx = embed_tokens.padding_idx
+
+        self.embed_tokens = embed_tokens
+        self.embed_scale = math.sqrt(embed_dim)
+        self.embed_positions = SinusoidalPositionalEmbedding(
+            embed_dim,
+            self.padding_idx,
+        )
+        self.embed_gvp_input_features = nn.Linear(15, embed_dim)
+        self.embed_confidence = nn.Linear(16, embed_dim)
+        self.embed_dihedrals = DihedralFeatures(embed_dim)
+
+        gvp_args = argparse.Namespace()
+        for k, v in vars(args).items():
+            if k.startswith("gvp_"):
+                setattr(gvp_args, k[4:], v)
+        self.gvp_encoder = GVPEncoder(gvp_args)
+        gvp_out_dim = gvp_args.node_hidden_dim_scalar + (3 *
+                gvp_args.node_hidden_dim_vector)
+        self.embed_gvp_output = nn.Linear(gvp_out_dim, embed_dim)
+
+        self.layers = nn.ModuleList([])
+        self.layers.extend(
+            [self.build_encoder_layer(args) for i in range(args.encoder_layers)]
+        )
+        self.num_layers = len(self.layers)
+        self.layer_norm = nn.LayerNorm(embed_dim)
+
+    def build_encoder_layer(self, args):
+        return TransformerEncoderLayer(args)
+
+    def forward_embedding(self, coords, padding_mask, confidence):
+        """
+        Args:
+            coords: N, CA, C backbone coordinates in shape length x 3 (atoms) x 3 
+            padding_mask: boolean Tensor (true for padding) of shape length
+            confidence: confidence scores between 0 and 1 of shape length
+        """
+        components = dict()
+        coord_mask = torch.all(torch.all(torch.isfinite(coords), dim=-1), dim=-1)
+        coords = nan_to_num(coords)
+        mask_tokens = (
+            padding_mask * self.dictionary.padding_idx + 
+            ~padding_mask * self.dictionary.get_idx("<mask>")
+        )
+        components["tokens"] = self.embed_tokens(mask_tokens) * self.embed_scale
+        components["diherals"] = self.embed_dihedrals(coords)
+
+        # GVP encoder
+        gvp_out_scalars, gvp_out_vectors = self.gvp_encoder(coords,
+                coord_mask, padding_mask, confidence)
+        R = get_rotation_frames(coords)
+        # Rotate to local rotation frame for rotation-invariance
+        gvp_out_features = torch.cat([
+            gvp_out_scalars,
+            rotate(gvp_out_vectors, R.transpose(-2, -1)).flatten(-2, -1),
+        ], dim=-1)
+        components["gvp_out"] = self.embed_gvp_output(gvp_out_features)
+
+        components["confidence"] = self.embed_confidence(
+             rbf(confidence, 0., 1.))
+
+        # In addition to GVP encoder outputs, also directly embed GVP input node
+        # features to the Transformer
+        scalar_features, vector_features = GVPInputFeaturizer.get_node_features(
+            coords, coord_mask, with_coord_mask=False)
+        features = torch.cat([
+            scalar_features,
+            rotate(vector_features, R.transpose(-2, -1)).flatten(-2, -1),
+        ], dim=-1)
+        components["gvp_input_features"] = self.embed_gvp_input_features(features)
+
+        embed = sum(components.values())
+        # for k, v in components.items():
+        #     print(k, torch.mean(v, dim=(0,1)), torch.std(v, dim=(0,1)))
+
+        x = embed
+        x = x + self.embed_positions(mask_tokens)
+        x = self.dropout_module(x)
+        return x, components 
+
+    def forward(
+        self,
+        coords,
+        encoder_padding_mask,
+        confidence,
+        return_all_hiddens: bool = False,
+    ):
+        """
+        Args:
+            coords (Tensor): backbone coordinates
+                shape batch_size x num_residues x num_atoms (3 for N, CA, C) x 3
+            encoder_padding_mask (ByteTensor): the positions of
+                  padding elements of shape `(batch_size x num_residues)`
+            confidence (Tensor): the confidence score of shape (batch_size x
+                num_residues). The value is between 0. and 1. for each residue
+                coordinate, or -1. if no coordinate is given
+            return_all_hiddens (bool, optional): also return all of the
+                intermediate hidden states (default: False).
+
+        Returns:
+            dict:
+                - **encoder_out** (Tensor): the last encoder layer's output of
+                  shape `(num_residues, batch_size, embed_dim)`
+                - **encoder_padding_mask** (ByteTensor): the positions of
+                  padding elements of shape `(batch_size, num_residues)`
+                - **encoder_embedding** (Tensor): the (scaled) embedding lookup
+                  of shape `(batch_size, num_residues, embed_dim)`
+                - **encoder_states** (List[Tensor]): all intermediate
+                  hidden states of shape `(num_residues, batch_size, embed_dim)`.
+                  Only populated if *return_all_hiddens* is True.
+        """
+        x, encoder_embedding = self.forward_embedding(coords,
+                encoder_padding_mask, confidence)
+        # account for padding while computing the representation
+        x = x * (1 - encoder_padding_mask.unsqueeze(-1).type_as(x))
+
+        # B x T x C -> T x B x C
+        x = x.transpose(0, 1)
+
+        encoder_states = []
+
+        if return_all_hiddens:
+            encoder_states.append(x)
+
+        # encoder layers
+        for layer in self.layers:
+            x = layer(
+                x, encoder_padding_mask=encoder_padding_mask
+            )
+            if return_all_hiddens:
+                assert encoder_states is not None
+                encoder_states.append(x)
+
+        if self.layer_norm is not None:
+            x = self.layer_norm(x)
+
+        return {
+            "encoder_out": [x],  # T x B x C
+            "encoder_padding_mask": [encoder_padding_mask],  # B x T
+            "encoder_embedding": [encoder_embedding],  # dictionary
+            "encoder_states": encoder_states,  # List[T x B x C]
+        }

esm/esm/inverse_folding/gvp_utils.py

@@ -0,0 +1,68 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import torch
+
+
+def flatten_graph(node_embeddings, edge_embeddings, edge_index):
+    """
+    Flattens the graph into a batch size one (with disconnected subgraphs for
+    each example) to be compatible with pytorch-geometric package.
+    Args:
+        node_embeddings: node embeddings in tuple form (scalar, vector)
+                - scalar: shape batch size x nodes x node_embed_dim
+                - vector: shape batch size x nodes x node_embed_dim x 3
+        edge_embeddings: edge embeddings of in tuple form (scalar, vector)
+                - scalar: shape batch size x edges x edge_embed_dim
+                - vector: shape batch size x edges x edge_embed_dim x 3
+        edge_index: shape batch_size x 2 (source node and target node) x edges
+    Returns:
+        node_embeddings: node embeddings in tuple form (scalar, vector)
+                - scalar: shape batch total_nodes x node_embed_dim
+                - vector: shape batch total_nodes x node_embed_dim x 3
+        edge_embeddings: edge embeddings of in tuple form (scalar, vector)
+                - scalar: shape batch total_edges x edge_embed_dim
+                - vector: shape batch total_edges x edge_embed_dim x 3
+        edge_index: shape 2 x total_edges
+    """
+    x_s, x_v = node_embeddings
+    e_s, e_v = edge_embeddings
+    batch_size, N = x_s.shape[0], x_s.shape[1]
+    node_embeddings = (torch.flatten(x_s, 0, 1), torch.flatten(x_v, 0, 1))
+    edge_embeddings = (torch.flatten(e_s, 0, 1), torch.flatten(e_v, 0, 1))
+
+    edge_mask = torch.any(edge_index != -1, dim=1)
+    # Re-number the nodes by adding batch_idx * N to each batch
+    edge_index = edge_index + (torch.arange(batch_size, device=edge_index.device) *
+            N).unsqueeze(-1).unsqueeze(-1)
+    edge_index = edge_index.permute(1, 0, 2).flatten(1, 2)
+    edge_mask = edge_mask.flatten()
+    edge_index = edge_index[:, edge_mask] 
+    edge_embeddings = (
+        edge_embeddings[0][edge_mask, :],
+        edge_embeddings[1][edge_mask, :]
+    )
+    return node_embeddings, edge_embeddings, edge_index 
+
+
+def unflatten_graph(node_embeddings, batch_size):
+    """
+    Unflattens node embeddings.
+    Args:
+        node_embeddings: node embeddings in tuple form (scalar, vector)
+                - scalar: shape batch total_nodes x node_embed_dim
+                - vector: shape batch total_nodes x node_embed_dim x 3
+        batch_size: int
+    Returns:
+        node_embeddings: node embeddings in tuple form (scalar, vector)
+                - scalar: shape batch size x nodes x node_embed_dim
+                - vector: shape batch size x nodes x node_embed_dim x 3
+    """
+    x_s, x_v = node_embeddings
+    x_s = x_s.reshape(batch_size, -1, x_s.shape[1])
+    x_v = x_v.reshape(batch_size, -1, x_v.shape[1], x_v.shape[2])
+    return (x_s, x_v)
+
+

esm/esm/inverse_folding/multichain_util.py

@@ -0,0 +1,151 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import biotite.structure
+import numpy as np
+import torch
+from typing import Sequence, Tuple, List
+
+from esm.inverse_folding.util import (
+    load_structure,
+    extract_coords_from_structure,
+    load_coords,
+    get_sequence_loss,
+    get_encoder_output,
+)
+
+
+def extract_coords_from_complex(structure: biotite.structure.AtomArray):
+    """
+    Args:
+        structure: biotite AtomArray
+    Returns:
+        Tuple (coords_list, seq_list)
+        - coords: Dictionary mapping chain ids to L x 3 x 3 array for N, CA, C
+          coordinates representing the backbone of each chain
+        - seqs: Dictionary mapping chain ids to native sequences of each chain
+    """
+    coords = {}
+    seqs = {}
+    all_chains = biotite.structure.get_chains(structure)
+    for chain_id in all_chains:
+        chain = structure[structure.chain_id == chain_id]
+        coords[chain_id], seqs[chain_id] = extract_coords_from_structure(chain)
+    return coords, seqs
+
+
+def load_complex_coords(fpath, chains):
+    """
+    Args:
+        fpath: filepath to either pdb or cif file
+        chains: the chain ids (the order matters for autoregressive model)
+    Returns:
+        Tuple (coords_list, seq_list)
+        - coords: Dictionary mapping chain ids to L x 3 x 3 array for N, CA, C
+          coordinates representing the backbone of each chain
+        - seqs: Dictionary mapping chain ids to native sequences of each chain
+    """
+    structure = load_structure(fpath, chains)
+    return extract_coords_from_complex(structure)
+
+
+def _concatenate_coords(coords, target_chain_id, padding_length=10):
+    """
+    Args:
+        coords: Dictionary mapping chain ids to L x 3 x 3 array for N, CA, C
+            coordinates representing the backbone of each chain
+        target_chain_id: The chain id to sample sequences for
+        padding_length: Length of padding between concatenated chains
+    Returns:
+        Tuple (coords, seq)
+            - coords is an L x 3 x 3 array for N, CA, C coordinates, a
+              concatenation of the chains with padding in between
+            - seq is the extracted sequence, with padding tokens inserted
+              between the concatenated chains
+    """
+    pad_coords = np.full((padding_length, 3, 3), np.nan, dtype=np.float32)
+    # For best performance, put the target chain first in concatenation.
+    coords_list = [coords[target_chain_id]]
+    for chain_id in coords:
+        if chain_id == target_chain_id:
+            continue
+        coords_list.append(pad_coords)
+        coords_list.append(coords[chain_id])
+    coords_concatenated = np.concatenate(coords_list, axis=0)
+    return coords_concatenated
+
+
+def sample_sequence_in_complex(model, coords, target_chain_id, temperature=1.,
+        padding_length=10):
+    """
+    Samples sequence for one chain in a complex.
+    Args:
+        model: An instance of the GVPTransformer model
+        coords: Dictionary mapping chain ids to L x 3 x 3 array for N, CA, C
+            coordinates representing the backbone of each chain
+        target_chain_id: The chain id to sample sequences for
+        padding_length: padding length in between chains
+    Returns:
+        Sampled sequence for the target chain
+    """
+    target_chain_len = coords[target_chain_id].shape[0]
+    all_coords = _concatenate_coords(coords, target_chain_id)
+
+    # Supply padding tokens for other chains to avoid unused sampling for speed
+    padding_pattern = ['<pad>'] * all_coords.shape[0]
+    for i in range(target_chain_len):
+        padding_pattern[i] = '<mask>'
+    sampled = model.sample(all_coords, partial_seq=padding_pattern,
+            temperature=temperature)
+    sampled = sampled[:target_chain_len]
+    return sampled
+
+
+def score_sequence_in_complex(model, alphabet, coords, target_chain_id,
+        target_seq, padding_length=10):
+    """
+    Scores sequence for one chain in a complex.
+    Args:
+        model: An instance of the GVPTransformer model
+        alphabet: Alphabet for the model
+        coords: Dictionary mapping chain ids to L x 3 x 3 array for N, CA, C
+            coordinates representing the backbone of each chain
+        target_chain_id: The chain id to sample sequences for
+        target_seq: Target sequence for the target chain for scoring.
+        padding_length: padding length in between chains
+    Returns:
+        Tuple (ll_fullseq, ll_withcoord)
+        - ll_fullseq: Average log-likelihood over the full target chain
+        - ll_withcoord: Average log-likelihood in target chain excluding those
+            residues without coordinates
+    """
+    all_coords = _concatenate_coords(coords, target_chain_id)
+
+    loss, target_padding_mask = get_sequence_loss(model, alphabet, all_coords,
+            target_seq)
+    ll_fullseq = -np.sum(loss * ~target_padding_mask) / np.sum(
+            ~target_padding_mask)
+
+    # Also calculate average when excluding masked portions
+    coord_mask = np.all(np.isfinite(coords[target_chain_id]), axis=(-1, -2))
+    ll_withcoord = -np.sum(loss * coord_mask) / np.sum(coord_mask)
+    return ll_fullseq, ll_withcoord
+
+
+def get_encoder_output_for_complex(model, alphabet, coords, target_chain_id):
+    """
+    Args:
+        model: An instance of the GVPTransformer model
+        alphabet: Alphabet for the model
+        coords: Dictionary mapping chain ids to L x 3 x 3 array for N, CA, C
+            coordinates representing the backbone of each chain
+        target_chain_id: The chain id to sample sequences for
+    Returns:
+        Dictionary mapping chain id to encoder output for each chain
+    """
+    all_coords = _concatenate_coords(coords, target_chain_id)
+    all_rep = get_encoder_output(model, alphabet, all_coords)
+    target_chain_len = coords[target_chain_id].shape[0]
+    return all_rep[:target_chain_len]

esm/esm/inverse_folding/transformer_decoder.py

@@ -0,0 +1,228 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Contents of this file were adapted from the open source fairseq repository.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import math
+from typing import Any, Dict, List, Optional
+
+import torch
+import torch.nn as nn
+from torch import Tensor
+
+from esm.modules import SinusoidalPositionalEmbedding
+from .transformer_layer import TransformerDecoderLayer
+
+
+def fill_with_neg_inf(t):
+    """FP16-compatible function that fills a tensor with -inf."""
+    return t.float().fill_(float("-inf")).type_as(t)
+
+
+class TransformerDecoder(nn.Module):
+    """
+    Transformer decoder consisting of *args.decoder.layers* layers. Each layer
+    is a :class:`TransformerDecoderLayer`.
+
+    Args:
+        args (argparse.Namespace): parsed command-line arguments
+        dictionary (~fairseq.data.Dictionary): decoding dictionary
+        embed_tokens (torch.nn.Embedding): output embedding
+        no_encoder_attn (bool, optional): whether to attend to encoder outputs
+            (default: False).
+    """
+
+    def __init__(
+        self,
+        args,
+        dictionary,
+        embed_tokens,
+    ):
+        super().__init__()
+        self.args = args
+        self.dictionary = dictionary
+        self._future_mask = torch.empty(0)
+
+        self.dropout_module = nn.Dropout(args.dropout)
+
+        input_embed_dim = embed_tokens.embedding_dim
+        embed_dim = args.decoder_embed_dim
+        self.embed_dim = embed_dim
+
+        self.padding_idx = embed_tokens.padding_idx
+
+        self.embed_tokens = embed_tokens
+        self.embed_scale = math.sqrt(embed_dim)
+
+        self.project_in_dim = (
+            nn.Linear(input_embed_dim, embed_dim, bias=False)
+            if embed_dim != input_embed_dim
+            else None
+        )
+        self.embed_positions = SinusoidalPositionalEmbedding(
+            embed_dim,
+            self.padding_idx,
+        )
+
+        self.layers = nn.ModuleList([])
+        self.layers.extend(
+            [
+                self.build_decoder_layer(args)
+                for _ in range(args.decoder_layers)
+            ]
+        )
+        self.num_layers = len(self.layers)
+        self.layer_norm = nn.LayerNorm(embed_dim)
+
+        self.build_output_projection(args, dictionary)
+
+    def build_output_projection(self, args, dictionary):
+        self.output_projection = nn.Linear(
+            args.decoder_embed_dim, len(dictionary), bias=False
+        )
+        nn.init.normal_(
+            self.output_projection.weight, mean=0, std=args.decoder_embed_dim ** -0.5
+        )
+
+    def build_decoder_layer(self, args):
+        return TransformerDecoderLayer(args)
+
+    def forward(
+        self,
+        prev_output_tokens,
+        encoder_out: Optional[Dict[str, List[Tensor]]] = None,
+        incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None,
+        features_only: bool = False,
+        return_all_hiddens: bool = False,
+    ):
+        """
+        Args:
+            prev_output_tokens (LongTensor): previous decoder outputs of shape
+                `(batch, tgt_len)`, for teacher forcing
+            encoder_out (optional): output from the encoder, used for
+                encoder-side attention, should be of size T x B x C
+            incremental_state (dict): dictionary used for storing state during
+                :ref:`Incremental decoding`
+            features_only (bool, optional): only return features without
+                applying output layer (default: False).
+
+        Returns:
+            tuple:
+                - the decoder's output of shape `(batch, tgt_len, vocab)`
+                - a dictionary with any model-specific outputs
+        """
+
+        x, extra = self.extract_features(
+            prev_output_tokens,
+            encoder_out=encoder_out,
+            incremental_state=incremental_state,
+        )
+
+        if not features_only:
+            x = self.output_layer(x)
+        x = x.transpose(1, 2) # B x T x C -> B x C x T
+        return x, extra
+
+    def extract_features(
+        self,
+        prev_output_tokens,
+        encoder_out: Optional[Dict[str, List[Tensor]]],
+        incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None,
+    ):
+        """
+        Similar to *forward* but only return features.
+
+        Includes several features from "Jointly Learning to Align and
+        Translate with Transformer Models" (Garg et al., EMNLP 2019).
+
+        Returns:
+            tuple:
+                - the decoder's features of shape `(batch, tgt_len, embed_dim)`
+                - a dictionary with any model-specific outputs
+        """
+        bs, slen = prev_output_tokens.size()
+
+        enc: Optional[Tensor] = None
+        padding_mask: Optional[Tensor] = None
+        if encoder_out is not None and len(encoder_out["encoder_out"]) > 0:
+            enc = encoder_out["encoder_out"][0]
+            assert (
+                enc.size()[1] == bs
+            ), f"Expected enc.shape == (t, {bs}, c) got {enc.shape}"
+        if encoder_out is not None and len(encoder_out["encoder_padding_mask"]) > 0:
+            padding_mask = encoder_out["encoder_padding_mask"][0]
+
+        # embed positions
+        positions = self.embed_positions(
+            prev_output_tokens
+        )
+
+        if incremental_state is not None:
+            prev_output_tokens = prev_output_tokens[:, -1:]
+            positions = positions[:, -1:]
+
+        # embed tokens and positions
+        x = self.embed_scale * self.embed_tokens(prev_output_tokens)
+
+        if self.project_in_dim is not None:
+            x = self.project_in_dim(x)
+
+        x += positions
+
+        x = self.dropout_module(x)
+
+        # B x T x C -> T x B x C
+        x = x.transpose(0, 1)
+
+        self_attn_padding_mask: Optional[Tensor] = None
+        if prev_output_tokens.eq(self.padding_idx).any():
+            self_attn_padding_mask = prev_output_tokens.eq(self.padding_idx)
+
+        # decoder layers
+        attn: Optional[Tensor] = None
+        inner_states: List[Optional[Tensor]] = [x]
+        for idx, layer in enumerate(self.layers):
+            if incremental_state is None:
+                self_attn_mask = self.buffered_future_mask(x)
+            else:
+                self_attn_mask = None
+
+            x, layer_attn, _ = layer(
+                x,
+                enc,
+                padding_mask,
+                incremental_state,
+                self_attn_mask=self_attn_mask,
+                self_attn_padding_mask=self_attn_padding_mask,
+                need_attn=False,
+                need_head_weights=False,
+            )
+            inner_states.append(x)
+
+        if self.layer_norm is not None:
+            x = self.layer_norm(x)
+
+        # T x B x C -> B x C x T
+        x = x.transpose(0, 1)
+
+        return x, {"inner_states": inner_states}
+
+    def output_layer(self, features):
+        """Project features to the vocabulary size."""
+        return self.output_projection(features)
+
+    def buffered_future_mask(self, tensor):
+        dim = tensor.size(0)
+        # self._future_mask.device != tensor.device is not working in TorchScript. This is a workaround.
+        if (
+            self._future_mask.size(0) == 0
+            or (not self._future_mask.device == tensor.device)
+            or self._future_mask.size(0) < dim
+        ):
+            self._future_mask = torch.triu(
+                fill_with_neg_inf(torch.zeros([dim, dim])), 1
+            )
+        self._future_mask = self._future_mask.to(tensor)
+        return self._future_mask[:dim, :dim]

esm/esm/inverse_folding/transformer_layer.py

@@ -0,0 +1,304 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Contents of this file were adapted from the open source fairseq repository.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from typing import Dict, List, Optional
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from esm.multihead_attention import MultiheadAttention
+from torch import Tensor
+
+
+class TransformerEncoderLayer(nn.Module):
+    """Encoder layer block.
+    `layernorm -> dropout -> add residual`
+
+    Args:
+        args (argparse.Namespace): parsed command-line arguments
+    """
+
+    def __init__(self, args):
+        super().__init__()
+        self.args = args
+        self.embed_dim = args.encoder_embed_dim
+        self.self_attn = self.build_self_attention(self.embed_dim, args)
+        self.self_attn_layer_norm = torch.nn.LayerNorm(self.embed_dim)
+        self.dropout_module = nn.Dropout(args.dropout)
+        self.activation_fn = F.relu
+        self.fc1 = self.build_fc1(
+            self.embed_dim,
+            args.encoder_ffn_embed_dim,
+        )
+        self.fc2 = self.build_fc2(
+            args.encoder_ffn_embed_dim,
+            self.embed_dim,
+        )
+
+        self.final_layer_norm = nn.LayerNorm(self.embed_dim)
+
+    def build_fc1(self, input_dim, output_dim):
+        return nn.Linear(input_dim, output_dim)
+
+    def build_fc2(self, input_dim, output_dim):
+        return nn.Linear(input_dim, output_dim)
+
+    def build_self_attention(self, embed_dim, args):
+        return MultiheadAttention(
+            embed_dim,
+            args.encoder_attention_heads,
+            dropout=args.attention_dropout,
+            self_attention=True,
+        )
+
+    def residual_connection(self, x, residual):
+        return residual + x
+
+    def forward(
+        self,
+        x,
+        encoder_padding_mask: Optional[Tensor],
+        attn_mask: Optional[Tensor] = None,
+    ):
+        """
+        Args:
+            x (Tensor): input to the layer of shape `(seq_len, batch, embed_dim)`
+            encoder_padding_mask (ByteTensor): binary ByteTensor of shape
+                `(batch, seq_len)` where padding elements are indicated by ``1``.
+            attn_mask (ByteTensor): binary tensor of shape `(tgt_len, src_len)`,
+                where `tgt_len` is the length of output and `src_len` is the
+                length of input, though here both are equal to `seq_len`.
+                `attn_mask[tgt_i, src_j] = 1` means that when calculating the
+                embedding for `tgt_i`, we exclude (mask out) `src_j`. This is
+                useful for strided self-attention.
+
+        Returns:
+            encoded output of shape `(seq_len, batch, embed_dim)`
+        """
+        # anything in original attn_mask = 1, becomes -1e8
+        # anything in original attn_mask = 0, becomes 0
+        # Note that we cannot use -inf here, because at some edge cases,
+        # the attention weight (before softmax) for some padded element in query
+        # will become -inf, which results in NaN in model parameters
+        if attn_mask is not None:
+            attn_mask = attn_mask.masked_fill(
+                attn_mask.to(torch.bool), -1e8 if x.dtype == torch.float32 else -1e4
+            )
+
+        residual = x
+        x = self.self_attn_layer_norm(x)
+        x, _ = self.self_attn(
+            query=x,
+            key=x,
+            value=x,
+            key_padding_mask=encoder_padding_mask,
+            need_weights=False,
+            attn_mask=attn_mask,
+        )
+        x = self.dropout_module(x)
+        x = self.residual_connection(x, residual)
+
+        residual = x
+        x = self.final_layer_norm(x)
+        x = self.activation_fn(self.fc1(x))
+        x = self.fc2(x)
+        x = self.dropout_module(x)
+        x = self.residual_connection(x, residual)
+        return x
+
+
+class TransformerDecoderLayer(nn.Module):
+    """Decoder layer block.
+    `layernorm -> dropout -> add residual`
+
+    Args:
+        args (argparse.Namespace): parsed command-line arguments
+        no_encoder_attn (bool, optional): whether to attend to encoder outputs
+            (default: False).
+    """
+
+    def __init__(
+        self, args, no_encoder_attn=False, add_bias_kv=False, add_zero_attn=False
+    ):
+        super().__init__()
+        self.embed_dim = args.decoder_embed_dim
+        self.dropout_module = nn.Dropout(args.dropout)
+
+        self.self_attn = self.build_self_attention(
+            self.embed_dim,
+            args,
+            add_bias_kv=add_bias_kv,
+            add_zero_attn=add_zero_attn,
+        )
+        self.nh = self.self_attn.num_heads
+        self.head_dim = self.self_attn.head_dim
+
+        self.activation_fn = F.relu
+
+        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
+
+        if no_encoder_attn:
+            self.encoder_attn = None
+            self.encoder_attn_layer_norm = None
+        else:
+            self.encoder_attn = self.build_encoder_attention(self.embed_dim, args)
+            self.encoder_attn_layer_norm = nn.LayerNorm(self.embed_dim)
+
+        self.ffn_layernorm = (
+            LayerNorm(args.decoder_ffn_embed_dim)
+            if getattr(args, "scale_fc", False)
+            else None
+        )
+        self.w_resid = (
+            nn.Parameter(
+                torch.ones(
+                    self.embed_dim,
+                ),
+                requires_grad=True,
+            )
+            if getattr(args, "scale_resids", False)
+            else None
+        )
+
+        self.fc1 = self.build_fc1(
+            self.embed_dim,
+            args.decoder_ffn_embed_dim,
+        )
+        self.fc2 = self.build_fc2(
+            args.decoder_ffn_embed_dim,
+            self.embed_dim,
+        )
+
+        self.final_layer_norm = nn.LayerNorm(self.embed_dim)
+        self.need_attn = True
+
+    def build_fc1(self, input_dim, output_dim):
+        return nn.Linear(input_dim, output_dim)
+
+    def build_fc2(self, input_dim, output_dim):
+        return nn.Linear(input_dim, output_dim)
+
+    def build_self_attention(
+        self, embed_dim, args, add_bias_kv=False, add_zero_attn=False
+    ):
+        return MultiheadAttention(
+            embed_dim,
+            args.decoder_attention_heads,
+            dropout=args.attention_dropout,
+            add_bias_kv=add_bias_kv,
+            add_zero_attn=add_zero_attn,
+            self_attention=True,
+        )
+
+    def build_encoder_attention(self, embed_dim, args):
+        return MultiheadAttention(
+            embed_dim,
+            args.decoder_attention_heads,
+            kdim=args.encoder_embed_dim,
+            vdim=args.encoder_embed_dim,
+            dropout=args.attention_dropout,
+            encoder_decoder_attention=True,
+        )
+
+    def residual_connection(self, x, residual):
+        return residual + x
+
+    def forward(
+        self,
+        x,
+        encoder_out: Optional[torch.Tensor] = None,
+        encoder_padding_mask: Optional[torch.Tensor] = None,
+        incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None,
+        prev_self_attn_state: Optional[List[torch.Tensor]] = None,
+        prev_attn_state: Optional[List[torch.Tensor]] = None,
+        self_attn_mask: Optional[torch.Tensor] = None,
+        self_attn_padding_mask: Optional[torch.Tensor] = None,
+        need_attn: bool = False,
+        need_head_weights: bool = False,
+    ):
+        """
+        Args:
+            x (Tensor): input to the layer of shape `(seq_len, batch, embed_dim)`
+            encoder_padding_mask (ByteTensor, optional): binary
+                ByteTensor of shape `(batch, src_len)` where padding
+                elements are indicated by ``1``.
+            need_attn (bool, optional): return attention weights
+            need_head_weights (bool, optional): return attention weights
+                for each head (default: return average over heads).
+
+        Returns:
+            encoded output of shape `(seq_len, batch, embed_dim)`
+        """
+        if need_head_weights:
+            need_attn = True
+
+        residual = x
+        x = self.self_attn_layer_norm(x)
+        if prev_self_attn_state is not None:
+            prev_key, prev_value = prev_self_attn_state[:2]
+            saved_state: Dict[str, Optional[Tensor]] = {
+                "prev_key": prev_key,
+                "prev_value": prev_value,
+            }
+            if len(prev_self_attn_state) >= 3:
+                saved_state["prev_key_padding_mask"] = prev_self_attn_state[2]
+            assert incremental_state is not None
+            self.self_attn._set_input_buffer(incremental_state, saved_state)
+        _self_attn_input_buffer = self.self_attn._get_input_buffer(incremental_state)
+        y = x
+
+        x, attn = self.self_attn(
+            query=x,
+            key=y,
+            value=y,
+            key_padding_mask=self_attn_padding_mask,
+            incremental_state=incremental_state,
+            need_weights=False,
+            attn_mask=self_attn_mask,
+        )
+        x = self.dropout_module(x)
+        x = self.residual_connection(x, residual)
+
+        if self.encoder_attn is not None and encoder_out is not None:
+            residual = x
+            x = self.encoder_attn_layer_norm(x)
+            if prev_attn_state is not None:
+                prev_key, prev_value = prev_attn_state[:2]
+                saved_state: Dict[str, Optional[Tensor]] = {
+                    "prev_key": prev_key,
+                    "prev_value": prev_value,
+                }
+                if len(prev_attn_state) >= 3:
+                    saved_state["prev_key_padding_mask"] = prev_attn_state[2]
+                assert incremental_state is not None
+                self.encoder_attn._set_input_buffer(incremental_state, saved_state)
+
+            x, attn = self.encoder_attn(
+                query=x,
+                key=encoder_out,
+                value=encoder_out,
+                key_padding_mask=encoder_padding_mask,
+                incremental_state=incremental_state,
+                static_kv=True,
+                need_weights=need_attn or (not self.training and self.need_attn),
+                need_head_weights=need_head_weights,
+            )
+            x = self.dropout_module(x)
+            x = self.residual_connection(x, residual)
+
+        residual = x
+        x = self.final_layer_norm(x)
+
+        x = self.activation_fn(self.fc1(x))
+        if self.ffn_layernorm is not None:
+            x = self.ffn_layernorm(x)
+        x = self.fc2(x)
+        x = self.dropout_module(x)
+        if self.w_resid is not None:
+            residual = torch.mul(self.w_resid, residual)
+        x = self.residual_connection(x, residual)
+        return x, attn, None

esm/esm/inverse_folding/util.py

@@ -0,0 +1,320 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import json
+import math
+
+import biotite.structure
+from biotite.structure.io import pdbx, pdb
+from biotite.structure.residues import get_residues
+from biotite.structure import filter_backbone
+from biotite.structure import get_chains
+from biotite.sequence import ProteinSequence
+import numpy as np
+from scipy.spatial import transform
+from scipy.stats import special_ortho_group
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.utils.data as data
+from typing import Sequence, Tuple, List
+
+from esm.data import BatchConverter
+
+
+def load_structure(fpath, chain=None):
+    """
+    Args:
+        fpath: filepath to either pdb or cif file
+        chain: the chain id or list of chain ids to load
+    Returns:
+        biotite.structure.AtomArray
+    """
+    if fpath.endswith('cif'):
+        with open(fpath) as fin:
+            pdbxf = pdbx.PDBxFile.read(fin)
+        structure = pdbx.get_structure(pdbxf, model=1)
+    elif fpath.endswith('pdb'):
+        with open(fpath) as fin:
+            pdbf = pdb.PDBFile.read(fin)
+        structure = pdb.get_structure(pdbf, model=1)
+    bbmask = filter_backbone(structure)
+    structure = structure[bbmask]
+    all_chains = get_chains(structure)
+    if len(all_chains) == 0:
+        raise ValueError('No chains found in the input file.')
+    if chain is None:
+        chain_ids = all_chains
+    elif isinstance(chain, list):
+        chain_ids = chain
+    else:
+        chain_ids = [chain] 
+    for chain in chain_ids:
+        if chain not in all_chains:
+            raise ValueError(f'Chain {chain} not found in input file')
+    chain_filter = [a.chain_id in chain_ids for a in structure]
+    structure = structure[chain_filter]
+    return structure
+
+
+def extract_coords_from_structure(structure: biotite.structure.AtomArray):
+    """
+    Args:
+        structure: An instance of biotite AtomArray
+    Returns:
+        Tuple (coords, seq)
+            - coords is an L x 3 x 3 array for N, CA, C coordinates
+            - seq is the extracted sequence
+    """
+    coords = get_atom_coords_residuewise(["N", "CA", "C"], structure)
+    residue_identities = get_residues(structure)[1]
+    seq = ''.join([ProteinSequence.convert_letter_3to1(r) for r in residue_identities])
+    return coords, seq
+
+
+def load_coords(fpath, chain):
+    """
+    Args:
+        fpath: filepath to either pdb or cif file
+        chain: the chain id
+    Returns:
+        Tuple (coords, seq)
+            - coords is an L x 3 x 3 array for N, CA, C coordinates
+            - seq is the extracted sequence
+    """
+    structure = load_structure(fpath, chain)
+    return extract_coords_from_structure(structure)
+
+
+def get_atom_coords_residuewise(atoms: List[str], struct: biotite.structure.AtomArray):
+    """
+    Example for atoms argument: ["N", "CA", "C"]
+    """
+    def filterfn(s, axis=None):
+        filters = np.stack([s.atom_name == name for name in atoms], axis=1)
+        sum = filters.sum(0)
+        if not np.all(sum <= np.ones(filters.shape[1])):
+            raise RuntimeError("structure has multiple atoms with same name")
+        index = filters.argmax(0)
+        coords = s[index].coord
+        coords[sum == 0] = float("nan")
+        return coords
+
+    return biotite.structure.apply_residue_wise(struct, struct, filterfn)
+
+
+def get_sequence_loss(model, alphabet, coords, seq):
+    batch_converter = CoordBatchConverter(alphabet)
+    batch = [(coords, None, seq)]
+    coords, confidence, strs, tokens, padding_mask = batch_converter(batch)
+    
+    prev_output_tokens = tokens[:, :-1]
+    target = tokens[:, 1:]
+    target_padding_mask = (target == alphabet.padding_idx)
+    logits, _ = model.forward(coords, padding_mask, confidence, prev_output_tokens)
+    loss = F.cross_entropy(logits, target, reduction='none')
+    loss = loss[0].detach().numpy()
+    target_padding_mask = target_padding_mask[0].numpy()
+    return loss, target_padding_mask
+
+
+def score_sequence(model, alphabet, coords, seq):
+    loss, target_padding_mask = get_sequence_loss(model, alphabet, coords, seq)
+    ll_fullseq = -np.sum(loss * ~target_padding_mask) / np.sum(~target_padding_mask)
+    # Also calculate average when excluding masked portions
+    coord_mask = np.all(np.isfinite(coords), axis=(-1, -2))
+    ll_withcoord = -np.sum(loss * coord_mask) / np.sum(coord_mask)
+    return ll_fullseq, ll_withcoord
+
+
+def get_encoder_output(model, alphabet, coords):
+    batch_converter = CoordBatchConverter(alphabet)
+    # the batch_converter is essential for forming the correct input format
+    batch = [(coords, None, None)]
+    coords, confidence, _, _, padding_mask = batch_converter(batch)
+    encoder_out = model.encoder.forward(coords, padding_mask, confidence,
+            return_all_hiddens=False)
+    # remove beginning and end (bos and eos tokens)
+    return encoder_out['encoder_out'][0][1:-1, 0]
+
+
+def rotate(v, R):
+    """
+    Rotates a vector by a rotation matrix.
+    
+    Args:
+        v: 3D vector, tensor of shape (length x batch_size x channels x 3)
+        R: rotation matrix, tensor of shape (length x batch_size x 3 x 3)
+
+    Returns:
+        Rotated version of v by rotation matrix R.
+    """
+    R = R.unsqueeze(-3)
+    v = v.unsqueeze(-1)
+    return torch.sum(v * R, dim=-2)
+
+
+def get_rotation_frames(coords):
+    """
+    Returns a local rotation frame defined by N, CA, C positions.
+
+    Args:
+        coords: coordinates, tensor of shape (batch_size x length x 3 x 3)
+        where the third dimension is in order of N, CA, C
+
+    Returns:
+        Local relative rotation frames in shape (batch_size x length x 3 x 3)
+    """
+    v1 = coords[:, :, 2] - coords[:, :, 1]
+    v2 = coords[:, :, 0] - coords[:, :, 1]
+    e1 = normalize(v1, dim=-1)
+    u2 = v2 - e1 * torch.sum(e1 * v2, dim=-1, keepdim=True)
+    e2 = normalize(u2, dim=-1)
+    e3 = torch.cross(e1, e2, dim=-1)
+    R = torch.stack([e1, e2, e3], dim=-2)
+    return R
+
+
+def nan_to_num(ts, val=0.0):
+    """
+    Replaces nans in tensor with a fixed value.    
+    """
+    val = torch.tensor(val, dtype=ts.dtype, device=ts.device)
+    return torch.where(~torch.isfinite(ts), val, ts)
+
+
+def rbf(values, v_min, v_max, n_bins=16):
+    """
+    Returns RBF encodings in a new dimension at the end.
+    """
+    rbf_centers = torch.linspace(v_min, v_max, n_bins, device=values.device)
+    rbf_centers = rbf_centers.view([1] * len(values.shape) + [-1])
+    rbf_std = (v_max - v_min) / n_bins
+    v_expand = torch.unsqueeze(values, -1)
+    z = (values.unsqueeze(-1) - rbf_centers) / rbf_std
+    return torch.exp(-z ** 2)
+
+
+def norm(tensor, dim, eps=1e-8, keepdim=False):
+    """
+    Returns L2 norm along a dimension.
+    """
+    return torch.sqrt(
+            torch.sum(torch.square(tensor), dim=dim, keepdim=keepdim) + eps)
+
+
+def normalize(tensor, dim=-1):
+    """
+    Normalizes a tensor along a dimension after removing nans.
+    """
+    return nan_to_num(
+        torch.div(tensor, norm(tensor, dim=dim, keepdim=True))
+    )
+
+
+class CoordBatchConverter(BatchConverter):
+    def __call__(self, raw_batch: Sequence[Tuple[Sequence, str]], device=None):
+        """
+        Args:
+            raw_batch: List of tuples (coords, confidence, seq)
+            In each tuple,
+                coords: list of floats, shape L x 3 x 3
+                confidence: list of floats, shape L; or scalar float; or None
+                seq: string of length L
+        Returns:
+            coords: Tensor of shape batch_size x L x 3 x 3
+            confidence: Tensor of shape batch_size x L
+            strs: list of strings
+            tokens: LongTensor of shape batch_size x L
+            padding_mask: ByteTensor of shape batch_size x L
+        """
+        self.alphabet.cls_idx = self.alphabet.get_idx("<cath>") 
+        batch = []
+        for coords, confidence, seq in raw_batch:
+            if confidence is None:
+                confidence = 1.
+            if isinstance(confidence, float) or isinstance(confidence, int):
+                confidence = [float(confidence)] * len(coords)
+            if seq is None:
+                seq = 'X' * len(coords)
+            batch.append(((coords, confidence), seq))
+
+        coords_and_confidence, strs, tokens = super().__call__(batch)
+
+        # pad beginning and end of each protein due to legacy reasons
+        coords = [
+            F.pad(torch.tensor(cd), (0, 0, 0, 0, 1, 1), value=np.inf)
+            for cd, _ in coords_and_confidence
+        ]
+        confidence = [
+            F.pad(torch.tensor(cf), (1, 1), value=-1.)
+            for _, cf in coords_and_confidence
+        ]
+        coords = self.collate_dense_tensors(coords, pad_v=np.nan)
+        confidence = self.collate_dense_tensors(confidence, pad_v=-1.)
+        if device is not None:
+            coords = coords.to(device)
+            confidence = confidence.to(device)
+            tokens = tokens.to(device)
+        padding_mask = torch.isnan(coords[:,:,0,0])
+        coord_mask = torch.isfinite(coords.sum(-2).sum(-1))
+        confidence = confidence * coord_mask + (-1.) * padding_mask
+        return coords, confidence, strs, tokens, padding_mask
+
+    def from_lists(self, coords_list, confidence_list=None, seq_list=None, device=None):
+        """
+        Args:
+            coords_list: list of length batch_size, each item is a list of
+            floats in shape L x 3 x 3 to describe a backbone
+            confidence_list: one of
+                - None, default to highest confidence
+                - list of length batch_size, each item is a scalar
+                - list of length batch_size, each item is a list of floats of
+                    length L to describe the confidence scores for the backbone
+                    with values between 0. and 1.
+            seq_list: either None or a list of strings
+        Returns:
+            coords: Tensor of shape batch_size x L x 3 x 3
+            confidence: Tensor of shape batch_size x L
+            strs: list of strings
+            tokens: LongTensor of shape batch_size x L
+            padding_mask: ByteTensor of shape batch_size x L
+        """
+        batch_size = len(coords_list)
+        if confidence_list is None:
+            confidence_list = [None] * batch_size
+        if seq_list is None:
+            seq_list = [None] * batch_size
+        raw_batch = zip(coords_list, confidence_list, seq_list)
+        return self.__call__(raw_batch, device)
+
+    @staticmethod
+    def collate_dense_tensors(samples, pad_v):
+        """
+        Takes a list of tensors with the following dimensions:
+            [(d_11,       ...,           d_1K),
+             (d_21,       ...,           d_2K),
+             ...,
+             (d_N1,       ...,           d_NK)]
+        and stack + pads them into a single tensor of:
+        (N, max_i=1,N { d_i1 }, ..., max_i=1,N {diK})
+        """
+        if len(samples) == 0:
+            return torch.Tensor()
+        if len(set(x.dim() for x in samples)) != 1:
+            raise RuntimeError(
+                f"Samples has varying dimensions: {[x.dim() for x in samples]}"
+            )
+        (device,) = tuple(set(x.device for x in samples))  # assumes all on same device
+        max_shape = [max(lst) for lst in zip(*[x.shape for x in samples])]
+        result = torch.empty(
+            len(samples), *max_shape, dtype=samples[0].dtype, device=device
+        )
+        result.fill_(pad_v)
+        for i in range(len(samples)):
+            result_i = result[i]
+            t = samples[i]
+            result_i[tuple(slice(0, k) for k in t.shape)] = t
+        return result

esm/esm/model/esm1.py

@@ -0,0 +1,200 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import math
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from ..modules import (
+    TransformerLayer,
+    LearnedPositionalEmbedding,
+    SinusoidalPositionalEmbedding,
+    RobertaLMHead,
+    ESM1bLayerNorm,
+    ContactPredictionHead,
+)
+
+
+class ProteinBertModel(nn.Module):
+    @classmethod
+    def add_args(cls, parser):
+        parser.add_argument(
+            "--num_layers", default=36, type=int, metavar="N", help="number of layers"
+        )
+        parser.add_argument(
+            "--embed_dim", default=1280, type=int, metavar="N", help="embedding dimension"
+        )
+        parser.add_argument(
+            "--logit_bias", action="store_true", help="whether to apply bias to logits"
+        )
+        parser.add_argument(
+            "--ffn_embed_dim",
+            default=5120,
+            type=int,
+            metavar="N",
+            help="embedding dimension for FFN",
+        )
+        parser.add_argument(
+            "--attention_heads",
+            default=20,
+            type=int,
+            metavar="N",
+            help="number of attention heads",
+        )
+
+    def __init__(self, args, alphabet):
+        super().__init__()
+        self.args = args
+        self.alphabet_size = len(alphabet)
+        self.padding_idx = alphabet.padding_idx
+        self.mask_idx = alphabet.mask_idx
+        self.cls_idx = alphabet.cls_idx
+        self.eos_idx = alphabet.eos_idx
+        self.prepend_bos = alphabet.prepend_bos
+        self.append_eos = alphabet.append_eos
+        self.emb_layer_norm_before = getattr(self.args, "emb_layer_norm_before", False)
+        if self.args.arch == "roberta_large":
+            self.model_version = "ESM-1b"
+            self._init_submodules_esm1b()
+        else:
+            self.model_version = "ESM-1"
+            self._init_submodules_esm1()
+
+    def _init_submodules_common(self):
+        self.embed_tokens = nn.Embedding(
+            self.alphabet_size, self.args.embed_dim, padding_idx=self.padding_idx
+        )
+        self.layers = nn.ModuleList(
+            [
+                TransformerLayer(
+                    self.args.embed_dim,
+                    self.args.ffn_embed_dim,
+                    self.args.attention_heads,
+                    add_bias_kv=(self.model_version != "ESM-1b"),
+                    use_esm1b_layer_norm=(self.model_version == "ESM-1b"),
+                )
+                for _ in range(self.args.layers)
+            ]
+        )
+
+        self.contact_head = ContactPredictionHead(
+            self.args.layers * self.args.attention_heads,
+            self.prepend_bos,
+            self.append_eos,
+            eos_idx=self.eos_idx,
+        )
+
+    def _init_submodules_esm1b(self):
+        self._init_submodules_common()
+        self.embed_scale = 1
+        self.embed_positions = LearnedPositionalEmbedding(
+            self.args.max_positions, self.args.embed_dim, self.padding_idx
+        )
+        self.emb_layer_norm_before = (
+            ESM1bLayerNorm(self.args.embed_dim) if self.emb_layer_norm_before else None
+        )
+        self.emb_layer_norm_after = ESM1bLayerNorm(self.args.embed_dim)
+        self.lm_head = RobertaLMHead(
+            embed_dim=self.args.embed_dim,
+            output_dim=self.alphabet_size,
+            weight=self.embed_tokens.weight,
+        )
+
+    def _init_submodules_esm1(self):
+        self._init_submodules_common()
+        self.embed_scale = math.sqrt(self.args.embed_dim)
+        self.embed_positions = SinusoidalPositionalEmbedding(self.args.embed_dim, self.padding_idx)
+        self.embed_out = nn.Parameter(torch.zeros((self.alphabet_size, self.args.embed_dim)))
+        self.embed_out_bias = None
+        if self.args.final_bias:
+            self.embed_out_bias = nn.Parameter(torch.zeros(self.alphabet_size))
+
+    def forward(self, tokens, repr_layers=[], need_head_weights=False, return_contacts=False):
+        if return_contacts:
+            need_head_weights = True
+
+        assert tokens.ndim == 2
+        padding_mask = tokens.eq(self.padding_idx)  # B, T
+
+        x = self.embed_scale * self.embed_tokens(tokens)
+
+        if getattr(self.args, "token_dropout", False):
+            x.masked_fill_((tokens == self.mask_idx).unsqueeze(-1), 0.0)
+            # x: B x T x C
+            mask_ratio_train = 0.15 * 0.8
+            src_lengths = (~padding_mask).sum(-1)
+            mask_ratio_observed = (tokens == self.mask_idx).sum(-1).float() / src_lengths
+            x = x * (1 - mask_ratio_train) / (1 - mask_ratio_observed)[:, None, None]
+
+        x = x + self.embed_positions(tokens)
+
+        if self.model_version == "ESM-1b":
+            if self.emb_layer_norm_before:
+                x = self.emb_layer_norm_before(x)
+            if padding_mask is not None:
+                x = x * (1 - padding_mask.unsqueeze(-1).type_as(x))
+
+        repr_layers = set(repr_layers)
+        hidden_representations = {}
+        if 0 in repr_layers:
+            hidden_representations[0] = x
+
+        if need_head_weights:
+            attn_weights = []
+
+        # (B, T, E) => (T, B, E)
+        x = x.transpose(0, 1)
+
+        if not padding_mask.any():
+            padding_mask = None
+
+        for layer_idx, layer in enumerate(self.layers):
+            x, attn = layer(
+                x, self_attn_padding_mask=padding_mask, need_head_weights=need_head_weights
+            )
+            if (layer_idx + 1) in repr_layers:
+                hidden_representations[layer_idx + 1] = x.transpose(0, 1)
+            if need_head_weights:
+                # (H, B, T, T) => (B, H, T, T)
+                attn_weights.append(attn.transpose(1, 0))
+
+        if self.model_version == "ESM-1b":
+            x = self.emb_layer_norm_after(x)
+            x = x.transpose(0, 1)  # (T, B, E) => (B, T, E)
+
+            # last hidden representation should have layer norm applied
+            if (layer_idx + 1) in repr_layers:
+                hidden_representations[layer_idx + 1] = x
+            x = self.lm_head(x)
+        else:
+            x = F.linear(x, self.embed_out, bias=self.embed_out_bias)
+            x = x.transpose(0, 1)  # (T, B, E) => (B, T, E)
+
+        result = {"logits": x, "representations": hidden_representations}
+        if need_head_weights:
+            # attentions: B x L x H x T x T
+            attentions = torch.stack(attn_weights, 1)
+            if self.model_version == "ESM-1":
+                # ESM-1 models have an additional null-token for attention, which we remove
+                attentions = attentions[..., :-1]
+            if padding_mask is not None:
+                attention_mask = 1 - padding_mask.type_as(attentions)
+                attention_mask = attention_mask.unsqueeze(1) * attention_mask.unsqueeze(2)
+                attentions = attentions * attention_mask[:, None, None, :, :]
+            result["attentions"] = attentions
+            if return_contacts:
+                contacts = self.contact_head(tokens, attentions)
+                result["contacts"] = contacts
+
+        return result
+
+    def predict_contacts(self, tokens):
+        return self(tokens, return_contacts=True)["contacts"]
+
+    @property
+    def num_layers(self):
+        return self.args.layers

esm/esm/model/esm2.py

@@ -0,0 +1,147 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from typing import Union
+import torch
+import torch.nn as nn
+
+import esm
+from esm.modules import ContactPredictionHead, ESM1bLayerNorm, RobertaLMHead, TransformerLayer
+
+
+class ESM2(nn.Module):
+    def __init__(
+        self,
+        num_layers: int = 33,
+        embed_dim: int = 1280,
+        attention_heads: int = 20,
+        alphabet: Union[esm.data.Alphabet, str] = "ESM-1b",
+        token_dropout: bool = True,
+    ):
+        super().__init__()
+        self.num_layers = num_layers
+        self.embed_dim = embed_dim
+        self.attention_heads = attention_heads
+        if not isinstance(alphabet, esm.data.Alphabet):
+            alphabet = esm.data.Alphabet.from_architecture(alphabet)
+        self.alphabet = alphabet
+        self.alphabet_size = len(alphabet)
+        self.padding_idx = alphabet.padding_idx
+        self.mask_idx = alphabet.mask_idx
+        self.cls_idx = alphabet.cls_idx
+        self.eos_idx = alphabet.eos_idx
+        self.prepend_bos = alphabet.prepend_bos
+        self.append_eos = alphabet.append_eos
+        self.token_dropout = token_dropout
+
+        self._init_submodules()
+
+    def _init_submodules(self):
+        self.embed_scale = 1
+        self.embed_tokens = nn.Embedding(
+            self.alphabet_size,
+            self.embed_dim,
+            padding_idx=self.padding_idx,
+        )
+
+        self.layers = nn.ModuleList(
+            [
+                TransformerLayer(
+                    self.embed_dim,
+                    4 * self.embed_dim,
+                    self.attention_heads,
+                    add_bias_kv=False,
+                    use_esm1b_layer_norm=True,
+                    use_rotary_embeddings=True,
+                )
+                for _ in range(self.num_layers)
+            ]
+        )
+
+        self.contact_head = ContactPredictionHead(
+            self.num_layers * self.attention_heads,
+            self.prepend_bos,
+            self.append_eos,
+            eos_idx=self.eos_idx,
+        )
+        self.emb_layer_norm_after = ESM1bLayerNorm(self.embed_dim)
+
+        self.lm_head = RobertaLMHead(
+            embed_dim=self.embed_dim,
+            output_dim=self.alphabet_size,
+            weight=self.embed_tokens.weight,
+        )
+
+    def forward(self, tokens, repr_layers=[], need_head_weights=False, return_contacts=False):
+        if return_contacts:
+            need_head_weights = True
+
+        assert tokens.ndim == 2
+        padding_mask = tokens.eq(self.padding_idx)  # B, T
+
+        x = self.embed_scale * self.embed_tokens(tokens)
+
+        if self.token_dropout:
+            x.masked_fill_((tokens == self.mask_idx).unsqueeze(-1), 0.0)
+            # x: B x T x C
+            mask_ratio_train = 0.15 * 0.8
+            src_lengths = (~padding_mask).sum(-1)
+            mask_ratio_observed = (tokens == self.mask_idx).sum(-1).to(x.dtype) / src_lengths
+            x = x * (1 - mask_ratio_train) / (1 - mask_ratio_observed)[:, None, None]
+
+        if padding_mask is not None:
+            x = x * (1 - padding_mask.unsqueeze(-1).type_as(x))
+
+        repr_layers = set(repr_layers)
+        hidden_representations = {}
+        if 0 in repr_layers:
+            hidden_representations[0] = x
+
+        if need_head_weights:
+            attn_weights = []
+
+        # (B, T, E) => (T, B, E)
+        x = x.transpose(0, 1)
+
+        if not padding_mask.any():
+            padding_mask = None
+
+        for layer_idx, layer in enumerate(self.layers):
+            x, attn = layer(
+                x,
+                self_attn_padding_mask=padding_mask,
+                need_head_weights=need_head_weights,
+            )
+            if (layer_idx + 1) in repr_layers:
+                hidden_representations[layer_idx + 1] = x.transpose(0, 1)
+            if need_head_weights:
+                # (H, B, T, T) => (B, H, T, T)
+                attn_weights.append(attn.transpose(1, 0))
+
+        x = self.emb_layer_norm_after(x)
+        x = x.transpose(0, 1)  # (T, B, E) => (B, T, E)
+
+        # last hidden representation should have layer norm applied
+        if (layer_idx + 1) in repr_layers:
+            hidden_representations[layer_idx + 1] = x
+        x = self.lm_head(x)
+
+        result = {"logits": x, "representations": hidden_representations}
+        if need_head_weights:
+            # attentions: B x L x H x T x T
+            attentions = torch.stack(attn_weights, 1)
+            if padding_mask is not None:
+                attention_mask = 1 - padding_mask.type_as(attentions)
+                attention_mask = attention_mask.unsqueeze(1) * attention_mask.unsqueeze(2)
+                attentions = attentions * attention_mask[:, None, None, :, :]
+            result["attentions"] = attentions
+            if return_contacts:
+                contacts = self.contact_head(tokens, attentions)
+                result["contacts"] = contacts
+
+        return result
+
+    def predict_contacts(self, tokens):
+        return self(tokens, return_contacts=True)["contacts"]

esm/esm/model/msa_transformer.py

@@ -0,0 +1,238 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import torch
+import torch.nn as nn
+
+from ..modules import (
+    AxialTransformerLayer,
+    LearnedPositionalEmbedding,
+    RobertaLMHead,
+    ESM1bLayerNorm,
+    ContactPredictionHead,
+)
+
+from ..axial_attention import RowSelfAttention, ColumnSelfAttention
+
+
+
+class MSATransformer(nn.Module):
+    @classmethod
+    def add_args(cls, parser):
+        # fmt: off
+        parser.add_argument(
+            "--num_layers",
+            default=12,
+            type=int,
+            metavar="N",
+            help="number of layers"
+        )
+        parser.add_argument(
+            "--embed_dim",
+            default=768,
+            type=int,
+            metavar="N",
+            help="embedding dimension"
+        )
+        parser.add_argument(
+            "--logit_bias",
+            action="store_true",
+            help="whether to apply bias to logits"
+        )
+        parser.add_argument(
+            "--ffn_embed_dim",
+            default=3072,
+            type=int,
+            metavar="N",
+            help="embedding dimension for FFN",
+        )
+        parser.add_argument(
+            "--attention_heads",
+            default=12,
+            type=int,
+            metavar="N",
+            help="number of attention heads",
+        )
+        parser.add_argument(
+            "--dropout",
+            default=0.1,
+            type=float,
+            help="Dropout to apply."
+        )
+        parser.add_argument(
+            "--attention_dropout",
+            default=0.1,
+            type=float,
+            help="Dropout to apply."
+        )
+        parser.add_argument(
+            "--activation_dropout",
+            default=0.1,
+            type=float,
+            help="Dropout to apply."
+        )
+        parser.add_argument(
+            "--max_tokens_per_msa",
+            default=2 ** 14,
+            type=int,
+            help=(
+                "Used during inference to batch attention computations in a single "
+                "forward pass. This allows increased input sizes with less memory."
+            ),
+        )
+        # fmt: on
+
+    def __init__(self, args, alphabet):
+        super().__init__()
+        self.args = args
+        self.alphabet_size = len(alphabet)
+        self.padding_idx = alphabet.padding_idx
+        self.mask_idx = alphabet.mask_idx
+        self.cls_idx = alphabet.cls_idx
+        self.eos_idx = alphabet.eos_idx
+        self.prepend_bos = alphabet.prepend_bos
+        self.append_eos = alphabet.append_eos
+
+        self.embed_tokens = nn.Embedding(
+            self.alphabet_size, self.args.embed_dim, padding_idx=self.padding_idx
+        )
+
+        if getattr(self.args, "embed_positions_msa", False):
+            emb_dim = getattr(self.args, "embed_positions_msa_dim", self.args.embed_dim)
+            self.msa_position_embedding = nn.Parameter(
+                0.01 * torch.randn(1, 1024, 1, emb_dim),
+                requires_grad=True,
+            )
+        else:
+            self.register_parameter("msa_position_embedding", None)
+
+        self.dropout_module = nn.Dropout(self.args.dropout)
+        self.layers = nn.ModuleList(
+            [
+                AxialTransformerLayer(
+                    self.args.embed_dim,
+                    self.args.ffn_embed_dim,
+                    self.args.attention_heads,
+                    self.args.dropout,
+                    self.args.attention_dropout,
+                    self.args.activation_dropout,
+                    getattr(self.args, "max_tokens_per_msa", self.args.max_tokens),
+                )
+                for _ in range(self.args.layers)
+            ]
+        )
+
+        self.contact_head = ContactPredictionHead(
+            self.args.layers * self.args.attention_heads,
+            self.prepend_bos,
+            self.append_eos,
+            eos_idx=self.eos_idx,
+        )
+        self.embed_positions = LearnedPositionalEmbedding(
+            self.args.max_positions,
+            self.args.embed_dim,
+            self.padding_idx,
+        )
+        self.emb_layer_norm_before = ESM1bLayerNorm(self.args.embed_dim)
+        self.emb_layer_norm_after = ESM1bLayerNorm(self.args.embed_dim)
+        self.lm_head = RobertaLMHead(
+            embed_dim=self.args.embed_dim,
+            output_dim=self.alphabet_size,
+            weight=self.embed_tokens.weight,
+        )
+
+    def forward(self, tokens, repr_layers=[], need_head_weights=False, return_contacts=False):
+        if return_contacts:
+            need_head_weights = True
+
+        assert tokens.ndim == 3
+        batch_size, num_alignments, seqlen = tokens.size()
+        padding_mask = tokens.eq(self.padding_idx)  # B, R, C
+        if not padding_mask.any():
+            padding_mask = None
+
+        x = self.embed_tokens(tokens)
+        x += self.embed_positions(tokens.view(batch_size * num_alignments, seqlen)).view(x.size())
+        if self.msa_position_embedding is not None:
+            if x.size(1) > 1024:
+                raise RuntimeError(
+                    "Using model with MSA position embedding trained on maximum MSA "
+                    f"depth of 1024, but received {x.size(1)} alignments."
+                )
+            x += self.msa_position_embedding[:, :num_alignments]
+
+        x = self.emb_layer_norm_before(x)
+
+        x = self.dropout_module(x)
+
+        if padding_mask is not None:
+            x = x * (1 - padding_mask.unsqueeze(-1).type_as(x))
+
+        repr_layers = set(repr_layers)
+        hidden_representations = {}
+        if 0 in repr_layers:
+            hidden_representations[0] = x
+
+        if need_head_weights:
+            row_attn_weights = []
+            col_attn_weights = []
+
+        # B x R x C x D -> R x C x B x D
+        x = x.permute(1, 2, 0, 3)
+
+        for layer_idx, layer in enumerate(self.layers):
+            x = layer(
+                x,
+                self_attn_padding_mask=padding_mask,
+                need_head_weights=need_head_weights,
+            )
+            if need_head_weights:
+                x, col_attn, row_attn = x
+                # H x C x B x R x R -> B x H x C x R x R
+                col_attn_weights.append(col_attn.permute(2, 0, 1, 3, 4))
+                # H x B x C x C -> B x H x C x C
+                row_attn_weights.append(row_attn.permute(1, 0, 2, 3))
+            if (layer_idx + 1) in repr_layers:
+                hidden_representations[layer_idx + 1] = x.permute(2, 0, 1, 3)
+
+        x = self.emb_layer_norm_after(x)
+        x = x.permute(2, 0, 1, 3)  # R x C x B x D -> B x R x C x D
+
+        # last hidden representation should have layer norm applied
+        if (layer_idx + 1) in repr_layers:
+            hidden_representations[layer_idx + 1] = x
+        x = self.lm_head(x)
+
+        result = {"logits": x, "representations": hidden_representations}
+        if need_head_weights:
+            # col_attentions: B x L x H x C x R x R
+            col_attentions = torch.stack(col_attn_weights, 1)
+            # row_attentions: B x L x H x C x C
+            row_attentions = torch.stack(row_attn_weights, 1)
+            result["col_attentions"] = col_attentions
+            result["row_attentions"] = row_attentions
+            if return_contacts:
+                contacts = self.contact_head(tokens, row_attentions)
+                result["contacts"] = contacts
+
+        return result
+
+    def predict_contacts(self, tokens):
+        return self(tokens, return_contacts=True)["contacts"]
+
+    @property
+    def num_layers(self):
+        return self.args.layers
+
+    def max_tokens_per_msa_(self, value: int) -> None:
+        """The MSA Transformer automatically batches attention computations when
+        gradients are disabled to allow you to pass in larger MSAs at test time than
+        you can fit in GPU memory. By default this occurs when more than 2^14 tokens
+        are passed in the input MSA. You can set this value to infinity to disable
+        this behavior.
+        """
+        for module in self.modules():
+            if isinstance(module, (RowSelfAttention, ColumnSelfAttention)):
+                module.max_tokens_per_msa = value

esm/esm/modules.py

@@ -0,0 +1,418 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import math
+from typing import Optional
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from .multihead_attention import MultiheadAttention  # noqa
+from .axial_attention import ColumnSelfAttention, RowSelfAttention
+
+
+def gelu(x):
+    """Implementation of the gelu activation function.
+
+    For information: OpenAI GPT's gelu is slightly different
+    (and gives slightly different results):
+    0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
+    """
+    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
+
+
+def symmetrize(x):
+    "Make layer symmetric in final two dimensions, used for contact prediction."
+    return x + x.transpose(-1, -2)
+
+
+def apc(x):
+    "Perform average product correct, used for contact prediction."
+    a1 = x.sum(-1, keepdims=True)
+    a2 = x.sum(-2, keepdims=True)
+    a12 = x.sum((-1, -2), keepdims=True)
+
+    avg = a1 * a2
+    avg.div_(a12)  # in-place to reduce memory
+    normalized = x - avg
+    return normalized
+
+
+class ESM1LayerNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-12, affine=True):
+        """Construct a layernorm layer in the TF style (eps inside the sqrt)."""
+        super().__init__()
+        self.hidden_size = (hidden_size,) if isinstance(hidden_size, int) else tuple(hidden_size)
+        self.eps = eps
+        self.affine = bool(affine)
+        if self.affine:
+            self.weight = nn.Parameter(torch.ones(hidden_size))
+            self.bias = nn.Parameter(torch.zeros(hidden_size))
+        else:
+            self.weight, self.bias = None, None
+
+    def forward(self, x):
+        dims = tuple(-(i + 1) for i in range(len(self.hidden_size)))
+        means = x.mean(dims, keepdim=True)
+        x_zeromean = x - means
+        variances = x_zeromean.pow(2).mean(dims, keepdim=True)
+        x = x_zeromean / torch.sqrt(variances + self.eps)
+        if self.affine:
+            x = (self.weight * x) + self.bias
+        return x
+
+
+try:
+    from apex.normalization import FusedLayerNorm as _FusedLayerNorm
+
+    class ESM1bLayerNorm(_FusedLayerNorm):
+        @torch.jit.unused
+        def forward(self, x):
+            if not x.is_cuda:
+                return super().forward(x)
+            else:
+                with torch.cuda.device(x.device):
+                    return super().forward(x)
+
+except ImportError:
+    from torch.nn import LayerNorm as ESM1bLayerNorm
+
+
+class TransformerLayer(nn.Module):
+    """Transformer layer block."""
+
+    def __init__(
+        self,
+        embed_dim,
+        ffn_embed_dim,
+        attention_heads,
+        add_bias_kv=True,
+        use_esm1b_layer_norm=False,
+        use_rotary_embeddings: bool = False,
+    ):
+        super().__init__()
+        self.embed_dim = embed_dim
+        self.ffn_embed_dim = ffn_embed_dim
+        self.attention_heads = attention_heads
+        self.use_rotary_embeddings = use_rotary_embeddings
+        self._init_submodules(add_bias_kv, use_esm1b_layer_norm)
+
+    def _init_submodules(self, add_bias_kv, use_esm1b_layer_norm):
+        BertLayerNorm = ESM1bLayerNorm if use_esm1b_layer_norm else ESM1LayerNorm
+
+        self.self_attn = MultiheadAttention(
+            self.embed_dim,
+            self.attention_heads,
+            add_bias_kv=add_bias_kv,
+            add_zero_attn=False,
+            use_rotary_embeddings=self.use_rotary_embeddings,
+        )
+        self.self_attn_layer_norm = BertLayerNorm(self.embed_dim)
+
+        self.fc1 = nn.Linear(self.embed_dim, self.ffn_embed_dim)
+        self.fc2 = nn.Linear(self.ffn_embed_dim, self.embed_dim)
+
+        self.final_layer_norm = BertLayerNorm(self.embed_dim)
+
+    def forward(
+        self, x, self_attn_mask=None, self_attn_padding_mask=None, need_head_weights=False
+    ):
+        residual = x
+        x = self.self_attn_layer_norm(x)
+        x, attn = self.self_attn(
+            query=x,
+            key=x,
+            value=x,
+            key_padding_mask=self_attn_padding_mask,
+            need_weights=True,
+            need_head_weights=need_head_weights,
+            attn_mask=self_attn_mask,
+        )
+        x = residual + x
+
+        residual = x
+        x = self.final_layer_norm(x)
+        x = gelu(self.fc1(x))
+        x = self.fc2(x)
+        x = residual + x
+
+        return x, attn
+
+
+class AxialTransformerLayer(nn.Module):
+    """Implements an Axial MSA Transformer block."""
+
+    def __init__(
+        self,
+        embedding_dim: int = 768,
+        ffn_embedding_dim: int = 3072,
+        num_attention_heads: int = 8,
+        dropout: float = 0.1,
+        attention_dropout: float = 0.1,
+        activation_dropout: float = 0.1,
+        max_tokens_per_msa: int = 2**14,
+    ) -> None:
+        super().__init__()
+
+        # Initialize parameters
+        self.embedding_dim = embedding_dim
+        self.dropout_prob = dropout
+
+        row_self_attention = RowSelfAttention(
+            embedding_dim,
+            num_attention_heads,
+            dropout=dropout,
+            max_tokens_per_msa=max_tokens_per_msa,
+        )
+
+        column_self_attention = ColumnSelfAttention(
+            embedding_dim,
+            num_attention_heads,
+            dropout=dropout,
+            max_tokens_per_msa=max_tokens_per_msa,
+        )
+
+        feed_forward_layer = FeedForwardNetwork(
+            embedding_dim,
+            ffn_embedding_dim,
+            activation_dropout=activation_dropout,
+            max_tokens_per_msa=max_tokens_per_msa,
+        )
+
+        self.row_self_attention = self.build_residual(row_self_attention)
+        self.column_self_attention = self.build_residual(column_self_attention)
+        self.feed_forward_layer = self.build_residual(feed_forward_layer)
+
+    def build_residual(self, layer: nn.Module):
+        return NormalizedResidualBlock(
+            layer,
+            self.embedding_dim,
+            self.dropout_prob,
+        )
+
+    def forward(
+        self,
+        x: torch.Tensor,
+        self_attn_mask: Optional[torch.Tensor] = None,
+        self_attn_padding_mask: Optional[torch.Tensor] = None,
+        need_head_weights: bool = False,
+    ):
+        """
+        LayerNorm is applied either before or after the self-attention/ffn
+        modules similar to the original Transformer implementation.
+        """
+        x, row_attn = self.row_self_attention(
+            x,
+            self_attn_mask=self_attn_mask,
+            self_attn_padding_mask=self_attn_padding_mask,
+        )
+        x, column_attn = self.column_self_attention(
+            x,
+            self_attn_mask=self_attn_mask,
+            self_attn_padding_mask=self_attn_padding_mask,
+        )
+        x = self.feed_forward_layer(x)
+        if need_head_weights:
+            return x, column_attn, row_attn
+        else:
+            return x
+
+
+class LearnedPositionalEmbedding(nn.Embedding):
+    """
+    This module learns positional embeddings up to a fixed maximum size.
+    Padding ids are ignored by either offsetting based on padding_idx
+    or by setting padding_idx to None and ensuring that the appropriate
+    position ids are passed to the forward function.
+    """
+
+    def __init__(self, num_embeddings: int, embedding_dim: int, padding_idx: int):
+        if padding_idx is not None:
+            num_embeddings_ = num_embeddings + padding_idx + 1
+        else:
+            num_embeddings_ = num_embeddings
+        super().__init__(num_embeddings_, embedding_dim, padding_idx)
+        self.max_positions = num_embeddings
+
+    def forward(self, input: torch.Tensor):
+        """Input is expected to be of size [bsz x seqlen]."""
+        if input.size(1) > self.max_positions:
+            raise ValueError(
+                f"Sequence length {input.size(1)} above maximum "
+                f" sequence length of {self.max_positions}"
+            )
+        mask = input.ne(self.padding_idx).int()
+        positions = (torch.cumsum(mask, dim=1).type_as(mask) * mask).long() + self.padding_idx
+        return F.embedding(
+            positions,
+            self.weight,
+            self.padding_idx,
+            self.max_norm,
+            self.norm_type,
+            self.scale_grad_by_freq,
+            self.sparse,
+        )
+
+
+class SinusoidalPositionalEmbedding(nn.Module):
+    def __init__(self, embed_dim, padding_idx, learned=False):
+        super().__init__()
+        self.embed_dim = embed_dim
+        self.padding_idx = padding_idx
+        self.register_buffer("_float_tensor", torch.FloatTensor(1))
+        self.weights = None
+
+    def forward(self, x):
+        bsz, seq_len = x.shape
+        max_pos = self.padding_idx + 1 + seq_len
+        if self.weights is None or max_pos > self.weights.size(0):
+            self.weights = self.get_embedding(max_pos)
+        self.weights = self.weights.type_as(self._float_tensor)
+
+        positions = self.make_positions(x)
+        return self.weights.index_select(0, positions.view(-1)).view(bsz, seq_len, -1).detach()
+
+    def make_positions(self, x):
+        mask = x.ne(self.padding_idx)
+        range_buf = torch.arange(x.size(1), device=x.device).expand_as(x) + self.padding_idx + 1
+        positions = range_buf.expand_as(x)
+        return positions * mask.long() + self.padding_idx * (1 - mask.long())
+
+    def get_embedding(self, num_embeddings):
+        half_dim = self.embed_dim // 2
+        emb = math.log(10000) / (half_dim - 1)
+        emb = torch.exp(torch.arange(half_dim, dtype=torch.float) * -emb)
+        emb = torch.arange(num_embeddings, dtype=torch.float).unsqueeze(1) * emb.unsqueeze(0)
+        emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1).view(num_embeddings, -1)
+        if self.embed_dim % 2 == 1:
+            # zero pad
+            emb = torch.cat([emb, torch.zeros(num_embeddings, 1)], dim=1)
+        if self.padding_idx is not None:
+            emb[self.padding_idx, :] = 0
+        return emb
+
+
+class RobertaLMHead(nn.Module):
+    """Head for masked language modeling."""
+
+    def __init__(self, embed_dim, output_dim, weight):
+        super().__init__()
+        self.dense = nn.Linear(embed_dim, embed_dim)
+        self.layer_norm = ESM1bLayerNorm(embed_dim)
+        self.weight = weight
+        self.bias = nn.Parameter(torch.zeros(output_dim))
+
+    def forward(self, features):
+        x = self.dense(features)
+        x = gelu(x)
+        x = self.layer_norm(x)
+        # project back to size of vocabulary with bias
+        x = F.linear(x, self.weight) + self.bias
+        return x
+
+
+class ContactPredictionHead(nn.Module):
+    """Performs symmetrization, apc, and computes a logistic regression on the output features"""
+
+    def __init__(
+        self,
+        in_features: int,
+        prepend_bos: bool,
+        append_eos: bool,
+        bias=True,
+        eos_idx: Optional[int] = None,
+    ):
+        super().__init__()
+        self.in_features = in_features
+        self.prepend_bos = prepend_bos
+        self.append_eos = append_eos
+        if append_eos and eos_idx is None:
+            raise ValueError("Using an alphabet with eos token, but no eos token was passed in.")
+        self.eos_idx = eos_idx
+        self.regression = nn.Linear(in_features, 1, bias)
+        self.activation = nn.Sigmoid()
+
+    def forward(self, tokens, attentions):
+        # remove eos token attentions
+        if self.append_eos:
+            eos_mask = tokens.ne(self.eos_idx).to(attentions)
+            eos_mask = eos_mask.unsqueeze(1) * eos_mask.unsqueeze(2)
+            attentions = attentions * eos_mask[:, None, None, :, :]
+            attentions = attentions[..., :-1, :-1]
+        # remove cls token attentions
+        if self.prepend_bos:
+            attentions = attentions[..., 1:, 1:]
+        batch_size, layers, heads, seqlen, _ = attentions.size()
+        attentions = attentions.view(batch_size, layers * heads, seqlen, seqlen)
+
+        # features: B x C x T x T
+        attentions = attentions.to(
+            self.regression.weight.device
+        )  # attentions always float32, may need to convert to float16
+        attentions = apc(symmetrize(attentions))
+        attentions = attentions.permute(0, 2, 3, 1)
+        return self.activation(self.regression(attentions).squeeze(3))
+
+
+class NormalizedResidualBlock(nn.Module):
+    def __init__(
+        self,
+        layer: nn.Module,
+        embedding_dim: int,
+        dropout: float = 0.1,
+    ):
+        super().__init__()
+        self.embedding_dim = embedding_dim
+
+        self.layer = layer
+        self.dropout_module = nn.Dropout(
+            dropout,
+        )
+        self.layer_norm = ESM1bLayerNorm(self.embedding_dim)
+
+    def forward(self, x, *args, **kwargs):
+        residual = x
+        x = self.layer_norm(x)
+        outputs = self.layer(x, *args, **kwargs)
+        if isinstance(outputs, tuple):
+            x, *out = outputs
+        else:
+            x = outputs
+            out = None
+
+        x = self.dropout_module(x)
+        x = residual + x
+
+        if out is not None:
+            return (x,) + tuple(out)
+        else:
+            return x
+
+
+class FeedForwardNetwork(nn.Module):
+    def __init__(
+        self,
+        embedding_dim: int,
+        ffn_embedding_dim: int,
+        activation_dropout: float = 0.1,
+        max_tokens_per_msa: int = 2**14,
+    ):
+        super().__init__()
+        self.embedding_dim = embedding_dim
+        self.ffn_embedding_dim = ffn_embedding_dim
+        self.max_tokens_per_msa = max_tokens_per_msa
+        self.activation_fn = nn.GELU()
+        self.activation_dropout_module = nn.Dropout(
+            activation_dropout,
+        )
+        self.fc1 = nn.Linear(embedding_dim, ffn_embedding_dim)
+        self.fc2 = nn.Linear(ffn_embedding_dim, embedding_dim)
+
+    def forward(self, x):
+        x = self.activation_fn(self.fc1(x))
+        x = self.activation_dropout_module(x)
+        x = self.fc2(x)
+        return x

esm/esm/multihead_attention.py

@@ -0,0 +1,508 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import math
+from typing import Dict, Optional, Tuple
+
+import torch
+import torch.nn.functional as F
+from torch import Tensor, nn
+from torch.nn import Parameter
+from esm.rotary_embedding import RotaryEmbedding
+
+import uuid
+
+
+def utils_softmax(x, dim: int, onnx_trace: bool = False):
+    if onnx_trace:
+        return F.softmax(x.float(), dim=dim)
+    else:
+        return F.softmax(x, dim=dim, dtype=torch.float32)
+
+
+class FairseqIncrementalState(object):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.init_incremental_state()
+
+    def init_incremental_state(self):
+        self._incremental_state_id = str(uuid.uuid4())
+
+    def _get_full_incremental_state_key(self, key: str) -> str:
+        return "{}.{}".format(self._incremental_state_id, key)
+
+    def get_incremental_state(
+        self,
+        incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]],
+        key: str,
+    ) -> Optional[Dict[str, Optional[Tensor]]]:
+        """Helper for getting incremental state for an nn.Module."""
+        full_key = self._get_full_incremental_state_key(key)
+        if incremental_state is None or full_key not in incremental_state:
+            return None
+        return incremental_state[full_key]
+
+    def set_incremental_state(
+        self,
+        incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]],
+        key: str,
+        value: Dict[str, Optional[Tensor]],
+    ) -> Optional[Dict[str, Dict[str, Optional[Tensor]]]]:
+        """Helper for setting incremental state for an nn.Module."""
+        if incremental_state is not None:
+            full_key = self._get_full_incremental_state_key(key)
+            incremental_state[full_key] = value
+        return incremental_state
+
+
+def with_incremental_state(cls):
+    cls.__bases__ = (FairseqIncrementalState,) + tuple(
+        b for b in cls.__bases__ if b != FairseqIncrementalState
+    )
+    return cls
+
+
+@with_incremental_state
+class MultiheadAttention(nn.Module):
+    """Multi-headed attention.
+
+    See "Attention Is All You Need" for more details.
+    """
+
+    def __init__(
+        self,
+        embed_dim,
+        num_heads,
+        kdim=None,
+        vdim=None,
+        dropout=0.0,
+        bias=True,
+        add_bias_kv: bool = False,
+        add_zero_attn: bool = False,
+        self_attention: bool = False,
+        encoder_decoder_attention: bool = False,
+        use_rotary_embeddings: bool = False,
+    ):
+        super().__init__()
+        self.embed_dim = embed_dim
+        self.kdim = kdim if kdim is not None else embed_dim
+        self.vdim = vdim if vdim is not None else embed_dim
+        self.qkv_same_dim = self.kdim == embed_dim and self.vdim == embed_dim
+
+        self.num_heads = num_heads
+        self.dropout = dropout
+        self.head_dim = embed_dim // num_heads
+        assert (
+            self.head_dim * num_heads == self.embed_dim
+        ), "embed_dim must be divisible by num_heads"
+        self.scaling = self.head_dim**-0.5
+
+        self.self_attention = self_attention
+        self.encoder_decoder_attention = encoder_decoder_attention
+
+        assert not self.self_attention or self.qkv_same_dim, (
+            "Self-attention requires query, key and " "value to be of the same size"
+        )
+
+        self.k_proj = nn.Linear(self.kdim, embed_dim, bias=bias)
+        self.v_proj = nn.Linear(self.vdim, embed_dim, bias=bias)
+        self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
+
+        self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
+
+        if add_bias_kv:
+            self.bias_k = Parameter(torch.Tensor(1, 1, embed_dim))
+            self.bias_v = Parameter(torch.Tensor(1, 1, embed_dim))
+        else:
+            self.bias_k = self.bias_v = None
+
+        self.add_zero_attn = add_zero_attn
+
+        self.reset_parameters()
+
+        self.onnx_trace = False
+        self.rot_emb = None
+        if use_rotary_embeddings:
+            self.rot_emb = RotaryEmbedding(dim=self.head_dim)
+
+        self.enable_torch_version = False
+        if hasattr(F, "multi_head_attention_forward"):
+            self.enable_torch_version = True
+        else:
+            self.enable_torch_version = False
+
+    def prepare_for_onnx_export_(self):
+        self.onnx_trace = True
+
+    def reset_parameters(self):
+        if self.qkv_same_dim:
+            # Empirically observed the convergence to be much better with
+            # the scaled initialization
+            nn.init.xavier_uniform_(self.k_proj.weight, gain=1 / math.sqrt(2))
+            nn.init.xavier_uniform_(self.v_proj.weight, gain=1 / math.sqrt(2))
+            nn.init.xavier_uniform_(self.q_proj.weight, gain=1 / math.sqrt(2))
+        else:
+            nn.init.xavier_uniform_(self.k_proj.weight)
+            nn.init.xavier_uniform_(self.v_proj.weight)
+            nn.init.xavier_uniform_(self.q_proj.weight)
+
+        nn.init.xavier_uniform_(self.out_proj.weight)
+        if self.out_proj.bias is not None:
+            nn.init.constant_(self.out_proj.bias, 0.0)
+        if self.bias_k is not None:
+            nn.init.xavier_normal_(self.bias_k)
+        if self.bias_v is not None:
+            nn.init.xavier_normal_(self.bias_v)
+
+    def forward(
+        self,
+        query,
+        key: Optional[Tensor],
+        value: Optional[Tensor],
+        key_padding_mask: Optional[Tensor] = None,
+        incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None,
+        need_weights: bool = True,
+        static_kv: bool = False,
+        attn_mask: Optional[Tensor] = None,
+        before_softmax: bool = False,
+        need_head_weights: bool = False,
+    ) -> Tuple[Tensor, Optional[Tensor]]:
+        """Input shape: Time x Batch x Channel
+
+        Args:
+            key_padding_mask (ByteTensor, optional): mask to exclude
+                keys that are pads, of shape `(batch, src_len)`, where
+                padding elements are indicated by 1s.
+            need_weights (bool, optional): return the attention weights,
+                averaged over heads (default: False).
+            attn_mask (ByteTensor, optional): typically used to
+                implement causal attention, where the mask prevents the
+                attention from looking forward in time (default: None).
+            before_softmax (bool, optional): return the raw attention
+                weights and values before the attention softmax.
+            need_head_weights (bool, optional): return the attention
+                weights for each head. Implies *need_weights*. Default:
+                return the average attention weights over all heads.
+        """
+        if need_head_weights:
+            need_weights = True
+
+        tgt_len, bsz, embed_dim = query.size()
+        assert embed_dim == self.embed_dim
+        assert list(query.size()) == [tgt_len, bsz, embed_dim]
+
+        if (
+            not self.rot_emb
+            and self.enable_torch_version
+            and not self.onnx_trace
+            and incremental_state is None
+            and not static_kv
+            # A workaround for quantization to work. Otherwise JIT compilation
+            # treats bias in linear module as method.
+            and not torch.jit.is_scripting()
+            and not need_head_weights
+        ):
+            assert key is not None and value is not None
+            return F.multi_head_attention_forward(
+                query,
+                key,
+                value,
+                self.embed_dim,
+                self.num_heads,
+                torch.empty([0]),
+                torch.cat((self.q_proj.bias, self.k_proj.bias, self.v_proj.bias)),
+                self.bias_k,
+                self.bias_v,
+                self.add_zero_attn,
+                self.dropout,
+                self.out_proj.weight,
+                self.out_proj.bias,
+                self.training,
+                key_padding_mask,
+                need_weights,
+                attn_mask,
+                use_separate_proj_weight=True,
+                q_proj_weight=self.q_proj.weight,
+                k_proj_weight=self.k_proj.weight,
+                v_proj_weight=self.v_proj.weight,
+            )
+        if incremental_state is not None:
+            saved_state = self._get_input_buffer(incremental_state)
+            if saved_state is not None and "prev_key" in saved_state:
+                # previous time steps are cached - no need to recompute
+                # key and value if they are static
+                if static_kv:
+                    assert self.encoder_decoder_attention and not self.self_attention
+                    key = value = None
+        else:
+            saved_state = None
+
+        if self.self_attention:
+            q = self.q_proj(query)
+            k = self.k_proj(query)
+            v = self.v_proj(query)
+        elif self.encoder_decoder_attention:
+            # encoder-decoder attention
+            q = self.q_proj(query)
+            if key is None:
+                assert value is None
+                k = v = None
+            else:
+                k = self.k_proj(key)
+                v = self.v_proj(key)
+
+        else:
+            assert key is not None and value is not None
+            q = self.q_proj(query)
+            k = self.k_proj(key)
+            v = self.v_proj(value)
+        q *= self.scaling
+
+        if self.bias_k is not None:
+            assert self.bias_v is not None
+            k = torch.cat([k, self.bias_k.repeat(1, bsz, 1)])
+            v = torch.cat([v, self.bias_v.repeat(1, bsz, 1)])
+            if attn_mask is not None:
+                attn_mask = torch.cat(
+                    [attn_mask, attn_mask.new_zeros(attn_mask.size(0), 1)], dim=1
+                )
+            if key_padding_mask is not None:
+                key_padding_mask = torch.cat(
+                    [
+                        key_padding_mask,
+                        key_padding_mask.new_zeros(key_padding_mask.size(0), 1),
+                    ],
+                    dim=1,
+                )
+
+        q = q.contiguous().view(tgt_len, bsz * self.num_heads, self.head_dim).transpose(0, 1)
+        if k is not None:
+            k = k.contiguous().view(-1, bsz * self.num_heads, self.head_dim).transpose(0, 1)
+        if v is not None:
+            v = v.contiguous().view(-1, bsz * self.num_heads, self.head_dim).transpose(0, 1)
+
+        if saved_state is not None:
+            # saved states are stored with shape (bsz, num_heads, seq_len, head_dim)
+            if "prev_key" in saved_state:
+                _prev_key = saved_state["prev_key"]
+                assert _prev_key is not None
+                prev_key = _prev_key.view(bsz * self.num_heads, -1, self.head_dim)
+                if static_kv:
+                    k = prev_key
+                else:
+                    assert k is not None
+                    k = torch.cat([prev_key, k], dim=1)
+            if "prev_value" in saved_state:
+                _prev_value = saved_state["prev_value"]
+                assert _prev_value is not None
+                prev_value = _prev_value.view(bsz * self.num_heads, -1, self.head_dim)
+                if static_kv:
+                    v = prev_value
+                else:
+                    assert v is not None
+                    v = torch.cat([prev_value, v], dim=1)
+            prev_key_padding_mask: Optional[Tensor] = None
+            if "prev_key_padding_mask" in saved_state:
+                prev_key_padding_mask = saved_state["prev_key_padding_mask"]
+            assert k is not None and v is not None
+            key_padding_mask = MultiheadAttention._append_prev_key_padding_mask(
+                key_padding_mask=key_padding_mask,
+                prev_key_padding_mask=prev_key_padding_mask,
+                batch_size=bsz,
+                src_len=k.size(1),
+                static_kv=static_kv,
+            )
+
+            saved_state["prev_key"] = k.view(bsz, self.num_heads, -1, self.head_dim)
+            saved_state["prev_value"] = v.view(bsz, self.num_heads, -1, self.head_dim)
+            saved_state["prev_key_padding_mask"] = key_padding_mask
+            # In this branch incremental_state is never None
+            assert incremental_state is not None
+            incremental_state = self._set_input_buffer(incremental_state, saved_state)
+        assert k is not None
+        src_len = k.size(1)
+
+        # This is part of a workaround to get around fork/join parallelism
+        # not supporting Optional types.
+        if key_padding_mask is not None and key_padding_mask.dim() == 0:
+            key_padding_mask = None
+
+        if key_padding_mask is not None:
+            assert key_padding_mask.size(0) == bsz
+            assert key_padding_mask.size(1) == src_len
+
+        if self.add_zero_attn:
+            assert v is not None
+            src_len += 1
+            k = torch.cat([k, k.new_zeros((k.size(0), 1) + k.size()[2:])], dim=1)
+            v = torch.cat([v, v.new_zeros((v.size(0), 1) + v.size()[2:])], dim=1)
+            if attn_mask is not None:
+                attn_mask = torch.cat(
+                    [attn_mask, attn_mask.new_zeros(attn_mask.size(0), 1)], dim=1
+                )
+            if key_padding_mask is not None:
+                key_padding_mask = torch.cat(
+                    [
+                        key_padding_mask,
+                        torch.zeros(key_padding_mask.size(0), 1).type_as(key_padding_mask),
+                    ],
+                    dim=1,
+                )
+
+        if self.rot_emb:
+            q, k = self.rot_emb(q, k)
+
+        attn_weights = torch.bmm(q, k.transpose(1, 2))
+        attn_weights = MultiheadAttention.apply_sparse_mask(attn_weights, tgt_len, src_len, bsz)
+
+        assert list(attn_weights.size()) == [bsz * self.num_heads, tgt_len, src_len]
+
+        if attn_mask is not None:
+            attn_mask = attn_mask.unsqueeze(0)
+            if self.onnx_trace:
+                attn_mask = attn_mask.repeat(attn_weights.size(0), 1, 1)
+            attn_weights += attn_mask
+
+        if key_padding_mask is not None:
+            # don't attend to padding symbols
+            attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
+            attn_weights = attn_weights.masked_fill(
+                key_padding_mask.unsqueeze(1).unsqueeze(2).to(torch.bool), float("-inf")
+            )
+            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)
+
+        if before_softmax:
+            return attn_weights, v
+
+        attn_weights_float = utils_softmax(attn_weights, dim=-1, onnx_trace=self.onnx_trace)
+        attn_weights = attn_weights_float.type_as(attn_weights)
+        attn_probs = F.dropout(
+            attn_weights_float.type_as(attn_weights),
+            p=self.dropout,
+            training=self.training,
+        )
+        assert v is not None
+        attn = torch.bmm(attn_probs, v)
+        assert list(attn.size()) == [bsz * self.num_heads, tgt_len, self.head_dim]
+        if self.onnx_trace and attn.size(1) == 1:
+            # when ONNX tracing a single decoder step (sequence length == 1)
+            # the transpose is a no-op copy before view, thus unnecessary
+            attn = attn.contiguous().view(tgt_len, bsz, embed_dim)
+        else:
+            attn = attn.transpose(0, 1).contiguous().view(tgt_len, bsz, embed_dim)
+        attn = self.out_proj(attn)
+        attn_weights: Optional[Tensor] = None
+        if need_weights:
+            attn_weights = attn_weights_float.view(
+                bsz, self.num_heads, tgt_len, src_len
+            ).type_as(attn).transpose(1, 0)
+            if not need_head_weights:
+                # average attention weights over heads
+                attn_weights = attn_weights.mean(dim=0)
+
+        return attn, attn_weights
+
+    @staticmethod
+    def _append_prev_key_padding_mask(
+        key_padding_mask: Optional[Tensor],
+        prev_key_padding_mask: Optional[Tensor],
+        batch_size: int,
+        src_len: int,
+        static_kv: bool,
+    ) -> Optional[Tensor]:
+        # saved key padding masks have shape (bsz, seq_len)
+        if prev_key_padding_mask is not None and static_kv:
+            new_key_padding_mask = prev_key_padding_mask
+        elif prev_key_padding_mask is not None and key_padding_mask is not None:
+            new_key_padding_mask = torch.cat(
+                [prev_key_padding_mask.float(), key_padding_mask.float()], dim=1
+            )
+        # During incremental decoding, as the padding token enters and
+        # leaves the frame, there will be a time when prev or current
+        # is None
+        elif prev_key_padding_mask is not None:
+            filler = torch.zeros(
+                (batch_size, src_len - prev_key_padding_mask.size(1)),
+                device=prev_key_padding_mask.device,
+            )
+            new_key_padding_mask = torch.cat(
+                [prev_key_padding_mask.float(), filler.float()], dim=1
+            )
+        elif key_padding_mask is not None:
+            filler = torch.zeros(
+                (batch_size, src_len - key_padding_mask.size(1)),
+                device=key_padding_mask.device,
+            )
+            new_key_padding_mask = torch.cat([filler.float(), key_padding_mask.float()], dim=1)
+        else:
+            new_key_padding_mask = prev_key_padding_mask
+        return new_key_padding_mask
+
+    @torch.jit.export
+    def reorder_incremental_state(
+        self, incremental_state: Dict[str, Dict[str, Optional[Tensor]]], new_order: Tensor
+    ):
+        """Reorder buffered internal state (for incremental generation)."""
+        input_buffer = self._get_input_buffer(incremental_state)
+        if input_buffer is not None:
+            for k in input_buffer.keys():
+                input_buffer_k = input_buffer[k]
+                if input_buffer_k is not None:
+                    if self.encoder_decoder_attention and input_buffer_k.size(0) == new_order.size(
+                        0
+                    ):
+                        break
+                    input_buffer[k] = input_buffer_k.index_select(0, new_order)
+            incremental_state = self._set_input_buffer(incremental_state, input_buffer)
+        return incremental_state
+
+    def _get_input_buffer(
+        self, incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]]
+    ) -> Dict[str, Optional[Tensor]]:
+        result = self.get_incremental_state(incremental_state, "attn_state")
+        if result is not None:
+            return result
+        else:
+            empty_result: Dict[str, Optional[Tensor]] = {}
+            return empty_result
+
+    def _set_input_buffer(
+        self,
+        incremental_state: Dict[str, Dict[str, Optional[Tensor]]],
+        buffer: Dict[str, Optional[Tensor]],
+    ):
+        return self.set_incremental_state(incremental_state, "attn_state", buffer)
+
+    def apply_sparse_mask(attn_weights, tgt_len: int, src_len: int, bsz: int):
+        return attn_weights
+
+    def upgrade_state_dict_named(self, state_dict, name):
+        prefix = name + "." if name != "" else ""
+        items_to_add = {}
+        keys_to_remove = []
+        for k in state_dict.keys():
+            if k.endswith(prefix + "in_proj_weight"):
+                # in_proj_weight used to be q + k + v with same dimensions
+                dim = int(state_dict[k].shape[0] / 3)
+                items_to_add[prefix + "q_proj.weight"] = state_dict[k][:dim]
+                items_to_add[prefix + "k_proj.weight"] = state_dict[k][dim : 2 * dim]
+                items_to_add[prefix + "v_proj.weight"] = state_dict[k][2 * dim :]
+
+                keys_to_remove.append(k)
+
+                k_bias = prefix + "in_proj_bias"
+                if k_bias in state_dict.keys():
+                    dim = int(state_dict[k].shape[0] / 3)
+                    items_to_add[prefix + "q_proj.bias"] = state_dict[k_bias][:dim]
+                    items_to_add[prefix + "k_proj.bias"] = state_dict[k_bias][dim : 2 * dim]
+                    items_to_add[prefix + "v_proj.bias"] = state_dict[k_bias][2 * dim :]
+
+                    keys_to_remove.append(prefix + "in_proj_bias")
+
+        for k in keys_to_remove:
+            del state_dict[k]
+
+        for key, value in items_to_add.items():
+            state_dict[key] = value

esm/esm/pretrained.py

@@ -0,0 +1,397 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import re
+import urllib
+import warnings
+from argparse import Namespace
+from pathlib import Path
+
+import torch
+
+import esm
+from esm.model.esm2 import ESM2
+
+
+def _has_regression_weights(model_name):
+    """Return whether we expect / require regression weights;
+    Right now that is all models except ESM-1v and ESM-IF"""
+    return not ("esm1v" in model_name or "esm_if" in model_name)
+
+
+def load_model_and_alphabet(model_name):
+    if model_name.endswith(".pt"):  # treat as filepath
+        return load_model_and_alphabet_local(model_name)
+    else:
+        return load_model_and_alphabet_hub(model_name)
+
+
+def load_hub_workaround(url):
+    try:
+        data = torch.hub.load_state_dict_from_url(url, progress=False, map_location="cpu")
+    except RuntimeError:
+        # Pytorch version issue - see https://github.com/pytorch/pytorch/issues/43106
+        fn = Path(url).name
+        data = torch.load(
+            f"{torch.hub.get_dir()}/checkpoints/{fn}",
+            map_location="cpu",
+        )
+    except urllib.error.HTTPError as e:
+        raise Exception(f"Could not load {url}, check if you specified a correct model name?")
+    return data
+
+
+def load_regression_hub(model_name):
+    url = f"https://dl.fbaipublicfiles.com/fair-esm/regression/{model_name}-contact-regression.pt"
+    regression_data = load_hub_workaround(url)
+    return regression_data
+
+
+def _download_model_and_regression_data(model_name):
+    url = f"https://dl.fbaipublicfiles.com/fair-esm/models/{model_name}.pt"
+    model_data = load_hub_workaround(url)
+    if _has_regression_weights(model_name):
+        regression_data = load_regression_hub(model_name)
+    else:
+        regression_data = None
+    return model_data, regression_data
+
+
+def load_model_and_alphabet_hub(model_name):
+    model_data, regression_data = _download_model_and_regression_data(model_name)
+    return load_model_and_alphabet_core(model_name, model_data, regression_data)
+
+
+def load_model_and_alphabet_local(model_location):
+    """Load from local path. The regression weights need to be co-located"""
+    model_location = Path(model_location)
+    model_data = torch.load(str(model_location), map_location="cpu")
+    model_name = model_location.stem
+    if _has_regression_weights(model_name):
+        regression_location = str(model_location.with_suffix("")) + "-contact-regression.pt"
+        regression_data = torch.load(regression_location, map_location="cpu")
+    else:
+        regression_data = None
+    return load_model_and_alphabet_core(model_name, model_data, regression_data)
+
+
+def has_emb_layer_norm_before(model_state):
+    """Determine whether layer norm needs to be applied before the encoder"""
+    return any(k.startswith("emb_layer_norm_before") for k, param in model_state.items())
+
+
+def _load_model_and_alphabet_core_v1(model_data):
+    import esm  # since esm.inverse_folding is imported below, you actually have to re-import esm here
+
+    alphabet = esm.Alphabet.from_architecture(model_data["args"].arch)
+
+    if model_data["args"].arch == "roberta_large":
+        # upgrade state dict
+        pra = lambda s: "".join(s.split("encoder_")[1:] if "encoder" in s else s)
+        prs1 = lambda s: "".join(s.split("encoder.")[1:] if "encoder" in s else s)
+        prs2 = lambda s: "".join(
+            s.split("sentence_encoder.")[1:] if "sentence_encoder" in s else s
+        )
+        model_args = {pra(arg[0]): arg[1] for arg in vars(model_data["args"]).items()}
+        model_state = {prs1(prs2(arg[0])): arg[1] for arg in model_data["model"].items()}
+        model_state["embed_tokens.weight"][alphabet.mask_idx].zero_()  # For token drop
+        model_args["emb_layer_norm_before"] = has_emb_layer_norm_before(model_state)
+        model_type = esm.ProteinBertModel
+
+    elif model_data["args"].arch == "protein_bert_base":
+
+        # upgrade state dict
+        pra = lambda s: "".join(s.split("decoder_")[1:] if "decoder" in s else s)
+        prs = lambda s: "".join(s.split("decoder.")[1:] if "decoder" in s else s)
+        model_args = {pra(arg[0]): arg[1] for arg in vars(model_data["args"]).items()}
+        model_state = {prs(arg[0]): arg[1] for arg in model_data["model"].items()}
+        model_type = esm.ProteinBertModel
+    elif model_data["args"].arch == "msa_transformer":
+
+        # upgrade state dict
+        pra = lambda s: "".join(s.split("encoder_")[1:] if "encoder" in s else s)
+        prs1 = lambda s: "".join(s.split("encoder.")[1:] if "encoder" in s else s)
+        prs2 = lambda s: "".join(
+            s.split("sentence_encoder.")[1:] if "sentence_encoder" in s else s
+        )
+        prs3 = lambda s: s.replace("row", "column") if "row" in s else s.replace("column", "row")
+        model_args = {pra(arg[0]): arg[1] for arg in vars(model_data["args"]).items()}
+        model_state = {prs1(prs2(prs3(arg[0]))): arg[1] for arg in model_data["model"].items()}
+        if model_args.get("embed_positions_msa", False):
+            emb_dim = model_state["msa_position_embedding"].size(-1)
+            model_args["embed_positions_msa_dim"] = emb_dim  # initial release, bug: emb_dim==1
+
+        model_type = esm.MSATransformer
+
+    elif "invariant_gvp" in model_data["args"].arch:
+        import esm.inverse_folding
+
+        model_type = esm.inverse_folding.gvp_transformer.GVPTransformerModel
+        model_args = vars(model_data["args"])  # convert Namespace -> dict
+
+        def update_name(s):
+            # Map the module names in checkpoints trained with internal code to
+            # the updated module names in open source code
+            s = s.replace("W_v", "embed_graph.embed_node")
+            s = s.replace("W_e", "embed_graph.embed_edge")
+            s = s.replace("embed_scores.0", "embed_confidence")
+            s = s.replace("embed_score.", "embed_graph.embed_confidence.")
+            s = s.replace("seq_logits_projection.", "")
+            s = s.replace("embed_ingraham_features", "embed_dihedrals")
+            s = s.replace("embed_gvp_in_local_frame.0", "embed_gvp_output")
+            s = s.replace("embed_features_in_local_frame.0", "embed_gvp_input_features")
+            return s
+
+        model_state = {
+            update_name(sname): svalue
+            for sname, svalue in model_data["model"].items()
+            if "version" not in sname
+        }
+
+    else:
+        raise ValueError("Unknown architecture selected")
+
+    model = model_type(
+        Namespace(**model_args),
+        alphabet,
+    )
+
+    return model, alphabet, model_state
+
+
+def _load_model_and_alphabet_core_v2(model_data):
+    def upgrade_state_dict(state_dict):
+        """Removes prefixes 'model.encoder.sentence_encoder.' and 'model.encoder.'."""
+        prefixes = ["encoder.sentence_encoder.", "encoder."]
+        pattern = re.compile("^" + "|".join(prefixes))
+        state_dict = {pattern.sub("", name): param for name, param in state_dict.items()}
+        return state_dict
+
+    cfg = model_data["cfg"]["model"]
+    state_dict = model_data["model"]
+    state_dict = upgrade_state_dict(state_dict)
+    alphabet = esm.data.Alphabet.from_architecture("ESM-1b")
+    model = ESM2(
+        num_layers=cfg.encoder_layers,
+        embed_dim=cfg.encoder_embed_dim,
+        attention_heads=cfg.encoder_attention_heads,
+        alphabet=alphabet,
+        token_dropout=cfg.token_dropout,
+    )
+    return model, alphabet, state_dict
+
+
+def load_model_and_alphabet_core(model_name, model_data, regression_data=None):
+    if regression_data is not None:
+        model_data["model"].update(regression_data["model"])
+
+    if model_name.startswith("esm2"):
+        model, alphabet, model_state = _load_model_and_alphabet_core_v2(model_data)
+    else:
+        model, alphabet, model_state = _load_model_and_alphabet_core_v1(model_data)
+
+    expected_keys = set(model.state_dict().keys())
+    found_keys = set(model_state.keys())
+
+    if regression_data is None:
+        expected_missing = {"contact_head.regression.weight", "contact_head.regression.bias"}
+        error_msgs = []
+        missing = (expected_keys - found_keys) - expected_missing
+        if missing:
+            error_msgs.append(f"Missing key(s) in state_dict: {missing}.")
+        unexpected = found_keys - expected_keys
+        if unexpected:
+            error_msgs.append(f"Unexpected key(s) in state_dict: {unexpected}.")
+
+        if error_msgs:
+            raise RuntimeError(
+                "Error(s) in loading state_dict for {}:\n\t{}".format(
+                    model.__class__.__name__, "\n\t".join(error_msgs)
+                )
+            )
+        if expected_missing - found_keys:
+            warnings.warn(
+                "Regression weights not found, predicting contacts will not produce correct results."
+            )
+
+    model.load_state_dict(model_state, strict=regression_data is not None)
+
+    return model, alphabet
+
+
+def esm1_t34_670M_UR50S():
+    """34 layer transformer model with 670M params, trained on Uniref50 Sparse.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm1_t34_670M_UR50S")
+
+
+def esm1_t34_670M_UR50D():
+    """34 layer transformer model with 670M params, trained on Uniref50 Dense.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm1_t34_670M_UR50D")
+
+
+def esm1_t34_670M_UR100():
+    """34 layer transformer model with 670M params, trained on Uniref100.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm1_t34_670M_UR100")
+
+
+def esm1_t12_85M_UR50S():
+    """12 layer transformer model with 85M params, trained on Uniref50 Sparse.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm1_t12_85M_UR50S")
+
+
+def esm1_t6_43M_UR50S():
+    """6 layer transformer model with 43M params, trained on Uniref50 Sparse.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm1_t6_43M_UR50S")
+
+
+def esm1b_t33_650M_UR50S():
+    """33 layer transformer model with 650M params, trained on Uniref50 Sparse.
+    This is our best performing model, which will be described in a future publication.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm1b_t33_650M_UR50S")
+
+
+def esm_msa1_t12_100M_UR50S():
+    warnings.warn(
+        "This model had a minor bug in the positional embeddings, "
+        "please use ESM-MSA-1b: esm.pretrained.esm_msa1b_t12_100M_UR50S()",
+    )
+    return load_model_and_alphabet_hub("esm_msa1_t12_100M_UR50S")
+
+
+def esm_msa1b_t12_100M_UR50S():
+    return load_model_and_alphabet_hub("esm_msa1b_t12_100M_UR50S")
+
+
+def esm1v_t33_650M_UR90S():
+    """33 layer transformer model with 650M params, trained on Uniref90.
+    This is model 1 of a 5 model ensemble.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm1v_t33_650M_UR90S_1")
+
+
+def esm1v_t33_650M_UR90S_1():
+    """33 layer transformer model with 650M params, trained on Uniref90.
+    This is model 1 of a 5 model ensemble.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm1v_t33_650M_UR90S_1")
+
+
+def esm1v_t33_650M_UR90S_2():
+    """33 layer transformer model with 650M params, trained on Uniref90.
+    This is model 2 of a 5 model ensemble.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm1v_t33_650M_UR90S_2")
+
+
+def esm1v_t33_650M_UR90S_3():
+    """33 layer transformer model with 650M params, trained on Uniref90.
+    This is model 3 of a 5 model ensemble.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm1v_t33_650M_UR90S_3")
+
+
+def esm1v_t33_650M_UR90S_4():
+    """33 layer transformer model with 650M params, trained on Uniref90.
+    This is model 4 of a 5 model ensemble.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm1v_t33_650M_UR90S_4")
+
+
+def esm1v_t33_650M_UR90S_5():
+    """33 layer transformer model with 650M params, trained on Uniref90.
+    This is model 5 of a 5 model ensemble.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm1v_t33_650M_UR90S_5")
+
+
+def esm_if1_gvp4_t16_142M_UR50():
+    """Inverse folding model with 142M params, with 4 GVP-GNN layers, 8
+    Transformer encoder layers, and 8 Transformer decoder layers, trained on
+    CATH structures and 12 million alphafold2 predicted structures from UniRef50
+    sequences.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm_if1_gvp4_t16_142M_UR50")
+
+
+def esm2_t6_8M_UR50D():
+    """6 layer ESM-2 model with 8M params, trained on UniRef50.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm2_t6_8M_UR50D")
+
+
+def esm2_t12_35M_UR50D():
+    """12 layer ESM-2 model with 35M params, trained on UniRef50.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm2_t12_35M_UR50D")
+
+
+def esm2_t30_150M_UR50D():
+    """30 layer ESM-2 model with 150M params, trained on UniRef50.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm2_t30_150M_UR50D")
+
+
+def esm2_t33_650M_UR50D():
+    """33 layer ESM-2 model with 650M params, trained on UniRef50.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm2_t33_650M_UR50D")
+
+
+def esm2_t36_3B_UR50D():
+    """36 layer ESM-2 model with 3B params, trained on UniRef50.
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm2_t36_3B_UR50D")
+
+
+def esm2_t48_15B_UR50D():
+    """48 layer ESM-2 model with 15B params, trained on UniRef50.
+    If you have OOM while loading this model, please refer to README
+    on how to employ FSDP and ZeRO CPU offloading
+
+    Returns a tuple of (Model, Alphabet).
+    """
+    return load_model_and_alphabet_hub("esm2_t48_15B_UR50D")

esm/esm/rotary_embedding.py

@@ -0,0 +1,69 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from typing import Tuple
+
+import torch
+
+
+def rotate_half(x):
+    x1, x2 = x.chunk(2, dim=-1)
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(x, cos, sin):
+    cos = cos[:, : x.shape[-2], :]
+    sin = sin[:, : x.shape[-2], :]
+
+    return (x * cos) + (rotate_half(x) * sin)
+
+
+class RotaryEmbedding(torch.nn.Module):
+    """
+    The rotary position embeddings from RoFormer_ (Su et. al).
+    A crucial insight from the method is that the query and keys are
+    transformed by rotation matrices which depend on the relative positions.
+    Other implementations are available in the Rotary Transformer repo_ and in
+    GPT-NeoX_, GPT-NeoX was an inspiration
+    .. _RoFormer: https://arxiv.org/abs/2104.09864
+    .. _repo: https://github.com/ZhuiyiTechnology/roformer
+    .. _GPT-NeoX: https://github.com/EleutherAI/gpt-neox
+    .. warning: Please note that this embedding is not registered on purpose, as it is transformative
+        (it does not create the embedding dimension) and will likely be picked up (imported) on a ad-hoc basis
+    """
+
+    def __init__(self, dim: int, *_, **__):
+        super().__init__()
+        # Generate and save the inverse frequency buffer (non trainable)
+        inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2).float() / dim))
+        self.register_buffer("inv_freq", inv_freq)
+
+        self._seq_len_cached = None
+        self._cos_cached = None
+        self._sin_cached = None
+
+    def _update_cos_sin_tables(self, x, seq_dimension=1):
+        seq_len = x.shape[seq_dimension]
+
+        # Reset the tables if the sequence length has changed,
+        # or if we're on a new device (possibly due to tracing for instance)
+        if seq_len != self._seq_len_cached or self._cos_cached.device != x.device:
+            self._seq_len_cached = seq_len
+            t = torch.arange(x.shape[seq_dimension], device=x.device).type_as(self.inv_freq)
+            freqs = torch.einsum("i,j->ij", t, self.inv_freq)
+            emb = torch.cat((freqs, freqs), dim=-1).to(x.device)
+
+            self._cos_cached = emb.cos()[None, :, :]
+            self._sin_cached = emb.sin()[None, :, :]
+
+        return self._cos_cached, self._sin_cached
+
+    def forward(self, q: torch.Tensor, k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+        self._cos_cached, self._sin_cached = self._update_cos_sin_tables(k, seq_dimension=-2)
+
+        return (
+            apply_rotary_pos_emb(q, self._cos_cached, self._sin_cached),
+            apply_rotary_pos_emb(k, self._cos_cached, self._sin_cached),
+        )

esm/esm/version.py

@@ -0,0 +1,6 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+version = "1.0.2"

esm/scripts/extract.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3 -u
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+import pathlib
+import sys 
+print("using", sys.executable)
+
+sys.path.insert( 0,"/home/user/.local/lib/python3.8/site-packages")
+sys.path.insert( 0,"/home/user/app/esm/")
+import os 
+
+import torch
+
+from esm import Alphabet, FastaBatchedDataset, ProteinBertModel, pretrained, MSATransformer
+
+
+def create_parser():
+    parser = argparse.ArgumentParser(
+        description="Extract per-token representations and model outputs for sequences in a FASTA file"  # noqa
+    )
+
+    parser.add_argument(
+        "model_location",
+        type=str,
+        help="PyTorch model file OR name of pretrained model to download (see README for models)",
+    )
+    parser.add_argument(
+        "fasta_file",
+        type=pathlib.Path,
+        help="FASTA file on which to extract representations",
+    )
+    parser.add_argument(
+        "output_dir",
+        type=pathlib.Path,
+        help="output directory for extracted representations",
+    )
+
+    parser.add_argument("--toks_per_batch", type=int, default=4096, help="maximum batch size")
+    parser.add_argument(
+        "--repr_layers",
+        type=int,
+        default=[-1],
+        nargs="+",
+        help="layers indices from which to extract representations (0 to num_layers, inclusive)",
+    )
+    parser.add_argument(
+        "--include",
+        type=str,
+        nargs="+",
+        choices=["mean", "per_tok", "bos", "contacts"],
+        help="specify which representations to return",
+        required=True,
+    )
+    parser.add_argument(
+        "--truncation_seq_length",
+        type=int,
+        default=1022,
+        help="truncate sequences longer than the given value",
+    )
+
+    parser.add_argument("--nogpu", action="store_true", help="Do not use GPU even if available")
+    return parser
+
+
+def main(args):
+    model, alphabet = pretrained.load_model_and_alphabet(args.model_location)
+    model.eval()
+    if isinstance(model, MSATransformer):
+        raise ValueError(
+            "This script currently does not handle models with MSA input (MSA Transformer)."
+        )
+    if torch.cuda.is_available() and not args.nogpu:
+        model = model.cuda()
+        print("Transferred model to GPU")
+
+    dataset = FastaBatchedDataset.from_file(args.fasta_file)
+    batches = dataset.get_batch_indices(args.toks_per_batch, extra_toks_per_seq=1)
+    data_loader = torch.utils.data.DataLoader(
+        dataset, collate_fn=alphabet.get_batch_converter(args.truncation_seq_length), batch_sampler=batches
+    )
+    print(f"Read {args.fasta_file} with {len(dataset)} sequences")
+
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    return_contacts = "contacts" in args.include
+
+    assert all(-(model.num_layers + 1) <= i <= model.num_layers for i in args.repr_layers)
+    repr_layers = [(i + model.num_layers + 1) % (model.num_layers + 1) for i in args.repr_layers]
+
+    with torch.no_grad():
+        for batch_idx, (labels, strs, toks) in enumerate(data_loader):
+            print(
+                f"Processing {batch_idx + 1} of {len(batches)} batches ({toks.size(0)} sequences)"
+            )
+            if torch.cuda.is_available() and not args.nogpu:
+                toks = toks.to(device="cuda", non_blocking=True)
+
+            out = model(toks, repr_layers=repr_layers, return_contacts=return_contacts)
+
+            logits = out["logits"].to(device="cpu")
+            representations = {
+                layer: t.to(device="cpu") for layer, t in out["representations"].items()
+            }
+            if return_contacts:
+                contacts = out["contacts"].to(device="cpu")
+
+            for i, label in enumerate(labels):
+                args.output_file = args.output_dir / f"{label}.pt"
+                args.output_file.parent.mkdir(parents=True, exist_ok=True)
+                result = {"label": label}
+                # Call clone on tensors to ensure tensors are not views into a larger representation
+                # See https://github.com/pytorch/pytorch/issues/1995
+                if "per_tok" in args.include:
+                    result["representations"] = {
+                        layer: t[i, 1 : len(strs[i]) + 1].clone()
+                        for layer, t in representations.items()
+                    }
+                if "mean" in args.include:
+                    result["mean_representations"] = {
+                        layer: t[i, 1 : len(strs[i]) + 1].mean(0).clone()
+                        for layer, t in representations.items()
+                    }
+                if "bos" in args.include:
+                    result["bos_representations"] = {
+                        layer: t[i, 0].clone() for layer, t in representations.items()
+                    }
+                if return_contacts:
+                    result["contacts"] = contacts[i, : len(strs[i]), : len(strs[i])].clone()
+
+                torch.save(
+                    result,
+                    args.output_file,
+                )
+
+
+if __name__ == "__main__":
+    parser = create_parser()
+    args = parser.parse_args()
+    main(args)