Upload 42 files

.gitattributes

@@ -32,3 +32,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/data_analysis.ipynb filter=lfs diff=lfs merge=lfs -text
+model-image.png filter=lfs diff=lfs merge=lfs -text
+task_yield/visualize_embedding.ipynb filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,19 @@
+*.ipynb_checkpoints
+__pycache__
+*.csv
+*.tsv
+*.smi
+*.bin
+*.pth
+*.pt
+*.tar
+*.tar.gz
+*.zip
+*.gz
+*.tgz
+*.rar
+*.safetensors
+*.npy
+*.pkl
+
+!data/demo_reaction_data.csv

CompoundT5/CompoundT5/CompoundT5-config/config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "/home/patrick/hugging_face/t5/t5-v1_1-base",
+  "architectures": [
+    "T5ForConditionalGeneration"
+  ],
+  "d_ff": 2048,
+  "d_kv": 64,
+  "d_model": 768,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "gelu_new",
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "t5",
+  "num_decoder_layers": 12,
+  "num_heads": 12,
+  "num_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.21.0.dev0",
+  "use_cache": true,
+  "vocab_size": 41
+}

CompoundT5/CompoundT5/CompoundT5-config/tokenizer.json

@@ -0,0 +1,287 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "</s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "<unk>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "Nmt"
+      },
+      {
+        "type": "NFKC"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "Regex": " {2,}"
+        },
+        "content": " "
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "Sequence",
+    "pretokenizers": [
+      {
+        "type": "Metaspace",
+        "replacement": "▁",
+        "add_prefix_space": true
+      },
+      {
+        "type": "Digits",
+        "individual_digits": true
+      },
+      {
+        "type": "Punctuation",
+        "behavior": "Isolated"
+      }
+    ]
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "</s>",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "</s>": {
+        "id": "</s>",
+        "ids": [
+          1
+        ],
+        "tokens": [
+          "</s>"
+        ]
+      }
+    }
+  },
+  "decoder": {
+    "type": "Metaspace",
+    "replacement": "▁",
+    "add_prefix_space": true
+  },
+  "model": {
+    "type": "Unigram",
+    "unk_id": 2,
+    "vocab": [
+      [
+        "<pad>",
+        0.0
+      ],
+      [
+        "</s>",
+        0.0
+      ],
+      [
+        "<unk>",
+        0.0
+      ],
+      [
+        "▁",
+        -0.6931471808026011
+      ],
+      [
+        "c",
+        -2.289498028516334
+      ],
+      [
+        "C",
+        -2.3191188737900035
+      ],
+      [
+        "(",
+        -3.157145613029357
+      ],
+      [
+        ")",
+        -3.157145613029357
+      ],
+      [
+        "1",
+        -3.4337494413900735
+      ],
+      [
+        "O",
+        -3.8003416456793744
+      ],
+      [
+        "2",
+        -3.8354203318153104
+      ],
+      [
+        "N",
+        -3.9489619191823486
+      ],
+      [
+        "]",
+        -4.114143160310146
+      ],
+      [
+        "[",
+        -4.114143160310146
+      ],
+      [
+        "@",
+        -4.185726512332149
+      ],
+      [
+        "H",
+        -4.201161413116868
+      ],
+      [
+        "=",
+        -4.26644820084319
+      ],
+      [
+        "n",
+        -4.300186073016661
+      ],
+      [
+        "3",
+        -4.824395958274135
+      ],
+      [
+        "+",
+        -5.412930408280779
+      ],
+      [
+        "F",
+        -5.636658395691338
+      ],
+      [
+        "-",
+        -5.944123069167032
+      ],
+      [
+        "S",
+        -6.23059354933377
+      ],
+      [
+        "s",
+        -6.3086720535935505
+      ],
+      [
+        "l",
+        -6.356164827135707
+      ],
+      [
+        "4",
+        -6.474778787500576
+      ],
+      [
+        "o",
+        -6.5919851676767856
+      ],
+      [
+        "#",
+        -7.471440033681638
+      ],
+      [
+        "r",
+        -7.600338586268233
+      ],
+      [
+        "B",
+        -7.600338586268233
+      ],
+      [
+        "/",
+        -8.02057032804323
+      ],
+      [
+        "5",
+        -8.905241806184042
+      ],
+      [
+        "\\",
+        -9.431656471484382
+      ],
+      [
+        "I",
+        -10.348187932078408
+      ],
+      [
+        "6",
+        -12.084066778027127
+      ],
+      [
+        "7",
+        -15.584016494881563
+      ],
+      [
+        "p",
+        -17.628494092721255
+      ],
+      [
+        "8",
+        -18.37808350350985
+      ],
+      [
+        "P",
+        -19.003564863395415
+      ],
+      [
+        ".",
+        -20.190108874992006
+      ],
+      [
+        "9",
+        -21.023442208325346
+      ]
+    ]
+  }
+}

CompoundT5/CompoundT5/new_run_t5_mlm_flax.py

@@ -0,0 +1,1143 @@
+#!/usr/bin/env python
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Pretraining the library models for T5-like span-masked language modeling on a text file or a dataset.
+
+Here is the full list of checkpoints on the hub that can be pretrained by this script:
+https://huggingface.co/models?filter=t5
+"""
+
+import json
+import logging
+import os
+import sys
+import time
+from dataclasses import asdict, dataclass, field
+
+# You can also adapt this script on your own masked language modeling task. Pointers for this are left as comments.
+from enum import Enum
+
+# from transformers.utils import get_full_repo_name, send_example_telemetry
+from functools import partialmethod
+from itertools import chain
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import flax
+import jax
+import jax.numpy as jnp
+import numpy as np
+import optax
+from datasets import load_dataset
+from flax import jax_utils, traverse_util
+from flax.training import train_state
+from flax.training.common_utils import get_metrics, onehot, shard
+from tqdm import tqdm
+from transformers import (
+    CONFIG_MAPPING,
+    FLAX_MODEL_FOR_MASKED_LM_MAPPING,
+    AutoTokenizer,
+    BatchEncoding,
+    FlaxT5ForConditionalGeneration,
+    HfArgumentParser,
+    PreTrainedTokenizerBase,
+    T5Config,
+    is_tensorboard_available,
+    set_seed,
+)
+from transformers.models.t5.modeling_flax_t5 import shift_tokens_right
+
+tqdm.__init__ = partialmethod(tqdm.__init__, disable=True)
+
+
+MODEL_CONFIG_CLASSES = list(FLAX_MODEL_FOR_MASKED_LM_MAPPING.keys())
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
+
+
+@dataclass
+class TrainingArguments:
+    output_dir: str = field(
+        metadata={
+            "help": "The output directory where the model predictions and checkpoints will be written."
+        },
+    )
+    overwrite_output_dir: bool = field(
+        default=False,
+        metadata={
+            "help": (
+                "Overwrite the content of the output directory. "
+                "Use this to continue training if output_dir points to a checkpoint directory."
+            )
+        },
+    )
+    do_train: bool = field(default=False, metadata={"help": "Whether to run training."})
+    do_eval: bool = field(
+        default=False, metadata={"help": "Whether to run eval on the dev set."}
+    )
+    per_device_train_batch_size: int = field(
+        default=8, metadata={"help": "Batch size per GPU/TPU core/CPU for training."}
+    )
+    per_device_eval_batch_size: int = field(
+        default=8, metadata={"help": "Batch size per GPU/TPU core/CPU for evaluation."}
+    )
+    learning_rate: float = field(
+        default=5e-5, metadata={"help": "The initial learning rate for AdamW."}
+    )
+    weight_decay: float = field(
+        default=0.0, metadata={"help": "Weight decay for AdamW if we apply some."}
+    )
+    adam_beta1: float = field(
+        default=0.9, metadata={"help": "Beta1 for AdamW optimizer"}
+    )
+    adam_beta2: float = field(
+        default=0.999, metadata={"help": "Beta2 for AdamW optimizer"}
+    )
+    adam_epsilon: float = field(
+        default=1e-8, metadata={"help": "Epsilon for AdamW optimizer."}
+    )
+    adafactor: bool = field(
+        default=False,
+        metadata={"help": "Whether or not to replace AdamW by Adafactor."},
+    )
+    num_train_epochs: float = field(
+        default=3.0, metadata={"help": "Total number of training epochs to perform."}
+    )
+    warmup_steps: int = field(
+        default=0, metadata={"help": "Linear warmup over warmup_steps."}
+    )
+    logging_steps: int = field(
+        default=500, metadata={"help": "Log every X updates steps."}
+    )
+    save_steps: int = field(
+        default=500, metadata={"help": "Save checkpoint every X updates steps."}
+    )
+    eval_steps: int = field(
+        default=None, metadata={"help": "Run an evaluation every X steps."}
+    )
+    seed: int = field(
+        default=42,
+        metadata={"help": "Random seed that will be set at the beginning of training."},
+    )
+    push_to_hub: bool = field(
+        default=False,
+        metadata={
+            "help": "Whether or not to upload the trained model to the model hub after training."
+        },
+    )
+    hub_model_id: str = field(
+        default=None,
+        metadata={
+            "help": "The name of the repository to keep in sync with the local `output_dir`."
+        },
+    )
+    hub_token: str = field(
+        default=None, metadata={"help": "The token to use to push to the Model Hub."}
+    )
+
+    def __post_init__(self):
+        if self.output_dir is not None:
+            self.output_dir = os.path.expanduser(self.output_dir)
+
+    def to_dict(self):
+        """
+        Serializes this instance while replace `Enum` by their values (for JSON serialization support). It obfuscates
+        the token values by removing their value.
+        """
+        d = asdict(self)
+        for k, v in d.items():
+            if isinstance(v, Enum):
+                d[k] = v.value
+            if isinstance(v, list) and len(v) > 0 and isinstance(v[0], Enum):
+                d[k] = [x.value for x in v]
+            if k.endswith("_token"):
+                d[k] = f"<{k.upper()}>"
+        return d
+
+
+@dataclass
+class ModelArguments:
+    """
+    Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.
+    """
+
+    model_name_or_path: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The model checkpoint for weights initialization.Don't set if you want to train a model from scratch."
+            )
+        },
+    )
+    model_type: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "If training from scratch, pass a model type from the list: "
+            + ", ".join(MODEL_TYPES)
+        },
+    )
+    config_name: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "Pretrained config name or path if not the same as model_name"
+        },
+    )
+    tokenizer_name: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "Pretrained tokenizer name or path if not the same as model_name"
+        },
+    )
+    cache_dir: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "Where do you want to store the pretrained models downloaded from s3"
+        },
+    )
+    use_fast_tokenizer: bool = field(
+        default=True,
+        metadata={
+            "help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."
+        },
+    )
+    dtype: Optional[str] = field(
+        default="float32",
+        metadata={
+            "help": (
+                "Floating-point format in which the model weights should be initialized and trained. Choose one of"
+                " `[float32, float16, bfloat16]`."
+            )
+        },
+    )
+
+
+#     use_auth_token: bool = field(
+#         default=False,
+#         metadata={
+#             "help": (
+#                 "Will use the token generated when running `transformers-cli login` (necessary to use this script "
+#                 "with private models)."
+#             )
+#         },
+#     )
+
+
+@dataclass
+class DataTrainingArguments:
+    """
+    Arguments pertaining to what data we are going to input our model for training and eval.
+    """
+
+    dataset_name: Optional[str] = field(
+        default=None,
+        metadata={"help": "The name of the dataset to use (via the datasets library)."},
+    )
+    dataset_config_name: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "The configuration name of the dataset to use (via the datasets library)."
+        },
+    )
+    train_file: Optional[str] = field(
+        default=None, metadata={"help": "The input training data file (a text file)."}
+    )
+    validation_file: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."
+        },
+    )
+    train_ref_file: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "An optional input train ref data file for whole word masking in Chinese."
+        },
+    )
+    validation_ref_file: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "An optional input validation ref data file for whole word masking in Chinese."
+        },
+    )
+    overwrite_cache: bool = field(
+        default=False,
+        metadata={"help": "Overwrite the cached training and evaluation sets"},
+    )
+    validation_split_percentage: Optional[int] = field(
+        default=5,
+        metadata={
+            "help": "The percentage of the train set used as validation set in case there's no validation split"
+        },
+    )
+    max_seq_length: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The maximum total input sequence length after tokenization and masking. Sequences longer than this"
+                " will be truncated. Default to the max input length of the model."
+            )
+        },
+    )
+    preprocessing_num_workers: Optional[int] = field(
+        default=None,
+        metadata={"help": "The number of processes to use for the preprocessing."},
+    )
+    mlm_probability: float = field(
+        default=0.15,
+        metadata={
+            "help": "Ratio of tokens to mask for span masked language modeling loss"
+        },
+    )
+    mean_noise_span_length: float = field(
+        default=3.0,
+        metadata={"help": "Mean span length of masked tokens"},
+    )
+
+    def __post_init__(self):
+        if (
+            self.dataset_name is None
+            and self.train_file is None
+            and self.validation_file is None
+        ):
+            raise ValueError(
+                "Need either a dataset name or a training/validation file."
+            )
+        else:
+            if self.train_file is not None:
+                extension = self.train_file.split(".")[-1]
+                assert extension in ["csv", "json", "txt"], (
+                    "`train_file` should be a csv, a json or a txt file."
+                )
+            if self.validation_file is not None:
+                extension = self.validation_file.split(".")[-1]
+                assert extension in ["csv", "json", "txt"], (
+                    "`validation_file` should be a csv, a json or a txt file."
+                )
+
+
+def compute_input_and_target_lengths(
+    inputs_length, noise_density, mean_noise_span_length
+):
+    """This function is copy of `random_spans_helper <https://github.com/google-research/text-to-text-transfer-transformer/blob/84f8bcc14b5f2c03de51bd3587609ba8f6bbd1cd/t5/data/preprocessors.py#L2466>`__ .
+
+    Training parameters to avoid padding with random_spans_noise_mask.
+    When training a model with random_spans_noise_mask, we would like to set the other
+    training hyperparmeters in a way that avoids padding.
+    This function helps us compute these hyperparameters.
+    We assume that each noise span in the input is replaced by extra_tokens_per_span_inputs sentinel tokens,
+    and each non-noise span in the targets is replaced by extra_tokens_per_span_targets sentinel tokens.
+    This function tells us the required number of tokens in the raw example (for split_tokens())
+    as well as the length of the encoded targets. Note that this function assumes
+    the inputs and targets will have EOS appended and includes that in the reported length.
+
+    Args:
+        inputs_length: an integer - desired length of the tokenized inputs sequence
+        noise_density: a float
+        mean_noise_span_length: a float
+    Returns:
+        tokens_length: length of original text in tokens
+        targets_length: an integer - length in tokens of encoded targets sequence
+    """
+
+    def _tokens_length_to_inputs_length_targets_length(tokens_length):
+        num_noise_tokens = int(round(tokens_length * noise_density))
+        num_nonnoise_tokens = tokens_length - num_noise_tokens
+        num_noise_spans = int(round(num_noise_tokens / mean_noise_span_length))
+        # inputs contain all nonnoise tokens, sentinels for all noise spans
+        # and one EOS token.
+        _input_length = num_nonnoise_tokens + num_noise_spans + 1
+        _output_length = num_noise_tokens + num_noise_spans + 1
+        return _input_length, _output_length
+
+    tokens_length = inputs_length
+
+    while (
+        _tokens_length_to_inputs_length_targets_length(tokens_length + 1)[0]
+        <= inputs_length
+    ):
+        tokens_length += 1
+
+    inputs_length, targets_length = _tokens_length_to_inputs_length_targets_length(
+        tokens_length
+    )
+
+    # minor hack to get the targets length to be equal to inputs length
+    # which is more likely to have been set to a nice round number.
+    if noise_density == 0.5 and targets_length > inputs_length:
+        tokens_length -= 1
+        targets_length -= 1
+    return tokens_length, targets_length
+
+
+@flax.struct.dataclass
+class FlaxDataCollatorForT5MLM:
+    """
+    Data collator used for T5 span-masked language modeling.
+    It is made sure that after masking the inputs are of length `data_args.max_seq_length` and targets are also of fixed length.
+    For more information on how T5 span-masked language modeling works, one can take a look
+    at the `official paper <https://arxiv.org/pdf/1910.10683.pdf>`__
+    or the `official code for preprocessing <https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py>`__ .
+
+    Args:
+        tokenizer (:class:`~transformers.PreTrainedTokenizer` or :class:`~transformers.PreTrainedTokenizerFast`):
+            The tokenizer used for encoding the data.
+        noise_density (:obj:`float`):
+            The probability with which to (randomly) mask tokens in the input.
+        mean_noise_span_length (:obj:`float`):
+            The average span length of the masked tokens.
+        input_length (:obj:`int`):
+            The expected input length after masking.
+        target_length (:obj:`int`):
+            The expected target length after masking.
+        pad_token_id: (:obj:`int`):
+            The pad token id of the model
+        decoder_start_token_id: (:obj:`int):
+            The decoder start token id of the model
+    """
+
+    tokenizer: PreTrainedTokenizerBase
+    noise_density: float
+    mean_noise_span_length: float
+    input_length: int
+    target_length: int
+    pad_token_id: int
+    decoder_start_token_id: int
+
+    def __call__(self, examples: List[Dict[str, np.ndarray]]) -> Dict[str, np.ndarray]:
+        # convert list to dict and tensorize input
+        batch = BatchEncoding(
+            {
+                k: np.array([examples[i][k] for i in range(len(examples))])
+                for k, v in examples[0].items()
+            }
+        )
+
+        input_ids = batch["input_ids"]
+        batch_size, expandend_input_length = input_ids.shape
+
+        mask_indices = np.asarray(
+            [
+                self.random_spans_noise_mask(expandend_input_length)
+                for i in range(batch_size)
+            ]
+        )
+        labels_mask = ~mask_indices
+
+        input_ids_sentinel = self.create_sentinel_ids(mask_indices.astype(np.int8))
+        labels_sentinel = self.create_sentinel_ids(labels_mask.astype(np.int8))
+
+        batch["input_ids"] = self.filter_input_ids(input_ids, input_ids_sentinel)
+        batch["labels"] = self.filter_input_ids(input_ids, labels_sentinel)
+
+        if batch["input_ids"].shape[-1] != self.input_length:
+            raise ValueError(
+                f"`input_ids` are incorrectly preprocessed. `input_ids` length is {batch['input_ids'].shape[-1]}, but"
+                f" should be {self.target_length}."
+            )
+
+        if batch["labels"].shape[-1] != self.target_length:
+            raise ValueError(
+                f"`labels` are incorrectly preprocessed. `labels` length is {batch['labels'].shape[-1]}, but should be"
+                f" {self.target_length}."
+            )
+
+        # to check that tokens are correctly preprocessed, one can run `self.tokenizer.batch_decode(input_ids)` and `self.tokenizer.batch_decode(labels)` here...
+        batch["decoder_input_ids"] = shift_tokens_right(
+            batch["labels"], self.pad_token_id, self.decoder_start_token_id
+        )
+
+        return batch
+
+    def create_sentinel_ids(self, mask_indices):
+        """
+        Sentinel ids creation given the indices that should be masked.
+        The start indices of each mask are replaced by the sentinel ids in increasing
+        order. Consecutive mask indices to be deleted are replaced with `-1`.
+        """
+        start_indices = mask_indices - np.roll(mask_indices, 1, axis=-1) * mask_indices
+        start_indices[:, 0] = mask_indices[:, 0]
+
+        sentinel_ids = np.where(
+            start_indices != 0, np.cumsum(start_indices, axis=-1), start_indices
+        )
+        sentinel_ids = np.where(
+            sentinel_ids != 0, (len(self.tokenizer) - sentinel_ids), 0
+        )
+        sentinel_ids -= mask_indices - start_indices
+
+        return sentinel_ids
+
+    def filter_input_ids(self, input_ids, sentinel_ids):
+        """
+        Puts sentinel mask on `input_ids` and fuse consecutive mask tokens into a single mask token by deleting.
+        This will reduce the sequence length from `expanded_inputs_length` to `input_length`.
+        """
+        batch_size = input_ids.shape[0]
+
+        input_ids_full = np.where(sentinel_ids != 0, sentinel_ids, input_ids)
+        # input_ids tokens and sentinel tokens are >= 0, tokens < 0 are
+        # masked tokens coming after sentinel tokens and should be removed
+        input_ids = input_ids_full[input_ids_full >= 0].reshape((batch_size, -1))
+        input_ids = np.concatenate(
+            [
+                input_ids,
+                np.full((batch_size, 1), self.tokenizer.eos_token_id, dtype=np.int32),
+            ],
+            axis=-1,
+        )
+        return input_ids
+
+    def random_spans_noise_mask(self, length):
+        """This function is copy of `random_spans_helper <https://github.com/google-research/text-to-text-transfer-transformer/blob/84f8bcc14b5f2c03de51bd3587609ba8f6bbd1cd/t5/data/preprocessors.py#L2682>`__ .
+
+        Noise mask consisting of random spans of noise tokens.
+        The number of noise tokens and the number of noise spans and non-noise spans
+        are determined deterministically as follows:
+        num_noise_tokens = round(length * noise_density)
+        num_nonnoise_spans = num_noise_spans = round(num_noise_tokens / mean_noise_span_length)
+        Spans alternate between non-noise and noise, beginning with non-noise.
+        Subject to the above restrictions, all masks are equally likely.
+
+        Args:
+            length: an int32 scalar (length of the incoming token sequence)
+            noise_density: a float - approximate density of output mask
+            mean_noise_span_length: a number
+
+        Returns:
+            a boolean tensor with shape [length]
+        """
+
+        orig_length = length
+
+        num_noise_tokens = int(np.round(length * self.noise_density))
+        # avoid degeneracy by ensuring positive numbers of noise and nonnoise tokens.
+        num_noise_tokens = min(max(num_noise_tokens, 1), length - 1)
+        num_noise_spans = int(np.round(num_noise_tokens / self.mean_noise_span_length))
+
+        # avoid degeneracy by ensuring positive number of noise spans
+        num_noise_spans = max(num_noise_spans, 1)
+        num_nonnoise_tokens = length - num_noise_tokens
+
+        # pick the lengths of the noise spans and the non-noise spans
+        def _random_segmentation(num_items, num_segments):
+            """Partition a sequence of items randomly into non-empty segments.
+            Args:
+                num_items: an integer scalar > 0
+                num_segments: an integer scalar in [1, num_items]
+            Returns:
+                a Tensor with shape [num_segments] containing positive integers that add
+                up to num_items
+            """
+            mask_indices = np.arange(num_items - 1) < (num_segments - 1)
+            np.random.shuffle(mask_indices)
+            first_in_segment = np.pad(mask_indices, [[1, 0]])
+            segment_id = np.cumsum(first_in_segment)
+            # count length of sub segments assuming that list is sorted
+            _, segment_length = np.unique(segment_id, return_counts=True)
+            return segment_length
+
+        noise_span_lengths = _random_segmentation(num_noise_tokens, num_noise_spans)
+        nonnoise_span_lengths = _random_segmentation(
+            num_nonnoise_tokens, num_noise_spans
+        )
+
+        interleaved_span_lengths = np.reshape(
+            np.stack([nonnoise_span_lengths, noise_span_lengths], axis=1),
+            [num_noise_spans * 2],
+        )
+        span_starts = np.cumsum(interleaved_span_lengths)[:-1]
+        span_start_indicator = np.zeros((length,), dtype=np.int8)
+        span_start_indicator[span_starts] = True
+        span_num = np.cumsum(span_start_indicator)
+        is_noise = np.equal(span_num % 2, 1)
+
+        return is_noise[:orig_length]
+
+
+def generate_batch_splits(samples_idx: jnp.ndarray, batch_size: int) -> jnp.ndarray:
+    num_samples = len(samples_idx)
+    samples_to_remove = num_samples % batch_size
+
+    if samples_to_remove != 0:
+        samples_idx = samples_idx[:-samples_to_remove]
+    sections_split = num_samples // batch_size
+    batch_idx = np.split(samples_idx, sections_split)
+    return batch_idx
+
+
+def write_train_metric(summary_writer, train_metrics, train_time, step):
+    summary_writer.scalar("train_time", train_time, step)
+
+    train_metrics = get_metrics(train_metrics)
+    for key, vals in train_metrics.items():
+        tag = f"train_{key}"
+        for i, val in enumerate(vals):
+            summary_writer.scalar(tag, val, step - len(vals) + i + 1)
+
+
+def write_eval_metric(summary_writer, eval_metrics, step):
+    for metric_name, value in eval_metrics.items():
+        summary_writer.scalar(f"eval_{metric_name}", value, step)
+
+
+def main():
+    # See all possible arguments in src/transformers/training_args.py
+    # or by passing the --help flag to this script.
+    # We now keep distinct sets of args, for a cleaner separation of concerns.
+
+    parser = HfArgumentParser(
+        (ModelArguments, DataTrainingArguments, TrainingArguments)
+    )
+    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
+        # If we pass only one argument to the script and it's the path to a json file,
+        # let's parse it to get our arguments.
+        model_args, data_args, training_args = parser.parse_json_file(
+            json_file=os.path.abspath(sys.argv[1])
+        )
+    else:
+        model_args, data_args, training_args = parser.parse_args_into_dataclasses()
+
+    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
+    # information sent is the one passed as arguments along with your Python/PyTorch versions.
+    #     send_example_telemetry("run_t5_mlm", model_args, data_args, framework="flax")
+
+    if (
+        os.path.exists(training_args.output_dir)
+        and os.listdir(training_args.output_dir)
+        and training_args.do_train
+        and not training_args.overwrite_output_dir
+    ):
+        raise ValueError(
+            f"Output directory ({training_args.output_dir}) already exists and is not empty."
+            "Use --overwrite_output_dir to overcome."
+        )
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
+        level=logging.INFO,
+        datefmt="[%X]",
+    )
+
+    # Log on each process the small summary:
+    logger = logging.getLogger(__name__)
+
+    # Set the verbosity to info of the Transformers logger (on main process only):
+    logger.info(f"Training/evaluation parameters {training_args}")
+
+    # Set seed before initializing model.
+    set_seed(training_args.seed)
+
+    # Handle the repository creation
+    #     if training_args.push_to_hub:
+    #         if training_args.hub_model_id is None:
+    #             repo_name = get_full_repo_name(
+    #                 Path(training_args.output_dir).absolute().name, token=training_args.hub_token
+    #             )
+    #         else:
+    #             repo_name = training_args.hub_model_id
+    #         repo = Repository(training_args.output_dir, clone_from=repo_name)
+
+    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
+    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
+    # (the dataset will be downloaded automatically from the datasets Hub).
+    #
+    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
+    # 'text' is found. You can easily tweak this behavior (see below).
+    if data_args.dataset_name is not None:
+        # Downloading and loading a dataset from the hub.
+        datasets = load_dataset(
+            data_args.dataset_name,
+            data_args.dataset_config_name,
+            cache_dir=model_args.cache_dir,
+            #             use_auth_token=True if model_args.use_auth_token else None,
+        )
+
+        if "validation" not in datasets.keys():
+            datasets["validation"] = load_dataset(
+                data_args.dataset_name,
+                data_args.dataset_config_name,
+                split=f"train[:{data_args.validation_split_percentage}%]",
+                cache_dir=model_args.cache_dir,
+                #                 use_auth_token=True if model_args.use_auth_token else None,
+            )
+            datasets["train"] = load_dataset(
+                data_args.dataset_name,
+                data_args.dataset_config_name,
+                split=f"train[{data_args.validation_split_percentage}%:]",
+                cache_dir=model_args.cache_dir,
+                #                 use_auth_token=True if model_args.use_auth_token else None,
+            )
+    else:
+        data_files = {}
+        if data_args.train_file is not None:
+            data_files["train"] = data_args.train_file
+        if data_args.validation_file is not None:
+            data_files["validation"] = data_args.validation_file
+        extension = data_args.train_file.split(".")[-1]
+        if extension == "txt":
+            extension = "text"
+        datasets = load_dataset(
+            extension,
+            data_files=data_files,
+            cache_dir=model_args.cache_dir,
+            #             use_auth_token=True if model_args.use_auth_token else None,
+        )
+
+        if "validation" not in datasets.keys():
+            datasets["validation"] = load_dataset(
+                extension,
+                data_files=data_files,
+                split=f"train[:{data_args.validation_split_percentage}%]",
+                cache_dir=model_args.cache_dir,
+                #                 use_auth_token=True if model_args.use_auth_token else None,
+            )
+            datasets["train"] = load_dataset(
+                extension,
+                data_files=data_files,
+                split=f"train[{data_args.validation_split_percentage}%:]",
+                cache_dir=model_args.cache_dir,
+                #                 use_auth_token=True if model_args.use_auth_token else None,
+            )
+    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
+    # https://huggingface.co/docs/datasets/loading_datasets.html.
+
+    # Load pretrained model and tokenizer
+
+    if model_args.tokenizer_name:
+        tokenizer = AutoTokenizer.from_pretrained(
+            model_args.tokenizer_name,
+            cache_dir=model_args.cache_dir,
+            use_fast=model_args.use_fast_tokenizer,
+            #             use_auth_token=True if model_args.use_auth_token else None,
+        )
+    elif model_args.model_name_or_path:
+        tokenizer = AutoTokenizer.from_pretrained(
+            model_args.model_name_or_path,
+            cache_dir=model_args.cache_dir,
+            use_fast=model_args.use_fast_tokenizer,
+            #             use_auth_token=True if model_args.use_auth_token else None,
+        )
+    else:
+        raise ValueError(
+            "You are instantiating a new tokenizer from scratch. This is not supported by this script."
+            "You can do it from another script, save it, and load it from here, using --tokenizer_name."
+        )
+
+    if model_args.config_name:
+        config = T5Config.from_pretrained(
+            model_args.config_name,
+            cache_dir=model_args.cache_dir,
+            vocab_size=len(tokenizer),
+            #             use_auth_token=True if model_args.use_auth_token else None,
+        )
+    elif model_args.model_name_or_path:
+        config = T5Config.from_pretrained(
+            model_args.model_name_or_path,
+            cache_dir=model_args.cache_dir,
+            use_auth_token=True if model_args.use_auth_token else None,
+        )
+    else:
+        config = CONFIG_MAPPING[model_args.model_type]()
+        logger.warning("You are instantiating a new config instance from scratch.")
+
+    # Preprocessing the datasets.
+    # First we tokenize all the texts.
+    if training_args.do_train:
+        column_names = datasets["train"].column_names
+    else:
+        column_names = datasets["validation"].column_names
+    text_column_name = "text" if "text" in column_names else column_names[0]
+
+    max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length)
+
+    # Otherwise, we tokenize every text, then concatenate them together before splitting them in smaller parts.
+    # Since we make sure that all sequences are of the same length, no attention_mask is needed.
+    def tokenize_function(examples):
+        return tokenizer(examples[text_column_name], return_attention_mask=False)
+
+    tokenized_datasets = datasets.map(
+        tokenize_function,
+        batched=True,
+        num_proc=data_args.preprocessing_num_workers,
+        remove_columns=column_names,
+        load_from_cache_file=not data_args.overwrite_cache,
+    )
+
+    # T5-like span masked language modeling will fuse consecutively masked tokens to a single sentinel token.
+    # To ensure that the input length is `max_seq_length`, we need to increase the maximum length
+    # according to `mlm_probability` and `mean_noise_span_length`. We can also define the label length accordingly.
+    expanded_inputs_length, targets_length = compute_input_and_target_lengths(
+        inputs_length=max_seq_length,
+        noise_density=data_args.mlm_probability,
+        mean_noise_span_length=data_args.mean_noise_span_length,
+    )
+
+    # Main data processing function that will concatenate all texts from our dataset and generate chunks of expanded_inputs_length.
+    def group_texts(examples):
+        # Concatenate all texts.
+        concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()}
+        total_length = len(concatenated_examples[list(examples.keys())[0]])
+        # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
+        # customize this part to your needs.
+        if total_length >= expanded_inputs_length:
+            total_length = (
+                total_length // expanded_inputs_length
+            ) * expanded_inputs_length
+        # Split by chunks of max_len.
+        result = {
+            k: [
+                t[i : i + expanded_inputs_length]
+                for i in range(0, total_length, expanded_inputs_length)
+            ]
+            for k, t in concatenated_examples.items()
+        }
+        return result
+
+    # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a
+    # remainder for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value
+    # might be slower to preprocess.
+    #
+    # To speed up this part, we use multiprocessing. See the documentation of the map method for more information:
+    # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map
+    tokenized_datasets = tokenized_datasets.map(
+        group_texts,
+        batched=True,
+        num_proc=data_args.preprocessing_num_workers,
+        load_from_cache_file=not data_args.overwrite_cache,
+    )
+
+    # Enable tensorboard only on the master node
+    has_tensorboard = is_tensorboard_available()
+    if has_tensorboard and jax.process_index() == 0:
+        try:
+            from flax.metrics.tensorboard import SummaryWriter
+
+            summary_writer = SummaryWriter(log_dir=Path(training_args.output_dir))
+        except ImportError as ie:
+            has_tensorboard = False
+            logger.warning(
+                f"Unable to display metrics through TensorBoard because some package are not installed: {ie}"
+            )
+    else:
+        logger.warning(
+            "Unable to display metrics through TensorBoard because the package is not installed: "
+            "Please run pip install tensorboard to enable."
+        )
+
+    # Initialize our training
+    rng = jax.random.PRNGKey(training_args.seed)
+    dropout_rngs = jax.random.split(rng, jax.local_device_count())
+
+    if model_args.model_name_or_path:
+        model = FlaxT5ForConditionalGeneration.from_pretrained(
+            model_args.model_name_or_path,
+            config=config,
+            seed=training_args.seed,
+            dtype=getattr(jnp, model_args.dtype),
+            #             use_auth_token=True if model_args.use_auth_token else None,
+        )
+    else:
+        config.vocab_size = len(tokenizer)
+        model = FlaxT5ForConditionalGeneration(
+            config,
+            seed=training_args.seed,
+            dtype=getattr(jnp, model_args.dtype),
+            #             use_auth_token=True if model_args.use_auth_token else None,
+        )
+
+    # Data collator
+    # This one will take care of randomly masking the tokens.
+    data_collator = FlaxDataCollatorForT5MLM(
+        tokenizer=tokenizer,
+        noise_density=data_args.mlm_probability,
+        mean_noise_span_length=data_args.mean_noise_span_length,
+        input_length=max_seq_length,
+        target_length=targets_length,
+        pad_token_id=model.config.pad_token_id,
+        decoder_start_token_id=model.config.decoder_start_token_id,
+    )
+
+    # Store some constant
+    num_epochs = int(training_args.num_train_epochs)
+    train_batch_size = (
+        int(training_args.per_device_train_batch_size) * jax.device_count()
+    )
+    eval_batch_size = int(training_args.per_device_eval_batch_size) * jax.device_count()
+
+    num_train_steps = len(tokenized_datasets["train"]) // train_batch_size * num_epochs
+
+    num_of_hosts = jax.process_count()
+    current_host_idx = jax.process_index()
+
+    # Create learning rate schedule
+    warmup_fn = optax.linear_schedule(
+        init_value=0.0,
+        end_value=training_args.learning_rate,
+        transition_steps=training_args.warmup_steps,
+    )
+    decay_fn = optax.linear_schedule(
+        init_value=training_args.learning_rate,
+        end_value=0,
+        transition_steps=num_train_steps - training_args.warmup_steps,
+    )
+    linear_decay_lr_schedule_fn = optax.join_schedules(
+        schedules=[warmup_fn, decay_fn], boundaries=[training_args.warmup_steps]
+    )
+
+    # We use Optax's "masking" functionality to not apply weight decay
+    # to bias and LayerNorm scale parameters. decay_mask_fn returns a
+    # mask boolean with the same structure as the parameters.
+    # The mask is True for parameters that should be decayed.
+    def decay_mask_fn(params):
+        flat_params = traverse_util.flatten_dict(params)
+        flat_mask = {
+            path: (
+                path[-1] != "bias"
+                and path[-2:]
+                not in [("layer_norm", "scale"), ("final_layer_norm", "scale")]
+            )
+            for path in flat_params
+        }
+        return traverse_util.unflatten_dict(flat_mask)
+
+    # create adam optimizer
+    if training_args.adafactor:
+        # We use the default parameters here to initialize adafactor,
+        # For more details about the parameters please check https://github.com/deepmind/optax/blob/ed02befef9bf81cbbf236be3d2b0e032e9ed4a40/optax/_src/alias.py#L74
+        optimizer = optax.adafactor(
+            learning_rate=linear_decay_lr_schedule_fn,
+        )
+    else:
+        optimizer = optax.adamw(
+            learning_rate=linear_decay_lr_schedule_fn,
+            b1=training_args.adam_beta1,
+            b2=training_args.adam_beta2,
+            weight_decay=training_args.weight_decay,
+            mask=decay_mask_fn,
+        )
+
+    # Setup train state
+    state = train_state.TrainState.create(
+        apply_fn=model.__call__, params=model.params, tx=optimizer
+    )
+
+    # Define gradient update step fn
+    def train_step(state, batch, dropout_rng):
+        dropout_rng, new_dropout_rng = jax.random.split(dropout_rng)
+
+        def loss_fn(params):
+            labels = batch.pop("labels")
+
+            logits = state.apply_fn(
+                **batch, params=params, dropout_rng=dropout_rng, train=True
+            )[0]
+
+            # compute loss
+            loss = optax.softmax_cross_entropy(
+                logits, onehot(labels, logits.shape[-1])
+            ).mean()
+
+            return loss
+
+        grad_fn = jax.value_and_grad(loss_fn)
+        loss, grad = grad_fn(state.params)
+        grad = jax.lax.pmean(grad, "batch")
+        new_state = state.apply_gradients(grads=grad)
+
+        metrics = jax.lax.pmean(
+            {"loss": loss, "learning_rate": linear_decay_lr_schedule_fn(state.step)},
+            axis_name="batch",
+        )
+
+        return new_state, metrics, new_dropout_rng
+
+    # Create parallel version of the train step
+    p_train_step = jax.pmap(train_step, "batch", donate_argnums=(0,))
+
+    # Define eval fn
+    def eval_step(params, batch):
+        labels = batch.pop("labels")
+
+        logits = model(**batch, params=params, train=False)[0]
+
+        # compute loss
+        loss = optax.softmax_cross_entropy(logits, onehot(labels, logits.shape[-1]))
+
+        # compute accuracy
+        accuracy = jnp.equal(jnp.argmax(logits, axis=-1), labels)
+
+        # summarize metrics
+        metrics = {"loss": loss.mean(), "accuracy": accuracy.mean()}
+        metrics = jax.lax.pmean(metrics, axis_name="batch")
+
+        return metrics
+
+    p_eval_step = jax.pmap(eval_step, "batch", donate_argnums=(0,))
+
+    # Replicate the train state on each device
+    state = jax_utils.replicate(state)
+
+    train_time = 0
+    eval_loss = float("inf")
+    epochs = tqdm(range(num_epochs), desc="Epoch ... ", position=0)
+    for epoch in epochs:
+        # ======================== Training ================================
+        train_start = time.time()
+        train_metrics = []
+
+        # Create sampling rng
+        rng, input_rng = jax.random.split(rng)
+
+        # Generate an epoch by shuffling sampling indices from the train dataset
+        num_train_samples = len(tokenized_datasets["train"])
+        train_samples_idx = np.random.permutation(np.arange(num_train_samples))
+        train_batch_idx = generate_batch_splits(train_samples_idx, train_batch_size)
+
+        # Gather the indexes for creating the batch and do a training step
+        for step, batch_idx in enumerate(
+            tqdm(train_batch_idx, desc="Training...", position=1)
+        ):
+            samples = [tokenized_datasets["train"][int(idx)] for idx in batch_idx]
+            model_inputs = data_collator(samples)
+
+            local_host_model_inputs = {
+                key: np.split(model_inputs.data[key], num_of_hosts, axis=0)[
+                    current_host_idx
+                ]
+                for key, value in model_inputs.data.items()
+            }
+
+            # Model forward
+            model_inputs = shard(local_host_model_inputs)
+            state, train_metric, dropout_rngs = p_train_step(
+                state, model_inputs, dropout_rngs
+            )
+            train_metrics.append(train_metric)
+
+            cur_step = epoch * (num_train_samples // train_batch_size) + step
+
+            if cur_step % training_args.logging_steps == 0 and cur_step > 0:
+                # Save metrics
+                train_metric = jax_utils.unreplicate(train_metric)
+                train_time += time.time() - train_start
+                if has_tensorboard and jax.process_index() == 0:
+                    write_train_metric(
+                        summary_writer, train_metrics, train_time, cur_step
+                    )
+
+                epochs.write(
+                    f"Step... ({cur_step} | Loss: {train_metric['loss'].mean()}, Learning Rate:"
+                    f" {train_metric['learning_rate'].mean()})"
+                )
+
+                train_metrics = []
+
+            if cur_step % training_args.eval_steps == 0 and cur_step > 0:
+                # ======================== Evaluating ==============================
+                num_eval_samples = len(tokenized_datasets["validation"])
+                eval_samples_idx = jnp.arange(num_eval_samples)
+                eval_batch_idx = generate_batch_splits(
+                    eval_samples_idx, eval_batch_size
+                )
+
+                eval_metrics = []
+                for i, batch_idx in enumerate(
+                    tqdm(eval_batch_idx, desc="Evaluating ...", position=2)
+                ):
+                    samples = [
+                        tokenized_datasets["validation"][int(idx)] for idx in batch_idx
+                    ]
+                    model_inputs = data_collator(samples)
+
+                    # Model forward
+                    model_inputs = shard(model_inputs.data)
+                    metrics = p_eval_step(state.params, model_inputs)
+                    eval_metrics.append(metrics)
+
+                # get eval metrics
+                eval_metrics = get_metrics(eval_metrics)
+                eval_metrics = jax.tree_map(jnp.mean, eval_metrics)
+
+                # Update progress bar
+                epochs.write(
+                    f"Step... ({cur_step} | Loss: {eval_metrics['loss']}, Acc: {eval_metrics['accuracy']})"
+                )
+
+                # Save metrics
+                if has_tensorboard and jax.process_index() == 0:
+                    write_eval_metric(summary_writer, eval_metrics, cur_step)
+
+                # Save model if eval_metrics['loss'] < eval_loss
+                if eval_metrics["loss"] < eval_loss:
+                    eval_loss = eval_metrics["loss"]
+                    if jax.process_index() == 0:
+                        params = jax.device_get(
+                            jax.tree_map(lambda x: x[0], state.params)
+                        )
+                        model.save_pretrained(training_args.output_dir, params=params)
+                        tokenizer.save_pretrained(training_args.output_dir)
+                        print(
+                            f"Step: {cur_step}, Current eval_loss is {eval_loss}, checkpoint is saved!!"
+                        )
+                else:
+                    eval_loss = eval_metrics["loss"]
+                    print(f"Step: {cur_step}, Current eval_loss is {eval_loss}")
+
+            # if cur_step % training_args.save_steps == 0 and cur_step > 0:
+            #     # save checkpoint after each epoch and push checkpoint to the hub
+            #     if jax.process_index() == 0:
+            #         params = jax.device_get(jax.tree_map(lambda x: x[0], state.params))
+            #         model.save_pretrained(training_args.output_dir, params=params)
+            #         tokenizer.save_pretrained(training_args.output_dir)
+            #         if training_args.push_to_hub:
+            #             repo.push_to_hub(commit_message=f"Saving weights and logs of step {cur_step}", blocking=False)
+
+    # Eval after training
+    if training_args.do_eval:
+        num_eval_samples = len(tokenized_datasets["validation"])
+        eval_samples_idx = jnp.arange(num_eval_samples)
+        eval_batch_idx = generate_batch_splits(eval_samples_idx, eval_batch_size)
+
+        eval_metrics = []
+        for i, batch_idx in enumerate(
+            tqdm(eval_batch_idx, desc="Evaluating ...", position=2)
+        ):
+            samples = [tokenized_datasets["validation"][int(idx)] for idx in batch_idx]
+            model_inputs = data_collator(samples)
+
+            # Model forward
+            model_inputs = shard(model_inputs.data)
+            metrics = p_eval_step(state.params, model_inputs)
+            eval_metrics.append(metrics)
+
+        # get eval metrics
+        eval_metrics = get_metrics(eval_metrics)
+        eval_metrics = jax.tree_map(
+            lambda metric: jnp.mean(metric).item(), eval_metrics
+        )
+
+        if jax.process_index() == 0:
+            eval_metrics = {
+                f"eval_{metric_name}": value
+                for metric_name, value in eval_metrics.items()
+            }
+            path = os.path.join(training_args.output_dir, "eval_results.json")
+            with open(path, "w") as f:
+                json.dump(eval_metrics, f, indent=4, sort_keys=True)
+
+
+if __name__ == "__main__":
+    main()

CompoundT5/CompoundT5/run.sh

@@ -0,0 +1,20 @@
+python ./new_run_t5_mlm_flax.py \
+  --output_dir="./CompoundT5-output" \
+  --model_type="t5" \
+  --config_name="./CompoundT5-config" \
+  --tokenizer_name="./CompoundT5-config" \
+  --dataset_name="sagawa/ZINC-canonicalized" \
+  --max_seq_length="512" \
+  --per_device_train_batch_size="5" \
+  --per_device_eval_batch_size="5" \
+  --adafactor \
+  --learning_rate="0.005" \
+  --weight_decay="0.001" \
+  --warmup_steps="2000" \
+  --overwrite_output_dir \
+  --logging_steps="500" \
+  --save_steps="100000" \
+  --num_train_epochs="30" \
+  --do_train \
+  --do_eval \
+  --eval_steps="100000"

CompoundT5/README.md

@@ -0,0 +1,35 @@
+# CompoundT5
+Here, we will explain how to do compound pre-training. 
+
+# Installation
+To get started, you will first need to install the necessary libraries. You can use the requirements.yaml file for this purpose. If the versions of torch and jax do not match your environment, you can change and run the following command:
+```
+conda install -c conda-forge rdkit gdown scikit-learn
+conda install pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch
+pip install tokenizers==0.12.1 transformers==4.21.0 datasets sentencepiece==0.1.96
+pip install "jax[cuda11_cudnn82]" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
+pip install flax
+```
+This will install all the necessary libraries for the project.
+
+The original data used for this study is uploaded to Google Drive and can be found at the following links:
+・[ZINC](https://drive.google.com/drive/folders/1SgM35D14JUqgNILxaiRQYbZoyooFOF-3)  
+・[ORD](https://drive.google.com/file/d/1Qbsl8_CmdIK_iNNY8F6wATVnDQNSW9Tc/view?usp=drive_link)  
+The pre-processed data is also available on [Hugging Face Hub](https://huggingface.co/sagawa) and can be used directly. 
+
+To download the data, you can run the following command:
+```
+python preprocess_data.py
+```
+To complete the preparation for compound pre-training, run the following command:
+```
+python prepare_model.py
+```
+
+# Compound pre-training
+Run the following command to conduct compound pre-training. In compound pre-training, T5 is trained on the ZINC dataset using span-masked language modeling. The pretraine model (CompoundT5) is uploaded to [Hugging Face Hub](https://huggingface.co/sagawa/CompoundT5).
+```
+cd CompoundT5
+sh run.sh
+```
+Please note that if your GPU memory size is small, you may encounter an out-of-memory error during T5 pre-training. If this occurs, you can try reducing the batch size or you can try putting XLA_PYTHON_CLIENT_MEM_FRACTION=.8 before python ./new_run_t5_mlm_flax.py in run.sh file. This reduces GPU memory preallocation.

CompoundT5/prepare_model.py

@@ -0,0 +1,208 @@
+# https://github.com/huggingface/transformers/blob/main/examples/flax/language-modeling/t5_tokenizer_model.py
+
+import argparse
+import json
+import os
+import sys
+from typing import Iterator, List, Union
+
+import datasets
+from datasets import load_dataset
+from tokenizers import (
+    AddedToken,
+    Regex,
+    Tokenizer,
+    decoders,
+    normalizers,
+    pre_tokenizers,
+    trainers,
+)
+from tokenizers.implementations.base_tokenizer import BaseTokenizer
+from tokenizers.models import Unigram
+from tokenizers.processors import TemplateProcessing
+from transformers import AutoTokenizer, T5Config
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from utils import seed_everything
+
+seed_everything(seed=42)
+
+script_dir = os.path.abspath(os.path.dirname(__file__))
+project_root = os.path.abspath(os.path.join(script_dir, ".."))
+data_dir = os.path.join(project_root, "data")
+
+
+class SentencePieceUnigramTokenizer(BaseTokenizer):
+    """
+    This class is a copy of `DeDLOC's tokenizer implementation <https://github.com/yandex-research/DeDLOC/blob/main/sahajbert/tokenizer/tokenizer_model.py>`__ .
+
+    Custom SentencePiece Unigram Tokenizer with NMT, NKFC, spaces and lower-casing characters normalization
+    Represents the Unigram algorithm, with the pretokenization used by SentencePiece
+    """
+
+    def __init__(
+        self,
+        replacement: str = "▁",
+        add_prefix_space: bool = True,
+        unk_token: Union[str, AddedToken] = "<unk>",
+        eos_token: Union[str, AddedToken] = "</s>",
+        pad_token: Union[str, AddedToken] = "<pad>",
+    ):
+        self.special_tokens = {
+            "pad": {"id": 0, "token": pad_token},
+            "eos": {"id": 1, "token": eos_token},
+            "unk": {"id": 2, "token": unk_token},
+        }
+
+        self.special_tokens_list = [None] * len(self.special_tokens)
+        for token_dict in self.special_tokens.values():
+            self.special_tokens_list[token_dict["id"]] = token_dict["token"]
+
+        tokenizer = Tokenizer(Unigram())
+
+        tokenizer.normalizer = normalizers.Sequence(
+            [
+                normalizers.Nmt(),
+                normalizers.NFKC(),
+                normalizers.Replace(Regex(" {2,}"), " "),
+                #                 normalizers.Lowercase(),
+            ]
+        )
+        tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
+            [
+                pre_tokenizers.Metaspace(
+                    replacement=replacement, add_prefix_space=add_prefix_space
+                ),
+                pre_tokenizers.Digits(individual_digits=True),
+                pre_tokenizers.Punctuation(),
+            ]
+        )
+        tokenizer.decoder = decoders.Metaspace(
+            replacement=replacement, add_prefix_space=add_prefix_space
+        )
+
+        tokenizer.post_processor = TemplateProcessing(
+            single=f"$A {self.special_tokens['eos']['token']}",
+            special_tokens=[
+                (self.special_tokens["eos"]["token"], self.special_tokens["eos"]["id"])
+            ],
+        )
+
+        parameters = {
+            "model": "SentencePieceUnigram",
+            "replacement": replacement,
+            "add_prefix_space": add_prefix_space,
+        }
+
+        super().__init__(tokenizer, parameters)
+
+    def train(
+        self,
+        files: Union[str, List[str]],
+        vocab_size: int = 8000,
+        show_progress: bool = True,
+    ):
+        """Train the model using the given files"""
+
+        trainer = trainers.UnigramTrainer(
+            vocab_size=vocab_size,
+            special_tokens=self.special_tokens_list,
+            show_progress=show_progress,
+        )
+
+        if isinstance(files, str):
+            files = [files]
+        self._tokenizer.train(files, trainer=trainer)
+
+        self.add_unk_id()
+
+    def train_from_iterator(
+        self,
+        iterator: Union[Iterator[str], Iterator[Iterator[str]]],
+        vocab_size: int = 8000,
+        show_progress: bool = True,
+    ):
+        """Train the model using the given iterator"""
+
+        trainer = trainers.UnigramTrainer(
+            vocab_size=vocab_size,
+            special_tokens=self.special_tokens_list,
+            show_progress=show_progress,
+        )
+
+        self._tokenizer.train_from_iterator(iterator, trainer=trainer)
+
+        self.add_unk_id()
+
+    def add_unk_id(self):
+        tokenizer_json = json.loads(self._tokenizer.to_str())
+
+        tokenizer_json["model"]["unk_id"] = self.special_tokens["unk"]["id"]
+
+        self._tokenizer = Tokenizer.from_str(json.dumps(tokenizer_json))
+
+
+def create_normal_tokenizer(dataset, model_name):
+    if isinstance(dataset, datasets.dataset_dict.DatasetDict):
+        training_corpus = (
+            dataset["train"][i : i + 1000]["smiles"]
+            for i in range(0, len(dataset), 1000)
+        )
+    else:
+        training_corpus = (
+            dataset[i : i + 1000]["smiles"] for i in range(0, len(dataset), 1000)
+        )
+
+    if "deberta" in model_name:
+        # Train tokenizer
+        old_tokenizer = AutoTokenizer.from_pretrained(model_name)
+        tokenizer = old_tokenizer.train_new_from_iterator(training_corpus, 1000)
+    elif "t5" in model_name:
+        tokenizer = SentencePieceUnigramTokenizer(
+            unk_token="<unk>", eos_token="</s>", pad_token="<pad>"
+        )
+        tokenizer.train_from_iterator(training_corpus, 1000)
+
+    return tokenizer
+
+
+def create_character_level_tokenizer(dataset, model_name):
+    df = dataset["train"].to_pandas()
+    df["smiles"] = [" ".join(list(i)) for i in df["smiles"]]
+    dataset = datasets.Dataset.from_pandas(df)
+
+    tokenizer = create_normal_tokenizer(dataset, model_name)
+
+    return tokenizer
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--use_character_level_tokenizer",
+        action="store_true",
+        default=False,
+        required=False,
+    )
+    return parser.parse_args()
+
+
+CFG = parse_args()
+
+
+# Initialize a dataset
+dataset = load_dataset(
+    "csv", data_files=os.path.join(data_dir, "ZINC-canonicalized.csv")
+)
+
+if CFG.use_character_level_tokenizer:
+    tokenizer = create_character_level_tokenizer(dataset, "t5")
+else:
+    tokenizer = create_normal_tokenizer(dataset, "t5")
+# Save files to disk
+tokenizer.save(os.path.join(script_dir, "CompoundT5/CompoundT5-config/tokenizer.json"))
+
+config = T5Config.from_pretrained(
+    "google/t5-v1_1-base", vocab_size=tokenizer.get_vocab_size()
+)
+config.save_pretrained(os.path.join(script_dir, "CompoundT5/CompoundT5-config/"))

CompoundT5/preprocess_data.py

@@ -0,0 +1,168 @@
+import os
+import subprocess
+import sys
+import warnings
+
+import pandas as pd
+from rdkit import Chem, RDLogger
+from sklearn.model_selection import train_test_split
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from utils import remove_atom_mapping, seed_everything
+
+seed_everything(seed=42)
+
+# Disable RDKit warnings and Python warnings
+RDLogger.DisableLog("rdApp.*")
+warnings.filterwarnings("ignore")
+
+script_dir = os.path.abspath(os.path.dirname(__file__))
+project_root = os.path.abspath(os.path.join(script_dir, ".."))
+data_dir = os.path.join(project_root, "data")
+
+files_to_download = [
+    "1ZPsoUYb4HcxFzK_ac9rb_pQj7oO3Gagh",
+    "1XwkxxHiaWFbSNhGyxnv6hAliutIMNrIp",
+    "1yIwUH_OhER9nuMo9HjBhBmyc6zvmrSPA",
+    "1skFRirstIUijhieshvJEScBD2aB3H1YU",
+    "1fa2MyLdN1vcA7Rysk8kLQENE92YejS9B",
+]
+
+for file_id in files_to_download:
+    subprocess.run(
+        f"gdown 'https://drive.google.com/uc?export=download&id={file_id}'", shell=True
+    )
+
+# Move downloaded files to data directory
+subprocess.run("mv *.smi " + data_dir, shell=True)
+subprocess.run("mv *.tsv " + data_dir, shell=True)
+
+
+# Function to process SMILES files and save canonicalized versions
+def process_smiles_files(file_paths):
+    unique_smiles = set()
+    for file_path in file_paths:
+        suppl = Chem.SmilesMolSupplier(file_path)
+        for mol in suppl:
+            if mol is not None:
+                try:
+                    sm = Chem.MolToSmiles(mol, canonical=True)
+                    unique_smiles.add(sm)
+                except:
+                    continue
+    df = pd.DataFrame({"smiles": list(unique_smiles)})
+    df.to_csv(os.path.join(data_dir, "ZINC-canonicalized.csv"), index=False)
+
+    train, valid = train_test_split(df, test_size=0.1)
+    # Save train and validation data
+    train.to_csv(os.path.join(data_dir, "ZINC-canonicalized-train.csv"), index=False)
+    valid.to_csv(os.path.join(data_dir, "ZINC-canonicalized-valid.csv"), index=False)
+
+
+# Process 16_p files
+process_smiles_files([os.path.join(data_dir, f"16_p{i}.smi") for i in range(4)])
+
+
+# Load reaction data
+ord_df = pd.read_csv(
+    os.path.join(data_dir, "all_ord_reaction_uniq_with_attr20240506_v1.tsv"),
+    sep="\t",
+    names=["id", "input", "product", "condition"],
+)
+
+
+def data_split(row):
+    categories = [
+        "CATALYST",
+        "REACTANT",
+        "REAGENT",
+        "SOLVENT",
+        "INTERNAL_STANDARD",
+        "NoData",
+    ]
+    data = {cat: [] for cat in categories}
+    input_data = row["input"]
+
+    if isinstance(input_data, str):
+        for item in input_data.split("."):
+            for cat in categories:
+                if cat in item:
+                    data[cat].append(item[item.find(":") + 1 :])
+                    break
+
+    for key, value in data.items():
+        data[key] = ".".join(value)
+
+    product_data = row["product"]
+    if isinstance(product_data, str):
+        product_data = product_data.replace(".PRODUCT", "PRODUCT")
+        pro_lis = []
+        for item in product_data.split("PRODUCT:"):
+            if item != "":
+                pro_lis.append(item)
+        data["PRODUCT"] = ".".join(pro_lis)
+    else:
+        data["PRODUCT"] = None
+
+    condition_data = row["condition"]
+    if isinstance(condition_data, str):
+        data["YIELD"] = (
+            float(condition_data.split(":")[1]) if "YIELD" in condition_data else None
+        )
+        temp_pos = condition_data.find("TEMP")
+        data["TEMP"] = (
+            float(condition_data[temp_pos:].split(":")[1])
+            if "TEMP" in condition_data
+            else None
+        )
+    else:
+        data["YIELD"] = None
+        data["TEMP"] = None
+
+    return list(data.values())
+
+
+# Split data and create cleaned DataFrame
+categories = [
+    "CATALYST",
+    "REACTANT",
+    "REAGENT",
+    "SOLVENT",
+    "INTERNAL_STANDARD",
+    "NoData",
+    "PRODUCT",
+    "YIELD",
+    "TEMP",
+]
+cleaned_data = {cat: [] for cat in categories}
+
+for _, row in ord_df.iterrows():
+    split_data = data_split(row)
+    for i, value in enumerate(split_data):
+        cleaned_data[categories[i]].append(value)
+
+cleaned_df = pd.DataFrame(cleaned_data)
+
+# Apply remove_atom_mapping function to relevant columns
+for column in [
+    "CATALYST",
+    "REACTANT",
+    "REAGENT",
+    "SOLVENT",
+    "INTERNAL_STANDARD",
+    "NoData",
+    "PRODUCT",
+]:
+    cleaned_df[column] = cleaned_df[column].apply(
+        lambda x: remove_atom_mapping(x) if isinstance(x, str) else None
+    )
+
+# Save cleaned DataFrame
+cleaned_df.to_csv(os.path.join(data_dir, "preprocessed_ord.tsv"), index=False)
+
+train, valid = train_test_split(cleaned_df, test_size=int(len(cleaned_df) * 0.1))
+train, test = train_test_split(train, test_size=int(len(cleaned_df) * 0.1))
+# Save train and validation data
+train.to_csv(os.path.join(data_dir, "preprocessed_ord_train.csv"), index=False)
+valid.to_csv(os.path.join(data_dir, "preprocessed_ord_valid.csv"), index=False)
+test.to_csv(os.path.join(data_dir, "preprocessed_ord_test.csv"), index=False)

LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Tatsuya Sagawa
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/additional_tokens.txt

@@ -0,0 +1,46 @@
+.
+6
+7
+8
+<
+>
+Ag
+Al
+Ar
+As
+Au
+Ba
+Bi
+Ca
+Cl
+Cu
+Fe
+Ge
+Hg
+K
+Li
+Mg
+Mn
+Mo
+Na
+Nd
+Ni
+P
+Pb
+Pd
+Pt
+Re
+Rh
+Ru
+Sb
+Si
+Sm
+Ta
+Ti
+Tl
+W
+Yb
+Zn
+Zr
+e
+p

data/data_analysis.ipynb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c94d83bc9377afc96d9f0d8033b1bf4e4b81c5ec9a6227c8c44be494cfb52c0
+size 14612076

data/demo_reaction_data.csv

@@ -0,0 +1,113 @@
+,id,REACTANT,PRODUCT,REAGENT,SOLVENT,CATALYST,YIELD
+0,ord-c2af606677024e008e8fb05d402e9b3b,B1C2CCCC1CCC2.C=CC1CCN(C(=O)OC(C)(C)C)C1.ClCCl.FC1(F)Oc2ccc(Br)cc2O1.N#N.O=C([O-])[O-].[K+].[K+].[Na+].[OH-],CC(C)(C)OC(=O)N1CCC(CCc2ccc3c(c2)OC(F)(F)O3)C1,Cl[Pd]Cl.[Fe+2].c1ccc(P(c2ccccc2)[c-]2cccc2)cc1.c1ccc(P(c2ccccc2)[c-]2cccc2)cc1,CN(C)C=O.O.O,Cl[Pd]Cl.[Fe+2].c1ccc(P(c2ccccc2)[c-]2cccc2)cc1.c1ccc(P(c2ccccc2)[c-]2cccc2)cc1,0.98
+1,ord-96c71ebfff6c4ee8bb6a5aa2960c2ba3,Brc1cncc(I)c1.C#C[Si](C)(C)C,C[Si](C)(C)C#Cc1cncc(Br)c1,Cl[Pd](Cl)([P](c1ccccc1)(c1ccccc1)c1ccccc1)[P](c1ccccc1)(c1ccccc1)c1ccccc1,CC#N.CCN(CC)CC,Cl[Pd](Cl)([P](c1ccccc1)(c1ccccc1)c1ccccc1)[P](c1ccccc1)(c1ccccc1)c1ccccc1,0.99
+2,ord-6e64400b2c7a4e9a8789b9e79cfcee25,Brc1ccc(-c2nc3ccccc3o2)nc1.CC1(C)OB(c2ccc(-c3ccc(N(c4ccccc4)c4ccccc4)cc3)nc2)OC1(C)C.O.O=C([O-])[O-].[Na+].[Na+],c1ccc(N(c2ccccc2)c2ccc(-c3ccc(-c4ccc(-c5nc6ccccc6o5)nc4)cn3)cc2)cc1,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,C1CCOC1.CC(C)=O.ClCCl.ClCCl,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,0.77
+3,ord-b77f09a6bfcb449c8320355cea96234a,Brc1ccc2ncccc2c1.Cc1ccc2c(cnn2C2CCCCO2)c1[B-](F)(F)F.[K+],Cc1ccc2c(cnn2C2CCCCO2)c1-c1ccc2ncccc2c1,C1CCC(P(C2CCCCC2)C2CCCCC2)CC1.CC(=O)[O-].CC(=O)[O-].CCN(CC)CC.[Pd+2],C1CCOC1.CC#N.CCc1cc(CC)cc(CC)c1.CCc1cccc(CC)c1.CN(C)C=O.Cc1ccccc1.O,C1CCC(P(C2CCCCC2)C2CCCCC2)CC1.CC(=O)[O-].CC(=O)[O-].[Pd+2],0.13
+4,ord-a78f05158222489f8516262e16c2e1d4,CC=CCOc1ccc(Cl)cc1I.CCCC[NH3+].O=C([O-])[O-].O=C[O-].[Cl-].[Na+].[Na+].[Na+],CCc1coc2ccc(Cl)cc12,CC(=O)[O-].CC(=O)[O-].[Pd+2],CN(C)C=O,CC(=O)[O-].CC(=O)[O-].[Pd+2],0.6
+5,ord-5dee9b14e71f4696a55160a9b199f823,COC(=O)[C@@H]1CCCN1C[C@@H](CO)NC(=O)OCc1ccccc1.[H][H],O=C1N[C@H](CO)CN2CCC[C@@H]12,[Pd],CO.CO,[Pd],0.85
+6,ord-414cb258bf5f49a68600ff46cab8a198,Cc1nc2ccccc2[nH]1.ClCCCCBr.[Na+].[OH-],Cc1nc2ccccc2n1CCCCCl,CCCC[N+](CCCC)(CCCC)CCCC.[Br-],ClCCl.ClCCl,CCCC[N+](CCCC)(CCCC)CCCC.[Br-],0.62
+7,ord-89b9b02c63de42c69f3741b8d60df759,Brc1ccc(-c2ccc3c(-c4ccccc4)c4ccccc4c(-c4ccccc4)c3c2)cc1.CC(C)(C)P(C(C)(C)C)C(C)(C)C.CC(C)(C)[O-].[Na+].c1ccc2c(c1)[nH]c1ccc(-c3cccc4c3oc3ccccc34)cc12,c1ccc(-c2c3ccccc3c(-c3ccccc3)c3cc(-c4ccc(-n5c6ccccc6c6cc(-c7cccc8c7oc7ccccc78)ccc65)cc4)ccc23)cc1,O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[Pd],CCCCCC.Cc1ccccc1,O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[Pd],0.71
+8,ord-79ba34c3a2a842aeac9844c9adfd81cd,C1COCCO1.CS(=O)(=O)c1ncc(OCC2CC2)c(Cl)n1.Cn1cc(B2OC(C)(C)C(C)(C)O2)c2c(c1=O)CCCC2.O.O=P([O-])([O-])[O-].[K+].[K+].[K+],Cn1cc(-c2nc(S(C)(=O)=O)ncc2OCC2CC2)c2c(c1=O)CCCC2,Cl[Pd]Cl.[Fe+2].c1ccc(P(c2ccccc2)[c-]2cccc2)cc1.c1ccc(P(c2ccccc2)[c-]2cccc2)cc1,CCOC(C)=O,Cl[Pd]Cl.[Fe+2].c1ccc(P(c2ccccc2)[c-]2cccc2)cc1.c1ccc(P(c2ccccc2)[c-]2cccc2)cc1,0.67
+9,ord-ff39310cbf9b4ffeae75d7f1c5107b64,COC(=O)c1cc2c(cc1[N+](=O)[O-])OCCO2.NN.O,COC(=O)c1cc2c(cc1N)OCCO2,[Pd],CO.CO,[Pd],0.5
+10,ord-3df4e020c7884aa982896480f6a21ef8,C1=COCCC1.CC(CCCCBr)(COC1CCCCO1)c1ccccc1.CC(CO)(CCCBr)c1ccccc1,CC(CCCBr)(COC1CCCCO1)c1ccccc1,Cc1ccc(S(=O)(=O)O)cc1.O,ClCCl,Cc1ccc(S(=O)(=O)O)cc1.O,0.95
+11,ord-da92b8fd5ded4564a5da0bad090429ef,CCOC(=O)C1CCN(Cc2ccccc2)CC1=O.Cl,CCOC(=O)C1CCNCC1=O.Cl,[Pd],CCO,[Pd],0.84
+12,ord-1dd27e38c75347908e354bc047bffa9d,COC(=O)c1cc2cc(C)cc([N+](=O)[O-])c2n1C(=O)OC(C)(C)C.O=C1CCC(=O)N1Br,COC(=O)c1cc2cc(CBr)cc([N+](=O)[O-])c2n1C(=O)OC(C)(C)C,CC(C)(C#N)N=NC(C)(C)C#N,ClC(Cl)(Cl)Cl,CC(C)(C#N)N=NC(C)(C)C#N,1.0
+13,ord-eb1e3da77b1a47679dbc3f0f04feb804,CC(=O)[O-].N#CC(c1ccnc(Cl)n1)c1nc2ccccc2s1.[Na+],N#CC(c1ccncn1)c1nc2ccccc2s1,[Pd],CC(=O)O,[Pd],0.13
+14,ord-652b7ae23fcd4311981e5f0157f7d978,Nc1cc2cc(Br)ccc2c(Br)n1.O=C[O-].O=C[O-].[NH4+].[NH4+],Nc1cc2cc(Br)ccc2cn1,O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[Pd].[Pd].[Pd].[Pd],CN(C)C=O,O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[Pd].[Pd].[Pd].[Pd],0.9
+15,ord-e2070dae8ca14929bbd39eaf3388600c,CCCCOc1c(CNC(=O)OC(C)(C)C)n(CC(C)C)c(=O)c2ccc(OCc3ccccc3)cc12,CCCCOc1c(CNC(=O)OC(C)(C)C)n(CC(C)C)c(=O)c2ccc(O)cc12,[C].[Pd],C1CCOC1.CCO,[C].[Pd],0.97
+16,ord-1c4c9374ded94ddd8102c9a833fd2784,Cn1cnc(C#N)c1.Fc1ccccc1Br,Cn1cnc(C#N)c1-c1ccccc1F.Cn1cnc(C#N)c1-c1ccccc1F,C=C[CH2-].C=C[CH2-].CC(C)(C)C(=O)[O-].CP(C)c1ccccc1.Cl[Pd+].Cl[Pd+].[K+],CCCC#N,C=C[CH2-].C=C[CH2-].CP(C)c1ccccc1.Cl[Pd+].Cl[Pd+],0.0
+17,ord-c4f8232274154e2a8a24dae87cdf7ee4,COc1ccc(N(C(=O)c2ccc(C)s2)C(=O)c2ccc(C)s2)c(Br)c1,COc1ccc2[nH]c(=O)c3sc(C)cc3c2c1,CC(C)(C)[P]([Pd][P](C(C)(C)C)(C(C)(C)C)C(C)(C)C)(C(C)(C)C)C(C)(C)C,,CC(C)(C)[P]([Pd][P](C(C)(C)C)(C(C)(C)C)C(C)(C)C)(C(C)(C)C)C(C)(C)C,0.48
+18,ord-3ec9d7738cee4b779ecd6978452b948a,CNCCNC.COc1cn(-c2cccc(Br)c2F)nc(-c2ccnn2-c2ccccc2)c1=O.O=C([O-])O.O=C1CCCCN1.O=P([O-])([O-])[O-].[K+].[K+].[K+].[Na+],COc1cn(-c2cccc(N3CCCCC3=O)c2F)nc(-c2ccnn2-c2ccccc2)c1=O,[Cu]I,C1COCCO1,[Cu]I,0.23
+19,ord-8cdfa484669a4173a9374bd739e1b02e,C1=CCCCC1.O=C(Oc1ccccc1[C@@H]1O[C@@]2(COCc3ccccc3)CO[C@@H]1[C@@]2(O)Cc1ccccc1)c1ccccc1,O=C(Oc1ccccc1[C@@H]1O[C@@]2(CO)CO[C@@H]1[C@@H]2O)c1ccccc1,[C].[OH-].[OH-].[Pd+2],CCO,[C].[OH-].[OH-].[Pd+2],0.85
+20,ord-a6fa804c926f474d9a4d3fb0992a554c,CCC(CC)(c1ccc(OCC(=O)C(C)(C)C)c(C)c1)c1ccc2sc(C(=O)O)cc2c1.COC(=O)CN.Cl.ClCCCl,CCC(CC)(c1ccc(OCC(=O)C(C)(C)C)c(C)c1)c1ccc2sc(C(=O)NCC(=O)O)cc2c1,CN(C)c1ccncc1,,CN(C)c1ccncc1,0.43
+21,ord-b28e4256e2644f609f32cb5a12358d19,C=C(C)C[C@]1(c2ccccc2)CCN([C@@H](C)c2ccc(Br)cc2)C(=O)N1.CC(C)(C)OO.CCO.Cc1ccc(S(=O)(=O)C#N)cc1.[SiH3]c1ccccc1,C[C@@H](c1ccc(Br)cc1)N1CC[C@](CC(C)(C)C#N)(c2ccccc2)NC1=O,[Co],,[Co],0.04
+22,ord-fed5b00f72254b7c8f95815d01fc8b4c,CC1(C)CC(=O)Oc2ccc(Br)cc21.Cc1ccccc1.[Na+].[OH-],C=C1CC(C)(C)c2cc(Br)ccc2O1,C[Al+]C.[CH3-].[Cl-].[Ti+3].c1cc[cH-]c1.c1cc[cH-]c1,C1CCOC1,C[Al+]C.[CH3-].[Cl-].[Ti+3].c1cc[cH-]c1.c1cc[cH-]c1,0.74
+23,ord-c514bb3cd598458da799ed29437f4961,CC1(C)O/C(=C2/C(=O)Nc3ccc(F)cc32)C=C1Br.CCOC(C)=O.O=C([O-])[O-].O=Cc1ccc(B(O)O)cc1.[K+].[K+],CC1(C)O/C(=C2/C(=O)Nc3ccc(F)cc32)C=C1c1ccc(C=O)cc1,[Pd].c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1,C1CCOC1.O,[Pd].c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1,0.52
+24,ord-97806c5415cd409eb317d696b2c1b264,C1CCOC1.O=C(Nc1cccc(Br)n1)C1(c2ccc3c(c2)OCO3)CC1.[Br-].[Zn+]CC1CCCCC1,O=C(Nc1cccc(CC2CCCCC2)n1)C1(c2ccc3c(c2)OCO3)CC1,Cl[Pd]Cl.[Fe+2].c1ccc(P(c2ccccc2)[c-]2cccc2)cc1.c1ccc(P(c2ccccc2)[c-]2cccc2)cc1,,Cl[Pd]Cl.[Fe+2].c1ccc(P(c2ccccc2)[c-]2cccc2)cc1.c1ccc(P(c2ccccc2)[c-]2cccc2)cc1,0.5
+25,ord-a569e66c25e6439d8b2d02d4f793c445,CCN(CC)CC.Cl.N#CC(=O)c1ccc(Cl)cc1Cl.O=C1CCCC(=O)C1,O=C1CCCC(=O)C1C(=O)c1ccc(Cl)cc1Cl,CC(=O)[O-].CC(=O)[O-].[Cl-].[Cl-].[Cu+2].[Zn+2],CCOCC.ClCCl,CC(=O)[O-].CC(=O)[O-].[Cl-].[Cl-].[Cu+2].[Zn+2],0.78
+26,ord-24ee85ae742841a4bfa0f009785a6cc5,O=C(c1ccc2[nH]c(C(=O)N3CCC(F)(F)CC3)cc2c1)N1CCC(N2CCOCC2)CC1.OB(O)c1cccc(Cl)c1.c1ccncc1,O=C(c1ccc2c(c1)cc(C(=O)N1CCC(F)(F)CC1)n2-c1cccc(Cl)c1)N1CCC(N2CCOCC2)CC1,CC(=O)[O-].CC(=O)[O-].[Cu+2],ClCCl,CC(=O)[O-].CC(=O)[O-].[Cu+2],0.64
+27,ord-a95d6e9e35984f7790a1a55dc6292199,COC(=O)c1nc(Br)c(F)cc1N.OB(O)c1ccccc1F,COC(=O)c1nc(-c2ccccc2F)c(F)cc1N,ClCCl.Cl[Pd]Cl.[Fe+2].c1ccc(P(c2ccccc2)[c-]2cccc2)cc1.c1ccc(P(c2ccccc2)[c-]2cccc2)cc1,,ClCCl.Cl[Pd]Cl.[Fe+2].c1ccc(P(c2ccccc2)[c-]2cccc2)cc1.c1ccc(P(c2ccccc2)[c-]2cccc2)cc1,0.99
+28,ord-b9347fbc2d1f439fb0b87974dc39c21e,CC1(C)c2cccc(P(c3ccccc3)c3ccccc3)c2Oc2c(P(c3ccccc3)c3ccccc3)cccc21.CCc1cc(N)ncn1.N#Cc1cc(Cl)c(-c2nc3ccnc(Br)c3s2)c(Cl)c1.O=C([O-])[O-].[Cs+].[Cs+],CCc1cc(Nc2nccc3nc(-c4c(Cl)cc(C#N)cc4Cl)sc23)ncn1,O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[Pd].[Pd],C1COCCO1,O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[Pd].[Pd],0.28
+29,ord-669abf374db4492fa86becc5fb208313,CCc1nc(C)cc(=O)n1CCOc1ccc([N+](=O)[O-])cc1.[H][H],CCc1nc(C)cc(=O)n1CCOc1ccc(N)cc1,[Pd],C1COCCO1,[Pd],0.7
+30,ord-79571893d7044aae86289e29886b41ff,C[C@@H]1CN(C(=O)OC(C)(C)C)[C@@H](C)CN1Cc1cc([N+](=O)[O-])cc2ccoc12.NN.O,C[C@@H]1CN(C(=O)OC(C)(C)C)[C@@H](C)CN1Cc1cc(N)cc2ccoc12,[Ni],C1CCOC1.CCO,[Ni],0.87
+31,ord-b53374957af946358cd3c222fe4ba0bb,CC1(C)OB(c2ccc3c(c2)CCCO3)OC1(C)C.CCO.CCOC(=O)C(=O)c1c(C)sc(C)c1Br.O=C([O-])[O-].[Na+].[Na+],Cc1sc(C)c(-c2ccc3c(c2)CCCO3)c1C(=O)C(=O)O,[Pd].c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1,Cc1ccccc1.O.O,[Pd].c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1,1.0
+32,ord-419341a196524ef49e44f08dda3c0e06,C=CCOc1ccc(COCCn2ccnn2)cc1.CN1C(=O)CC(=O)N(C)C1=O,Oc1ccc(COCCn2ccnn2)cc1,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,ClCCl.ClCCl,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,0.59
+33,ord-c156b60e2b0d45b3b88f357c0838a8f3,Brc1ccc2ncccc2c1.Cc1ccc2c(cnn2C2CCCCO2)c1B(O)O,Cc1ccc2c(cnn2C2CCCCO2)c1-c1ccc2ncccc2c1,CC(=O)[O-].CC(=O)[O-].CC(C)(C)P(C(C)(C)C)C(C)(C)C.O=C([O-])O.[Na+].[Pd+2],CO,CC(=O)[O-].CC(=O)[O-].CC(C)(C)P(C(C)(C)C)C(C)(C)C.[Pd+2],0.39
+34,ord-6a9fa314c63e49319d344313e3817ccb,Cc1ccc2c(cnn2C2CCCCO2)c1[B-](F)(F)F.O=S(=O)(Oc1ccc2ncccc2c1)C(F)(F)F.[K+],Cc1ccc2c(cnn2C2CCCCO2)c1-c1ccc2ncccc2c1,CC(=O)[O-].CC(=O)[O-].CC(C)(C)[O-].COc1cccc(OC)c1-c1ccccc1P(C1CCCCC1)C1CCCCC1.[Li+].[Pd+2],CCCCCC.CCc1cc(CC)cc(CC)c1.CCc1cccc(CC)c1.CN(C)C=O.CN(C)C=O.Cc1ccccc1.O,CC(=O)[O-].CC(=O)[O-].COc1cccc(OC)c1-c1ccccc1P(C1CCCCC1)C1CCCCC1.[Pd+2],0.11
+35,ord-1f77b54dfacd4b91acfea3bec7fc20e1,CC1(O)CCNCC1.CCN(CC)CC.C[C@H](OC(=O)C(Br)c1ccccc1)c1ccccc1,C[C@H](OC(=O)[C@@H](c1ccccc1)N1CCC(C)(O)CC1)c1ccccc1,CCCC[N+](CCCC)(CCCC)CCCC.[I-],C1CCOC1.C1CCOC1.CCOC(C)=O,CCCC[N+](CCCC)(CCCC)CCCC.[I-],0.6
+36,ord-ed45cb591b94415db1b32ac28a81a31d,COc1cc(C=O)c(F)cc1Cl.Cc1ccc(N)nc1.[C-]#[N+]C1CCCCC1,COc1cc(-c2nc3ccc(C)cn3c2NC2CCCCC2)c(F)cc1Cl,O=C(O)C(F)(F)F,CC(C)O.CC(C)O,O=C(O)C(F)(F)F,0.1
+37,ord-0092383681d845dcba430ac49717792e,CC(=O)[O-].CCOC(=O)c1cc(Cl)n2nccc2n1.[Na+],CCOC(=O)c1ccn2nccc2n1,[Pd],CCO.CCOC(C)=O,[Pd],0.78
+38,ord-bf17cc17249648a7a2825e5ddf7d6505,O=S(Cl)Cl.O=[N+]([O-])c1cc2c(O)ncnc2cc1F,O=[N+]([O-])c1cc2c(Cl)ncnc2cc1F,CN(C)C=O,,CN(C)C=O,0.94
+39,ord-de48fac1d2d04dec84902e698edeb682,CC[SiH](CC)CC.Cc1cc(C2=NCC(c3cc(Cl)cc(Cl)c3)(C(F)(F)F)C2)ccc1Br.Cc1cc(C2=NCC(c3cc(Cl)cc(Cl)c3)(C(F)(F)F)C2)ccc1Br.O=C([O-])[O-].[Na+].[Na+],Cc1cc(C2=NCC(c3cc(Cl)cc(Cl)c3)(C(F)(F)F)C2)ccc1C=O,Cl[Pd]Cl.[Fe+2].c1ccc(P(c2ccccc2)[c-]2cccc2)cc1.c1ccc(P(c2ccccc2)[c-]2cccc2)cc1,CN(C)C=O,Cl[Pd]Cl.[Fe+2].c1ccc(P(c2ccccc2)[c-]2cccc2)cc1.c1ccc(P(c2ccccc2)[c-]2cccc2)cc1,0.74
+40,ord-41d60ad3953644c4b04ae802bfc04e5a,C=CCO.CCN(CC)CC.O=[N+]([O-])c1ccc(Br)c(C(F)(F)F)c1,O=CCCc1ccc([N+](=O)[O-])cc1C(F)(F)F,CC(=O)[O-].CC(=O)[O-].CCCC[N+](CCCC)(CCCC)CCCC.[Cl-].[Pd+2],CN(C)C=O,CC(=O)[O-].CC(=O)[O-].CCCC[N+](CCCC)(CCCC)CCCC.[Cl-].[Pd+2],0.69
+41,ord-8afc8beb1f7840b8aba6c03887d2ef67,COc1ccc(-c2cnc(N)c(Cc3ccccc3)n2)cc1.COc1ccc2c(C(=O)Cl)cccc2c1.O,COc1ccc(-c2cnc(N(C(=O)c3cccc4cc(OC)ccc34)C(=O)c3cccc4cc(OC)ccc34)c(Cc3ccccc3)n2)cc1,CN(C)c1ccncc1,c1ccncc1,CN(C)c1ccncc1,0.68
+42,ord-a293a2555d6847cfb9346d2687aa4a50,C1CCOC1.CCN(CC)CC.COc1cc(C(=O)Nc2nc3c(OC)ccc(C4=CCN(C(=O)OC(C)(C)C)CC4)c3s2)cc(Cl)n1,COc1cc(C(=O)Nc2nc3c(OC)ccc(C4CCN(C(=O)OC(C)(C)C)CC4)c3s2)ccn1,[Pd],CO,[Pd],0.3
+43,ord-56e75ad899ba4adea278567a03433405,Brc1ccc2ncccc2c1.Cc1ccc2c(cnn2C2CCCCO2)c1[B-](F)(F)F.[K+],Cc1ccc2c(cnn2C2CCCCO2)c1-c1ccc2ncccc2c1,CC(=O)[O-].CC(=O)[O-].[Na+].[OH-].[Pd+2],CCc1cc(CC)cc(CC)c1.CCc1cccc(CC)c1.CN(C)C=O.CN(C)C=O.Cc1ccccc1.O.O,CC(=O)[O-].CC(=O)[O-].[Pd+2],0.19
+44,ord-421f24003cb04881a3e8f8edef479566,CC(=O)[Cu]C(C)=O.CCN(CC)CC.COC(=O)c1c(C)[nH]c2ccccc12.OB(O)c1ccccc1,COC(=O)c1c(C)n(-c2ccccc2)c2ccccc12,CN(C)c1ccncc1,ClCCl,CN(C)c1ccncc1,0.39
+45,ord-f97af96ff3dc4293996dd87a9db42bb9,CC(C)c1nn(Cc2ccccc2Br)c(=O)c(C(=O)NCC(=O)O)c1O.Cl.O=C([O-])[O-].OB(O)c1ccc(C(F)(F)F)cc1.[K+].[K+],CC(C)c1nn(Cc2ccccc2-c2ccc(C(F)(F)F)cc2)c(=O)c(C(=O)NCC(=O)O)c1O,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,C1COCCO1.O.O,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,0.46
+46,ord-952ab56552d14163b81797c4be7275fa,CC(C)(C)OC(=O)N1CCC(C)(C)c2ccc([N+](=O)[O-])cc21,CC(C)(C)OC(=O)N1CCC(C)(C)c2ccc(N)cc21,[Pd],CO,[Pd],0.95
+47,ord-804093bffa5d43049c093723ec65aa6e,Cc1ccc(N)cc1.FC(F)(F)c1ccc(Cl)cc1,Cc1ccc(Nc2ccc(C(F)(F)F)cc2)cc1,CC(C)c1cc(C(C)C)c(-c2ccccc2P(C2CCCCC2)(C2CCCCC2)->[Pd]2(<-Nc3ccccc3-c3ccccc32)OS(=O)(=O)C(F)(F)F)c(C(C)C)c1.CN1CCCN2CCCN=C12.c1ccc2oncc2c1,CS(C)=O.CS(C)=O.CS(C)=O.CS(C)=O.CS(C)=O,CC(C)c1cc(C(C)C)c(-c2ccccc2P(C2CCCCC2)(C2CCCCC2)->[Pd]2(<-Nc3ccccc3-c3ccccc32)OS(=O)(=O)C(F)(F)F)c(C(C)C)c1,0.19
+48,ord-3292ea938a434474adb4a63169f04be8,CCCC(=NOCC)C1=C(O)CC(c2c(C)c(C)c(OCc3ccccc3)c(C)c2C)CC1=O,CCCC(=NOCC)C1=C(O)CC(c2c(C)c(C)c(O)c(C)c2C)CC1=O,Cl.[Pd],CCOC(C)=O,Cl.[Pd],0.3
+49,ord-45f62759dcd5421bb66c8fd1b768bbe7,FC1(F)C(F)(F)C(F)(F)C(F)(I)C(F)(F)C1(F)F.Nc1ccccc1.O=C([O-])O.O=S([O-])S(=O)[O-].[Na+].[Na+].[Na+],Nc1ccc(C2(F)C(F)(F)C(F)(F)C(F)(F)C(F)(F)C2(F)F)cc1,CCCC[N+](CCCC)(CCCC)CCCC.O=S(=O)([O-])O,COC(C)(C)C.O,CCCC[N+](CCCC)(CCCC)CCCC.O=S(=O)([O-])O,0.68
+50,ord-7030178ac7694e1eace7144b87cfafa0,CCOC(=O)[C@H](C)Nc1ncccc1C#N.C[O-].Cl.[Na+],C[C@@H]1Nc2ncccc2CNC1=O,[Ni],CO,[Ni],0.23
+51,ord-50297983814043f68e5ed0382c170795,O=C1NC(=O)c2c1c(-c1ccccc1)cc1[nH]c3ccc(OP(=O)(OCc4ccccc4)OCc4ccccc4)cc3c21,O=C1NC(=O)c2c1c(-c1ccccc1)cc1[nH]c3ccc(OP(=O)(O)O)cc3c21,[Pd],C1CCOC1.CO,[Pd],0.71
+52,ord-72b281f722d64a2a86366b024fffcf4e,CCOCC.COC(=O)C(C)(SC)c1cccs1,COC(=O)C(C)c1cccs1,O=S(=O)([O-])[O-].[Cu+2].[Zn],CC(=O)O,O=S(=O)([O-])[O-].[Cu+2].[Zn],0.87
+53,ord-09fd3b562efe4258b5377913c5a128a3,CC(C)(C)C(=O)O.CCCCP(C12CC3CC(CC(C3)C1)C2)C12CC3CC(CC(C3)C1)C2.COC(=O)c1ccc2c(c1)CCCC2(O)c1nccs1.COc1ccnc(Nc2cc(C)cc(Br)n2)c1.[Cs+].[F-],COC(=O)c1ccc2c(c1)CCCC2(O)c1ncc(-c2cc(C)cc(Nc3cc(OC)ccn3)n2)s1,CC(=O)[O-].CC(=O)[O-].[Pd+2],C1COCCO1.C1COCCO1,CC(=O)[O-].CC(=O)[O-].[Pd+2],0.52
+54,ord-5a891173b2f14da481ad4b70dab850c7,C=CC(=O)NC1CCN(S(=O)(=O)c2ccc([N+](=O)[O-])cc2)CC1.CCO.[Cl-].[NH4+],C=CC(=O)NC1CCN(S(=O)(=O)c2ccc(N)cc2)CC1,[Fe],O,[Fe],0.51
+55,ord-947db249f38c44959a558af280c17a6a,N#Cc1cc(B(O)O)ccc1F.O.O=C([O-])[O-].O=C1C(Cc2c(Cl)cc(OS(=O)(=O)C(F)(F)F)cc2Cl)CCN1C1CCCCC1.[Na+].[Na+],N#Cc1cc(-c2cc(Cl)c(CC3CCN(C4CCCCC4)C3=O)c(Cl)c2)ccc1F,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,C1CCOC1,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,0.88
+56,ord-a5d240b1afd440d68f607db269eb8d83,CCN=C=NCCCN(C)C.COc1cccc(S(=O)(=O)N(CCCN(C)C)[C@@H](C(=O)OCc2ccccc2)C(C)(C)C)c1.Cl.Cl.NOC1CCCCO1.O.O.O.O.O.O.O.O.O.O.O=C([O-])O.On1nnc2cccnc21.[Na+],COc1cccc(S(=O)(=O)N(CCCN(C)C)[C@@H](C(=O)NO)C(C)(C)C)c1,[Pd],CN(C)C=O.CN(C)C=O.CO.CO,[Pd],1.0
+57,ord-ea572bd72b7c41448346a54c8f0ce405,COc1ccc(Cl)cc1.Cc1cc(C)c(B(O)O)c(C)c1,COc1ccc(-c2c(C)cc(C)cc2C)cc1,C1=C\CC/C=C\CC/1.C1=C\CC/C=C\CC/1.CP(C)C.O.O=P([O-])([O-])[O-].[K+].[K+].[K+].[Ni],C1COCCO1.C1COCCO1.C1COCCO1.C1COCCO1,C1=C\CC/C=C\CC/1.C1=C\CC/C=C\CC/1.CP(C)C.[Ni],0.0
+58,ord-4f2876eb72624720885d110d9afe8876,O=C(O)Cc1cccc(F)c1[N+](=O)[O-],O=C1Cc2cccc(F)c2N1,[Pd],CC(=O)O,[Pd],0.83
+59,ord-f70beae1ac0f46688a2b14e378fea5de,CC(C)(C)[O-].CC(C)(C)[Si](C)(C)OCC(OS(C)(=O)=O)c1ccc(Cl)c(F)c1.CSc1nccc(-c2cc[nH]c(=O)n2)n1.[K+],CSc1nccc(-c2ccn(C(CO[Si](C)(C)C(C)(C)C)c3ccc(Cl)c(F)c3)c(=O)n2)n1,CCCC[N+](CCCC)(CCCC)CCCC.[I-],C1CCOC1.C1CCOC1,CCCC[N+](CCCC)(CCCC)CCCC.[I-],0.33
+60,ord-dd4fc6ef7a154ea990b298e4c0e473d8,C[C@H]1CN(C(=O)OC(C)(C)C)CCN1c1ccc(N)nc1.Cn1nc(Cl)cc(Br)c1=O.O=C([O-])[O-].[Cs+].[Cs+],C[C@H]1CN(C(=O)OC(C)(C)C)CCN1c1ccc(Nc2cc(Cl)nn(C)c2=O)nc1,CC1(C)c2cccc(P(c3ccccc3)c3ccccc3)c2Oc2c(P(c3ccccc3)c3ccccc3)cccc21.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[Pd].[Pd],C1COCCO1,CC1(C)c2cccc(P(c3ccccc3)c3ccccc3)c2Oc2c(P(c3ccccc3)c3ccccc3)cccc21.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[Pd].[Pd],0.86
+61,ord-c01ebf59211542e7b8b22c41350e1260,Cc1ccc(C2CN(C)Cc3cc(B4OC(C)(C)C(C)(C)O4)ccc32)cc1.Cc1ccc(Cl)nn1.O=C([O-])[O-].[Cs+].[Cs+],Cc1ccc(C2CN(C)Cc3cc(-c4ccc(C)nn4)ccc32)cc1,Cl[Pd]Cl.[Fe+2].c1ccc(P(c2ccccc2)[c-]2cccc2)cc1.c1ccc(P(c2ccccc2)[c-]2cccc2)cc1,CN(C)C=O.O,Cl[Pd]Cl.[Fe+2].c1ccc(P(c2ccccc2)[c-]2cccc2)cc1.c1ccc(P(c2ccccc2)[c-]2cccc2)cc1,0.21
+62,ord-ad61602b94f64d8495ee716a39e6d2a1,CCCCCCc1csc2c(CCCCCC)c(C(=O)O)sc12.O=C=O.c1ccc2ncccc2c1,CCCCCCc1csc2c(CCCCCC)csc12,[Cu],CCCCCC,[Cu],0.68
+63,ord-9c5bfa45fb954b7c9fcf1a95e13440f3,CC(C)(C)[Si](C)(C)Cl.COc1ccc(F)c(-c2ccc(CO)cc2C(O)C(C)(C)C)c1,COc1ccc(F)c(-c2ccc(CO[Si](C)(C)C(C)(C)C)cc2C(O)C(C)(C)C)c1,CN(C)c1ccncc1,ClCCl,CN(C)c1ccncc1,0.96
+64,ord-490528e794064585840f9e8171a1c4ff,C1=COCCC1.Cc1cc(OC2CCCCO2)cc(CO)c1Br,Cc1cc(OC2CCCCO2)cc(COC2CCCCO2)c1Br,CC1(C)C2CCC1(CS(=O)(=O)O)C(=O)C2,ClCCl,CC1(C)C2CCC1(CS(=O)(=O)O)C(=O)C2,0.9
+65,ord-42093bc147ea48df87675a44b529ea9e,COC(=O)/C=C/c1ccc(N(Cc2ccccc2)Cc2ccccc2)nc1C(=O)OC.[BH4-].[Na+],COC(=O)CCc1ccc(N(Cc2ccccc2)Cc2ccccc2)nc1C(=O)OC,Cl[Ni]Cl.O.O.O.O.O.O,CO.[Cl-].[NH4+],Cl[Ni]Cl.O.O.O.O.O.O,0.86
+66,ord-8b9ed594e1bb490e8e63adf998a912d0,CC(C)(C)OC(=O)N1CC[C@H](O)[C@H]1CO.CC(C)(C)[Si](C)(C)Cl.CCN(CC)CC,CC(C)(C)OC(=O)N1CC[C@H](O)[C@H]1CO[Si](C)(C)C(C)(C)C,CN(C)c1ccncc1,ClCCl.ClCCl,CN(C)c1ccncc1,0.99
+67,ord-20adf5fd2c4745b282d3719f1519f23a,Brc1ccc2ncccc2c1.Cc1ccc2c(cnn2C2CCCCO2)c1[B-](F)(F)F.[K+],Cc1ccc2c(cnn2C2CCCCO2)c1-c1ccc2ncccc2c1,CC(=O)[O-].CC(=O)[O-].CN(C)c1ccc(P(C(C)(C)C)C(C)(C)C)cc1.CN(C)c1ccc(P(C(C)(C)C)C(C)(C)C)cc1.Cl[Pd]Cl.O=C([O-])O.[Na+].[Pd+2],C1CCOC1.CCc1cc(CC)cc(CC)c1.CCc1cccc(CC)c1.CN(C)C=O.Cc1ccccc1.O.O,CC(=O)[O-].CC(=O)[O-].CN(C)c1ccc(P(C(C)(C)C)C(C)(C)C)cc1.CN(C)c1ccc(P(C(C)(C)C)C(C)(C)C)cc1.Cl[Pd]Cl.[Pd+2],0.21
+68,ord-a84ce0c2384f46d9a1b1262447ac2d9e,CCOC(=O)CC(=O)OCC.COC1C=CC(OC)O1.O,CCOC(=O)C(C(=O)OCC)c1ccco1,[Cl-].[Cl-].[Zn+2],CC(=O)O,[Cl-].[Cl-].[Zn+2],0.29
+69,ord-74094cf1745040d383c9f9e1048d6de7,Brc1ccc(-c2ccc3ccc4cccc5ccc2c3c45)cc1.CC1(C)C=C(B2OC(C)(C)C(C)(C)O2)C=C2C=c3cc4c5ccccc5c5ccccc5c4cc3=C21.CCO.O=C([O-])[O-].[Na+].[Na+],CC1(C)C=C(c2ccc(-c3ccc4ccc5cccc6ccc3c4c56)cc2)C=C2C=c3cc4c5ccccc5c5ccccc5c4cc3=C21,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,CO.Cc1ccccc1,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,0.57
+70,ord-e2a772144e774571bc6c80d27ec8d125,C1CCOC1.C=CCO[C@H]1O[C@H](COCc2ccccc2)[C@@H](O[C@@H]2O[C@H](CF)[C@@H](OCc3ccccc3)[C@H](OCc3ccccc3)[C@H]2OCc2ccccc2)[C@H](OCc2ccccc2)[C@H]1OCc1ccccc1,C=CCOC1O[C@H](COCc2ccccc2)[C@@H](O[C@@H]2O[C@H](CF)[C@@H](OCc3ccccc3)[C@H](OCc3ccccc3)[C@H]2OCc2ccccc2)[C@H](OCc2ccccc2)[C@H]1OCc1ccccc1,Cl[Pd]Cl,CO,Cl[Pd]Cl,0.63
+71,ord-38e4779d915e47c48c745aea7ba79a76,CCCc1c(Cc2ccc(-c3ccccc3C#N)cc2F)c(=O)n([C@H]2CC[C@H](O)CC2)c2ncnn12.CCOC(=O)C=[N+]=[N-].Cc1ccccc1,CCCc1c(Cc2ccc(-c3ccccc3C#N)cc2F)c(=O)n([C@H]2CC[C@H](OCC(C)(C)O)CC2)c2ncnn12,CC(=O)[O-].[Rh+],,CC(=O)[O-].[Rh+],0.22
+72,ord-3d6ff4ad7dda45e294602d21def90e23,CC(=O)O.CCC=O.COC(=O)CC(=O)CCl,CCC=C(C(=O)CCl)C(=O)OC,C1CCNCC1,ClCCl.ClCCl,C1CCNCC1,0.92
+73,ord-85018eaedb81478f8d3570187284f89c,CCOP(=O)(OCC)[C@@H]1SC[C@@H](CO[Si](C)(C)C(C)(C)C)S1,CCOP(=O)(OCC)[C@@H]1SC[C@@H](CO)S1,CC(=O)Cl.[Cl-].[NH4+],CO,CC(=O)Cl.[Cl-].[NH4+],0.86
+74,ord-1489de592ed543aead9b9b3cc9eda428,CC(C)(C)OC(=O)OC(=O)OC(C)(C)C.CCOC(=O)c1sc(N)nc1C(F)(F)F,CCOC(=O)c1sc(NOC(=O)OC(C)(C)C)nc1C(F)(F)F,CN(C)c1ccncc1,ClCCl,CN(C)c1ccncc1,0.92
+75,ord-cb7e6f31ffa444cfad91d75ff54f1c8b,CC(C)(C)OC(=O)N1C[C@@H](COc2c(F)cccc2[N+](=O)[O-])OC[C@H]1CO[Si](C)(C)C(C)(C)C.[H][H],CC(C)(C)OC(=O)N1C[C@@H](COc2c(N)cccc2F)OC[C@H]1CO[Si](C)(C)C(C)(C)C,[Pd],CCOC(C)=O,[Pd],1.0
+76,ord-2686b5fe071e4571b88efe2f81c589cc,CC(C)(C)OC(=O)OC(=O)[O-].[N-]=[N+]=NC(=O)N=[N+]=[N-],CC(C)(C)OC(=O)C(N)=O,[OH-].[OH-].[Pd+2],CO,[OH-].[OH-].[Pd+2],0.55
+77,ord-c1d83ba8cfd3462c853244d9e87f8ffa,CC(C)n1nc(I)c2c(N)ncnc21.COc1ccc(B(O)O)cc1.O=C([O-])[O-].[Na+].[Na+],COc1ccc(-c2nn(C(C)C)c3ncnc(N)c23)cc1,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,CCO.COCCOC,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,0.16
+78,ord-dd35d66336ce4bc5b4c1d0e068dfa168,O=C(OCc1ccccc1)N1CCC2(CC1)C(=O)N(Cc1cccc([N+](=O)[O-])c1)CN2c1ccccc1.[Cl-].[NH4+],Nc1cccc(CN2CN(c3ccccc3)C3(CCN(C(=O)OCc4ccccc4)CC3)C2=O)c1,[Fe],CCO.O,[Fe],0.95
+79,ord-b7d4ad900fd64b83ac3ac7883e5a67fb,Nc1c(-c2ccccn2)cccc1[N+](=O)[O-],Nc1cccc(-c2ccccn2)c1N,[Pd],CCOC(C)=O,[Pd],0.89
+80,ord-992d3e706540481abd31c86e7f5a30a9,Cc1cc(Br)sc1CO.O=C([O-])[O-].OB(O)c1ccc(C(F)(F)F)cc1.[K+].[K+],Cc1cc(-c2ccc(C(F)(F)F)cc2)sc1CO,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,Cc1ccccc1,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,0.48
+81,ord-9df5ea16a6b54f7287d0342b6c69c662,Cc1ccncc1N1CCNC1=O.Cn1cc(Br)c2c(C#N)cccc21.N[C@@H]1CCCC[C@H]1N.O=C([O-])[O-].[K+].[K+],Cc1ccncc1N1CCN(c2cn(C)c3cccc(C#N)c23)C1=O,I[Cu]I,C1COCCO1,I[Cu]I,0.24
+82,ord-f64a42b092c94884876b2d8c6edce61d,CN1CC=C(c2ccc(N)nn2)CC1,CN1CCC(c2ccc(N)nn2)CC1,[Pd],CCO,[Pd],0.89
+83,ord-0d04174b23454275a39535f812e1524d,CC1(C)OBOC1(C)C.CCN(CC)CC.CN(C)c1ccc(-c2cnc3c(c2)c(I)cn3S(=O)(=O)c2ccccc2)cc1.ClCCl,CN(C)c1ccc(-c2cnc3c(c2)c(B2OC(C)(C)C(C)(C)O2)cn3S(=O)(=O)c2ccccc2)cc1,Cl[Pd]Cl.[Fe+2].c1ccc(P(c2ccccc2)[c-]2cccc2)cc1.c1ccc(P(c2ccccc2)[c-]2cccc2)cc1,C1COCCO1,Cl[Pd]Cl.[Fe+2].c1ccc(P(c2ccccc2)[c-]2cccc2)cc1.c1ccc(P(c2ccccc2)[c-]2cccc2)cc1,1.0
+84,ord-ad46416b40594ec5bdb903ea60cf5bf4,CCCCCO.Nc1ccccc1C(=O)O.O=C([O-])[O-].O=[N+]([O-])c1cc(Cl)ccc1Br.[K+].[K+],O=C(O)c1ccccc1Nc1ccc(Cl)cc1[N+](=O)[O-],[Cu],,[Cu],0.83
+85,ord-dfe7f7782502427c94741fe658e539f4,CC1(C)c2cccc(P(c3ccccc3)c3ccccc3)c2Oc2c(P(c3ccccc3)c3ccccc3)cccc21.CN(C)CCN.O=C([O-])[O-].O=[N+]([O-])c1cc(I)c2occc2c1.[Cs+].[Cs+],CN(C)CCNc1cc([N+](=O)[O-])cc2ccoc12,O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[Pd].[Pd],Cc1ccccc1C,O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[Pd].[Pd],0.44
+86,ord-b62d233adb304f30b5a36ca35fcd7b87,CC(=O)OC(c1cncc(Br)c1)C(F)(F)F.CS(=O)[O-].O=C(O)[C@@H]1CCCN1.O=C([O-])O.O=C([O-])[O-].[K+].[K+].[Na+].[Na+],CS(=O)(=O)c1cncc(C(O)C(F)(F)F)c1,I[Cu]I,CS(C)=O.O,I[Cu]I,0.17
+87,ord-e53b2ec8d81d4a98b6f257b0907e0246,CC(C)(C)[Si](C)(C)Oc1cc(C#N)ccc1NC(=S)Nc1ccccc1Br.CCN(CC)CC.CS(=O)(=O)Cl,CC(C)(C)[Si](C)(C)Oc1cc(C#N)ccc1N=C=Nc1ccccc1Br,CN(C)c1ccncc1,ClCCl,CN(C)c1ccncc1,1.0
+88,ord-097147ff6e314b5fba919dbc677dfb5a,COc1ccc(B(O)O)cn1.FC(F)(F)c1cccnc1Cl.O=C([O-])[O-].[K+].[K+],COc1ccc(-c2ncccc2C(F)(F)F)cn1,[Pd].c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1,Cc1ccccc1,[Pd].c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1,0.95
+89,ord-9a7a82dd9a9145ada8a907e3038be509,C1COCCO1.CO/C=C(/I)C(=O)OC.COCCOc1ccc(OCc2cc(Cl)ccc2B(O)O)cc1.O=P([O-])([O-])[O-].[K+].[K+].[K+],CO/C=C(/C(=O)OC)c1ccc(Cl)cc1COc1ccc(OCCOC)cc1,[Pd].c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1,CCOC(C)=O.O,[Pd].c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1.c1ccc(P(c2ccccc2)c2ccccc2)cc1,0.71
+90,ord-9f4ef94bf15a4ee19e832797fcbfc276,Brc1ccccc1-c1ccccc1.FC(F)n1ccnc1-c1ccccc1,FC(F)n1c(-c2ccccc2-c2ccccc2)cnc1-c1ccccc1,C=CC[Pd]Cl.C=CC[Pd]Cl.CC(C)(C)C(=O)[O-].CCCCP(CCCC)CCCC.[K+],CC(=O)N(C)C.CC(=O)N(C)C.CC(=O)N(C)C,C=CC[Pd]Cl.C=CC[Pd]Cl.CCCCP(CCCC)CCCC,0.06
+91,ord-3fc705ee410b4deab1cf90242dd2b028,O=C(O)Cc1ccc(-c2ccccc2)cc1[N+](=O)[O-],O=C1Cc2ccc(-c3ccccc3)cc2N1,[Fe],CC(=O)O,[Fe],0.93
+92,ord-e24b7b3e53074286a14395f6e069f883,O=[N+]([O-])c1nc[nH]n1.OB(O)c1cccc(C(F)(F)F)c1.c1ccncc1,O=[N+]([O-])c1ncn(-c2cccc(C(F)(F)F)c2)n1,CC(=O)[O-].CC(=O)[O-].[Cu+2],ClCCl,CC(=O)[O-].CC(=O)[O-].[Cu+2],0.49
+93,ord-16b2582c09fe4682a6b4b6a8ce6f6f3b,C=C(C)CN(C(C)=O)c1cc([N+](=O)[O-])ccc1Br.CC(=O)[O-].O=C[O-].[Na+].[Na+],CC(=O)N1CC(C)(C)c2ccc([N+](=O)[O-])cc21,CC(=O)[O-].CC(=O)[O-].CC[N+](CC)(CC)CC.O.[Cl-].[Pd+2],CN(C)C=O,CC(=O)[O-].CC(=O)[O-].CC[N+](CC)(CC)CC.O.[Cl-].[Pd+2],0.88
+94,ord-9385291980c14709b3314e8c4eac0620,CCN(C(C)C)C(C)C.CCN=C=NCCCN(C)C.Cl.O=C(O)/C=C/c1cnc2c(c1)CCC(=O)N2.O=C(O)C(F)(F)F.c1ccc2oc([C@H]3CCCN3)nc2c1,O=C1CCc2cc(/C=C/C(=O)N3CCCCC3c3nc4ccccc4o3)cnc2N1,CN(C)c1ccncc1,CN(C)C=O,CN(C)c1ccncc1,0.13
+95,ord-6114c9b5157c4a06afa58a4a0bff2316,CN(C)CC(=O)O.COc1cccc(F)c1O.Cl.Fc1ccc(I)cc1.O=C([O-])[O-].[Cs+].[Cs+],COc1cccc(F)c1Oc1ccc(F)cc1,[Cu],C1COCCO1,[Cu],0.21
+96,ord-3351aa586f3a40cc8b5aa2c4f0d2a2eb,Brc1ccc2nc(N3CCN(C4CC4)CC3)sc2c1.CC(N)=O.CC1(C)c2cccc(P(c3ccccc3)c3ccccc3)c2Oc2c(P(c3ccccc3)c3ccccc3)cccc21.O=C([O-])[O-].[Cs+].[Cs+],CC(=O)Nc1ccc2nc(N3CCN(C4CC4)CC3)sc2c1,CC(=O)[O-].CC(=O)[O-].[Pd+2],C1COCCO1.O,CC(=O)[O-].CC(=O)[O-].[Pd+2],0.24
+97,ord-5b335e29ecd2454d93b69f994db21e79,CC(C)(Cc1cnc2c(Br)cccn12)[N+](=O)[O-].O=C([O-])[O-].OB(O)c1cccs1.[Na+].[Na+],CC(C)(Cc1cnc2c(-c3cccs3)cccn12)[N+](=O)[O-],c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,C1COCCO1,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,1.0
+98,ord-a1b4e2bb9232496f86833446bc1afce1,Brc1cccnc1.CC1(C)CN(c2ccc3ncsc3c2)C(=O)N1.N[C@@H]1CCCC[C@H]1N.O=P([O-])([O-])[O-].[K+].[K+].[K+],CC1(C)CN(c2ccc3ncsc3c2)C(=O)N1c1cccnc1,I[Cu]I,C1COCCO1,I[Cu]I,0.11
+99,ord-ef50fb1edb404225bc2416be0fd42b65,O=C(NC1CN2CCC1CC2)c1cccc2oc(-c3ccc(I)cc3)nc12.O=C([O-])[O-].OB(O)c1ccccc1.[Na+].[Na+],O=C(NC1CN2CCC1CC2)c1cccc2oc(-c3ccc(-c4ccccc4)cc3)nc12,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,Cc1ccccc1,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,0.78
+100,ord-d685ae2873e64b72a0784c5ddba6999a,C1=CCCCC1.CC(Cc1cccc(C(=O)OCc2ccccc2)c1O)C[Si](C)(O[Si](C)(C)C)O[Si](C)(C)C,CC(Cc1cccc(C(=O)O)c1O)C[Si](C)(O[Si](C)(C)C)O[Si](C)(C)C,[Pd],CCO,[Pd],0.78
+101,ord-4c25b497c6ec4d309c397d2507c8f033,CC(C)(C)OC(=O)N1CCNCC1.CC(C)(C)[O-].CC(C)c1cc(C(C)C)c(-c2ccccc2P(C2CCCCC2)C2CCCCC2)c(C(C)C)c1.CC1C(=O)N(COCC[Si](C)(C)C)N=C2COc3ccc(Br)cc3N21.[Na+],CC1C(=O)N(COCC[Si](C)(C)C)N=C2COc3ccc(N4CCN(C(=O)OC(C)(C)C)CC4)cc3N21,CC(=O)O[Pd]OC(C)=O,Cc1ccccc1,CC(=O)O[Pd]OC(C)=O,0.33
+102,ord-f94786e6a22c4c27900d6b1cbb159b5e,CC(C)c1cc(C(C)C)c(S(=O)(=O)Cl)c(C(C)C)c1.CCN(CC)CC.COc1ccc([N+](=O)[O-])c([C@@H](OCc2cn([C@H]3C[C@@](O)([Si](C)(C)C(C)(C)C)[C@@H](CO[Si](C)(C)C(C)(C)C)O3)c(=O)[nH]c2=O)C(C)(C)C)c1,COc1ccc([N+](=O)[O-])c([C@@H](OCc2cn([C@H]3C[C@@](O)([Si](C)(C)C(C)(C)C)[C@@H](CO[Si](C)(C)C(C)(C)C)O3)c(=O)nc2N)C(C)(C)C)c1,CN(C)c1ccncc1,ClCCl,CN(C)c1ccncc1,0.65
+103,ord-7a092c57e7834c9a9cddcb7260af09a9,CC1(C)c2cccc(P(c3ccccc3)c3ccccc3)c2Oc2c(P(c3ccccc3)c3ccccc3)cccc21.CCS(=O)(=O)c1ccc(N)nc1.Cn1nc(Cl)cc(Br)c1=O.O=C([O-])[O-].[Cs+].[Cs+],CCS(=O)(=O)c1ccc(Nc2cc(Cl)nn(C)c2=O)nc1,O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[Pd].[Pd],C1COCCO1.ClCCl,O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[Pd].[Pd],0.25
+104,ord-7ba08394993f4086bd0ff9b487d332b6,C[C@@H]1CCc2nc(S)nc(O)c21.[NH4+].[OH-],C[C@@H]1CCc2ncnc(O)c21,[Ni],O,[Ni],0.99
+105,ord-37e1a8f0a8ab45e294a38778ad8848be,CC(=O)N(C)C.O=C1CCCCN1c1ccc(N2CCc3c(C(F)(F)F)nn(-c4ccc(F)c(Cl)c4)c3C2=O)cc1,N#Cc1cc(-n2nc(C(F)(F)F)c3c2C(=O)N(c2ccc(N4CCCCC4=O)cc2)CC3)ccc1F,O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[C-]#N.[C-]#N.[Fe+2].[Pd].[Pd].[Zn+2].[Zn].c1ccc(P(c2ccccc2)[c-]2cccc2)cc1.c1ccc(P(c2ccccc2)[c-]2cccc2)cc1,,O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[C-]#N.[C-]#N.[Fe+2].[Pd].[Pd].[Zn+2].[Zn].c1ccc(P(c2ccccc2)[c-]2cccc2)cc1.c1ccc(P(c2ccccc2)[c-]2cccc2)cc1,0.5
+106,ord-d5751a6de9ce4bea980568f8bdfecde6,CC(=O)Cl.Cc1c(Br)c(F)c(O)c(N)c1C#N.[Cl-].[NH4+],Cc1nc2c(C#N)c(C)c(Br)c(F)c2o1,CCN(C(C)C)C(C)C,CCOC(C)=O,CCN(C(C)C)C(C)C,0.73
+107,ord-3cac9ca75bf34bae9d8605b4ace7a1c5,CC(C)(C)[O-].CC(C)N1CCNCC1.COc1cc(Br)cc(C2OCCCO2)c1.Cl.[Na+].[Na+].[OH-].c1ccc(P(c2ccccc2)c2ccc3ccccc3c2-c2c(P(c3ccccc3)c3ccccc3)ccc3ccccc23)cc1,COc1cc(C=O)cc(N2CCN(C(C)C)CC2)c1,O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[Pd].[Pd],Cc1ccccc1,O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[Pd].[Pd],0.27
+108,ord-b146ca9198c348cca63bcac068757137,COCOc1cc(OC)ccc1I.O=C1CCC(=O)N1Br,COCOc1cc(OC)c(Br)cc1I,Cc1c(C(C)(C)C)cc(O)cc1C(C)(C)C,CC#N,Cc1c(C(C)(C)C)cc(O)cc1C(C)(C)C,0.76
+109,ord-50c61014210e4234b91331351ce47941,CCOCOCC.OCC(Cl)CCl,CCOCOCC(Cl)CCl,Cc1ccccc1S(=O)(=O)O.O,,Cc1ccccc1S(=O)(=O)O.O,0.69
+110,ord-f252f2d4246c4459989daae36b6c6aae,CC1(C)OB(c2ccnc(N3CCOCC3)c2)OC1(C)C.Cc1nc(NC(=O)NC(=O)C(C)C)ccc1Oc1ccnc(Cl)c1.O=C([O-])[O-].[K+].[K+],Cc1nc(NC(=O)NC(=O)C(C)C)ccc1Oc1ccnc(-c2ccnc(N3CCOCC3)c2)c1,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,C1COCCO1.O,c1ccc([P](c2ccccc2)(c2ccccc2)[Pd]([P](c2ccccc2)(c2ccccc2)c2ccccc2)([P](c2ccccc2)(c2ccccc2)c2ccccc2)[P](c2ccccc2)(c2ccccc2)c2ccccc2)cc1,0.28
+111,ord-e3950b01d3ad49e3a039c16486e0eb29,CC(C)(C)[O-].NC1CCOCC1.O=C(NCC(O)CN1CCc2ccccc2C1)c1cncc(Br)c1.[Na+].c1ccc(P(c2ccccc2)c2ccc3ccccc3c2-c2c(P(c3ccccc3)c3ccccc3)ccc3ccccc23)cc1,O=C(NCC(O)CN1CCc2ccccc2C1)c1cncc(NC2CCOCC2)c1,O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[Pd].[Pd],C1COCCO1,O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.O=C(/C=C/c1ccccc1)/C=C/c1ccccc1.[Pd].[Pd],0.24

generation_utils.py

@@ -0,0 +1,54 @@
+import pandas as pd
+import torch
+from torch.utils.data import Dataset
+
+
+def prepare_input(cfg, text):
+    inputs = cfg.tokenizer(
+        text,
+        add_special_tokens=True,
+        max_length=cfg.input_max_length,
+        padding="max_length",
+        truncation=True,
+        return_attention_mask=True,
+    )
+    return {k: torch.tensor(v, dtype=torch.long) for k, v in inputs.items()}
+
+
+class ReactionT5Dataset(Dataset):
+    def __init__(self, cfg, df):
+        self.cfg = cfg
+        self.inputs = df["input"].values
+
+    def __len__(self):
+        return len(self.inputs)
+
+    def __getitem__(self, idx):
+        return prepare_input(self.cfg, self.inputs[idx])
+
+
+def decode_output(output, cfg):
+    sequences = [
+        cfg.tokenizer.decode(seq, skip_special_tokens=True).replace(" ", "").rstrip(".")
+        for seq in output["sequences"]
+    ]
+    if cfg.num_beams > 1:
+        scores = output["sequences_scores"].tolist()
+        return sequences, scores
+    return sequences, None
+
+
+def save_multiple_predictions(input_data, sequences, scores, cfg):
+    output_list = [
+        [input_data.loc[i // cfg.num_return_sequences, "input"]]
+        + sequences[i : i + cfg.num_return_sequences]
+        + scores[i : i + cfg.num_return_sequences]
+        for i in range(0, len(sequences), cfg.num_return_sequences)
+    ]
+    columns = (
+        ["input"]
+        + [f"{i}th" for i in range(cfg.num_return_sequences)]
+        + ([f"{i}th score" for i in range(cfg.num_return_sequences)] if scores else [])
+    )
+    output_df = pd.DataFrame(output_list, columns=columns)
+    return output_df

models.py

@@ -0,0 +1,176 @@
+import torch
+import torch.nn as nn
+from transformers import (
+    AutoConfig,
+    AutoModel,
+    PreTrainedModel,
+    T5ForConditionalGeneration,
+)
+
+
+class ReactionT5Yield(nn.Module):
+    def __init__(self, cfg, config_path=None, pretrained=False):
+        super().__init__()
+        self.cfg = cfg
+        if config_path is None:
+            self.config = AutoConfig.from_pretrained(
+                self.cfg.pretrained_model_name_or_path, output_hidden_states=True
+            )
+        else:
+            self.config = torch.load(config_path, weights_only=False)
+        if pretrained:
+            self.model = AutoModel.from_pretrained(
+                self.cfg.pretrained_model_name_or_path
+            )
+        else:
+            self.model = AutoModel.from_config(self.config)
+        self.model.resize_token_embeddings(len(self.cfg.tokenizer))
+        self.fc_dropout1 = nn.Dropout(self.cfg.fc_dropout)
+        self.fc1 = nn.Linear(self.config.hidden_size, self.config.hidden_size // 2)
+        self.fc_dropout2 = nn.Dropout(self.cfg.fc_dropout)
+
+        self.fc2 = nn.Linear(self.config.hidden_size, self.config.hidden_size // 2)
+        self.fc3 = nn.Linear(self.config.hidden_size // 2 * 2, self.config.hidden_size)
+        self.fc4 = nn.Linear(self.config.hidden_size, self.config.hidden_size)
+        self.fc5 = nn.Linear(self.config.hidden_size, 1)
+
+        self._init_weights(self.fc1)
+        self._init_weights(self.fc2)
+        self._init_weights(self.fc3)
+        self._init_weights(self.fc4)
+        self._init_weights(self.fc5)
+
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=0.01)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=0.01)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+        elif isinstance(module, nn.LayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
+
+    def forward(self, inputs):
+        encoder_outputs = self.model.encoder(**inputs)
+        encoder_hidden_states = encoder_outputs[0]
+        outputs = self.model.decoder(
+            input_ids=torch.full(
+                (inputs["input_ids"].size(0), 1),
+                self.config.decoder_start_token_id,
+                dtype=torch.long,
+                device=inputs["input_ids"].device,
+            ),
+            encoder_hidden_states=encoder_hidden_states,
+        )
+        last_hidden_states = outputs[0]
+        output1 = self.fc1(
+            self.fc_dropout1(last_hidden_states).view(-1, self.config.hidden_size)
+        )
+        output2 = self.fc2(
+            encoder_hidden_states[:, 0, :].view(-1, self.config.hidden_size)
+        )
+        output = self.fc3(self.fc_dropout2(torch.hstack((output1, output2))))
+        output = self.fc4(output)
+        output = self.fc5(output)
+        return output
+
+    def generate_embedding(self, inputs):
+        encoder_outputs = self.model.encoder(**inputs)
+        encoder_hidden_states = encoder_outputs[0]
+        outputs = self.model.decoder(
+            input_ids=torch.full(
+                (inputs["input_ids"].size(0), 1),
+                self.config.decoder_start_token_id,
+                dtype=torch.long,
+                device=inputs["input_ids"].device,
+            ),
+            encoder_hidden_states=encoder_hidden_states,
+        )
+        last_hidden_states = outputs[0]
+        output1 = self.fc1(
+            self.fc_dropout1(last_hidden_states).view(-1, self.config.hidden_size)
+        )
+        output2 = self.fc2(
+            encoder_hidden_states[:, 0, :].view(-1, self.config.hidden_size)
+        )
+        return torch.hstack((output1, output2))
+
+
+class ReactionT5Yield2(PreTrainedModel):
+    config_class = AutoConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.config = config
+        self.model = T5ForConditionalGeneration.from_pretrained(
+            self.config._name_or_path
+        )
+        self.model.resize_token_embeddings(self.config.vocab_size)
+        self.fc1 = nn.Linear(self.config.hidden_size, self.config.hidden_size // 2)
+        self.fc2 = nn.Linear(self.config.hidden_size, self.config.hidden_size // 2)
+        self.fc3 = nn.Linear(self.config.hidden_size // 2 * 2, self.config.hidden_size)
+        self.fc4 = nn.Linear(self.config.hidden_size, self.config.hidden_size)
+        self.fc5 = nn.Linear(self.config.hidden_size, 1)
+
+        self._init_weights(self.fc1)
+        self._init_weights(self.fc2)
+        self._init_weights(self.fc3)
+        self._init_weights(self.fc4)
+        self._init_weights(self.fc5)
+
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=0.01)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=0.01)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+        elif isinstance(module, nn.LayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
+
+    def forward(self, inputs):
+        encoder_outputs = self.model.encoder(**inputs)
+        encoder_hidden_states = encoder_outputs[0]
+        outputs = self.model.decoder(
+            input_ids=torch.full(
+                (inputs["input_ids"].size(0), 1),
+                self.config.decoder_start_token_id,
+                dtype=torch.long,
+                device=inputs["input_ids"].device,
+            ),
+            encoder_hidden_states=encoder_hidden_states,
+        )
+        last_hidden_states = outputs[0]
+        output1 = self.fc1(last_hidden_states.view(-1, self.config.hidden_size))
+        output2 = self.fc2(
+            encoder_hidden_states[:, 0, :].view(-1, self.config.hidden_size)
+        )
+        output = self.fc3(torch.hstack((output1, output2)))
+        output = self.fc4(output)
+        output = self.fc5(output)
+        return output * 100
+
+    def generate_embedding(self, inputs):
+        encoder_outputs = self.model.encoder(**inputs)
+        encoder_hidden_states = encoder_outputs[0]
+        outputs = self.model.decoder(
+            input_ids=torch.full(
+                (inputs["input_ids"].size(0), 1),
+                self.config.decoder_start_token_id,
+                dtype=torch.long,
+                device=inputs["input_ids"].device,
+            ),
+            encoder_hidden_states=encoder_hidden_states,
+        )
+        last_hidden_states = outputs[0]
+        output1 = self.fc1(last_hidden_states.view(-1, self.config.hidden_size))
+        output2 = self.fc2(
+            encoder_hidden_states[:, 0, :].view(-1, self.config.hidden_size)
+        )
+        return torch.hstack((output1, output2))

task_forward/accuracy-and-invalidity-check.ipynb

@@ -0,0 +1,217 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "92432099",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prediction: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1112/1112 [3:05:38<00:00, 10.02s/it]\n",
+    "Top-1: 0.5% || Invalid 16.69%\n",
+    "Top-2: 1.0% || Invalid 23.80%\n",
+    "Top-3: 1.6% || Invalid 28.18%\n",
+    "Top-4: 2.1% || Invalid 31.25%\n",
+    "Top-5: 2.5% || Invalid 33.73%\n",
+    "prediction: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1112/1112 [3:05:18<00:00, 10.00s/it]\n",
+    "Top-1: 0.2% || Invalid 22.41%\n",
+    "Top-2: 0.7% || Invalid 28.65%\n",
+    "Top-3: 1.0% || Invalid 32.95%\n",
+    "Top-4: 1.3% || Invalid 36.12%\n",
+    "Top-5: 1.6% || Invalid 38.94%\n",
+    "prediction: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1112/1112 [3:07:23<00:00, 10.11s/it]\n",
+    "Top-1: 0.2% || Invalid 31.81%\n",
+    "Top-2: 0.6% || Invalid 36.80%\n",
+    "Top-3: 0.8% || Invalid 40.56%\n",
+    "Top-4: 1.0% || Invalid 43.56%\n",
+    "Top-5: 1.1% || Invalid 46.23%\n",
+    "prediction: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1112/1112 [3:04:23<00:00,  9.95s/it]\n",
+    "Top-1: 0.1% || Invalid 57.28%\n",
+    "Top-2: 0.3% || Invalid 61.50%\n",
+    "Top-3: 0.3% || Invalid 64.65%\n",
+    "Top-4: 0.4% || Invalid 67.02%\n",
+    "Top-5: 0.4% || Invalid 69.05%\n",
+    "prediction: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1112/1112 [3:07:16<00:00, 10.10s/it]\n",
+    "Top-1: 0.4% || Invalid 64.24%\n",
+    "Top-2: 0.6% || Invalid 67.45%\n",
+    "Top-3: 0.7% || Invalid 69.89%\n",
+    "Top-4: 0.7% || Invalid 71.78%\n",
+    "Top-5: 0.8% || Invalid 73.41%\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "6a089a12",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/tmp/ipykernel_2056154/465102246.py:21: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
+      "  ax.set_yticklabels([int(i) for i in ax.get_yticks()], fontsize=12)\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "<matplotlib.legend.Legend at 0x7f69ce998510>"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqkAAAG3CAYAAACXEsyxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABX/klEQVR4nO3deZyNdf/H8feZ7cxmDmOGGcvYQoSQVG5LpYWKViq5SYlSSdLdrU1UKEvqp/WOFOlOypIlEW6EEApZx9LMGGafM/t2rt8fx5xxzFgGM3MNr+fjcR6c7/W5rvO9zlzL+1zLORbDMAwBAAAAJuJR0R0AAAAATkVIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOmUOqSmp6dr1KhR6tatm4KDg2WxWDRjxowSa3fv3q1u3bopMDBQwcHB+uc//6n4+PhidQ6HQ++++64aNGggX19ftWrVSt98802pZwYAAACXhlKH1ISEBI0ZM0a7d+/W1Vdffdq66Ohode7cWQcOHNDYsWM1YsQILV68WLfeeqtyc3Pdal955RW99NJLuvXWW/V///d/ioiIUJ8+ffTf//639HMEAACASs9iGIZRmhFycnKUnJyssLAwbdmyRddee62++OILPfroo251Q4YM0YwZM7Rnzx5FRERIklasWKFbb71Vn376qQYNGiRJiomJUYMGDTRo0CBNnTpVkmQYhrp06aJDhw7p8OHD8vT0vAizCgAAgMqi1EdSrVarwsLCzlr3/fff66677nIFVEm65ZZb1KRJE82ZM8fVtmDBAuXl5WnIkCGuNovFoqeeekrR0dHasGFDabsIAACASs6rLCYaExOjuLg4tWvXrtiw9u3ba8mSJa7n27ZtU0BAgJo1a1asrnB4x44di00nJydHOTk5rucOh0NJSUmqXr26LBbLxZoVAAAAXCSGYSgtLU21atWSh8eZj5WWSUiNjY2VJIWHhxcbFh4erqSkJOXk5MhqtSo2NlY1a9YsFiwLxz169GiJrzFu3DiNHj36IvccAAAAZS0qKkp16tQ5Y02ZhNSsrCxJzksDTuXr6+uqsVqtrn/PVFeSkSNHavjw4a7nqampioiIUFRUlIKCgi54HgAAAHBx2e121a1bV1WqVDlrbZmEVD8/P0lyOx1fKDs7263Gz8/vnOpOZbVaSwy3QUFBhFQAAAATO5dLM8vky/wLT9UXnvY/WWxsrIKDg10BMzw8XMeOHdOpXzJQOG6tWrXKoosAAAAwsTIJqbVr11ZoaKi2bNlSbNimTZvUunVr1/PWrVsrMzNTu3fvdqv77bffXMMBAABweSmzn0W9//77tWjRIkVFRbnafvnlF+3bt0+9evVytd19993y9vbWRx995GozDEOffPKJateurQ4dOpRVFwEAAGBS53VN6tSpU5WSkuK68/7HH39UdHS0JOnZZ5+VzWbTyy+/rO+++0433XSTnnvuOaWnp2vChAlq2bKlBgwY4JpWnTp1NGzYME2YMEF5eXm69tprNX/+fK1du1Zff/01X+QPAABwGSr1L05JUv369XXkyJEShx06dEj169eXJO3atUvDhw/XunXr5OPjozvvvFOTJk1SzZo13cZxOBx655139Omnnyo2NlaNGzfWyJEj9cgjj5xzn+x2u2w2m1JTU7lxCgAAwIRKk9fOK6SaESEVAADA3EqT18rsmlQAAADgfBFSAQAAYDqEVAAAAJgOIRUAAACmQ0gFAACA6RBSAQAAYDqEVAAAAJgOIRUAAACmQ0gFAACA6RBSAQAAYDqEVAAAAJiOV0V3AAAAABXA4ZB27ZKSk6Vq1aSrrpI8zHP8kpAKnMzkKywqIZYpAGa0fr00daq0e7eUkyNZrVKzZtIzz0gdOlR07yQRUs/f5bTjuVzmtRKssKhkWKYAmNH69dKIEVJSkhQeLvn5SVlZ0rZtzvaJE02xjbIYhmFUdCcuBrvdLpvNptTUVAUFBZXti11OO57LZV5Pt8IeO+YM5iZZYVGJsEwBMCOHQ+rbV9q6VbriCsliKRpmGFJkpNS2rTRzZpkckCpNXiOkltbltOO5XOa1gldYXCQnb8oMo+j5qe1nqi2p5ky1pxvf4ZAGDZL++ENq2LBomSqsPXxYatVK+vBD5zJ1utc/l76fqa2sakua9/IY/0zv/fn8nc71tUrz3p1t/NL260zjl/UyUVIfL/bfqTxfy6zLxNnGv9jvXUKCtGKF86CT10kn1K+4wrmfT0+XUlKkr76SWrbUxVaavMbp/tJwOJxHFZOSpFq1nIEtO9s5zGaTYmKk11+XRo6s/GHG4ZDGjpWio53zmp1dNK9BQc72V191hliLpWjlcDiK/l/Sw+Eomn7hv6eru9BpnGk6J497/Li0cqVz5dy3z9l+8gqem+tcoQcNkkJDi9orasdhtp1MaTbGZbHhNiO7Xdq+3bkDsNuLD8/Pl5Yvl3r3dq5PAFBeEhOd+3OLRcrLK2ov3C/6+Tn3i8nJFdO/kxBSS2PXLudp7/Bw5x/v6FH34fn50ubN0pgxlX/Hc/JO9tCh4sPz86Xff5feeafyz2tiopSR4VxBTz6KWsgwnB9I/vpLql69/PuHyic/37k8eXmVvEx5eTk//BiG5OPjPsxicR+n8P+ntp+t9mzjn632Yo5fVrUX67XOZfzSvvdnGr807/OFjn8pLRMXY/zS/J3O9lqVdZk4cMB5kKlKFcnfv6jd19f5b1aW8yhrtWqqaITU0khOdl6X6efn/BRis7kPdzik1FSpQQOpXr2K6ePFcuSIM5TZbEVHhU9eyAvntXFjqVEjZ5uHR9HKUNKjcDol1Z087EzTOtvwc6k5dfjhw86wHRjo/NueujHIynKe/hg8uGheC98PM+xkSnqN8tzwl+dO4tTxzPg+S9KOHVK/flLVqs7l6lSFp9M+/LBMTqcBwGm1aiX98IPzJqmaNd23XYbhvKSvbVvnTdIVjJBaGtWqOT9dZGVJISHOx8nS052fTF54ofLveHbscF5Pd6adbGCg9NxzlX9eHQ5p1SrnClunTvEVNjJSat9e+uc/K/9lHCgfV13lvMFw2zbnBxsT7wQAXGY8PJw3P48Y4dy/hYUVv+fk6adNsb+r+B5UJoU7nmPHSr627tgxqXnzS2PHcznNa+EKW62ac4VNT5cKCpz/RkaaaoVFJcEyBcDMOnRw3vzcpo3zrM7hw85/27Y11U3R3N1fWoV3vCcnl/zpw0R/3At2Oc2rVPLXbTVv7gwTl9J8ovywTAEwswr4HnS+gqoivif1Ut3xXE7zKl0+P1yA8sMyBQAuhNTyuNv8ctrxXE7zCgAAygzfk1oePDwq/w1D5+pymlcAAGAKHA4DAACA6RBSAQAAYDqEVAAAAJgOIRUAAACmQ0gFAACA6RBSAQAAYDqEVAAAAJgOIRUAAACmQ0gFAACA6RBSAQAAYDqEVAAAAJgOIRUAAACmQ0gFAACA6RBSAQAAYDqEVAAAAJgOIRUAAACmQ0gFAACA6RBSAQAAYDqEVAAAAJgOIRUAAACmQ0gFAACA6RBSAQAAYDqEVAAAAJgOIRUAAACmQ0gFAACA6RBSAQAAYDqEVAAAAJgOIRUAAACmQ0gFAACA6RBSAQAAYDqEVAAAAJgOIRUAAACmQ0gFAACA6RBSAQAAYDplGlL379+vhx56SHXq1JG/v7+uvPJKjRkzRpmZmW5169evV8eOHeXv76+wsDANHTpU6enpZdk1AAAAmJhXWU04KipK7du3l81m0zPPPKPg4GBt2LBBo0aN0u+//64FCxZIkrZv366uXbuqWbNmmjx5sqKjozVx4kTt379fS5cuLavuAQAAwMTKLKTOnDlTKSkpWrduna666ipJ0qBBg+RwOPTVV18pOTlZ1apV08svv6xq1app9erVCgoKkiTVr19fTzzxhH7++WfddtttZdVFAAAAmFSZne632+2SpJo1a7q1h4eHy8PDQz4+PrLb7Vq+fLn69u3rCqiS1K9fPwUGBmrOnDll1T0AAACYWJmF1BtvvFGS9Pjjj2v79u2KiorSt99+q48//lhDhw5VQECAduzYofz8fLVr185tXB8fH7Vu3Vrbtm0rq+4BAADAxMrsdH+3bt305ptvauzYsVq4cKGr/ZVXXtFbb70lSYqNjZXkPLp6qvDwcK1du/a008/JyVFOTo7reeGRWwAAAFR+ZRZSJee1pZ07d9b999+v6tWra/HixRo7dqzCwsL0zDPPKCsrS5JktVqLjevr6+saXpJx48Zp9OjRZdZ3AAAAVJwyC6n//e9/NWjQIO3bt0916tSRJN13331yOBx66aWX9PDDD8vPz0+S3I6IFsrOznYNL8nIkSM1fPhw13O73a66dete5LkAAABARSiza1I/+ugjtWnTxhVQC/Xs2VOZmZnatm2b6zR/4Wn/k8XGxqpWrVqnnb7ValVQUJDbAwAAAJeGMgupx48fV0FBQbH2vLw8SVJ+fr5atGghLy8vbdmyxa0mNzdX27dvV+vWrcuqewAAADCxMgupTZo00bZt27Rv3z639m+++UYeHh5q1aqVbDabbrnlFs2aNUtpaWmumpkzZyo9PV29evUqq+4BAADAxCyGYRhlMeE1a9bo5ptvVvXq1fXMM8+oevXqWrRokZYuXaqBAwfqP//5jyRp69at6tChg5o3b65BgwYpOjpakyZNUufOnbVs2bJzfj273S6bzabU1FRO/QMAAJhQafJamYVUSdq0aZPeeOMNbdu2TYmJiWrQoIH69++vf/3rX/LyKrpna926dXrppZe0detWValSRb1799a4ceNUpUqVc34tQioAAIC5mSaklidCKgAAgLmVJq+V2TWpAAAAwPkipAIAAMB0CKkAAAAwHUIqAAAATIeQCgAAANMhpAIAAMB0CKkAAAAwHUIqAAAATIeQCgAAANMhpAIAAMB0CKkAAAAwHUIqAAAATIeQCgAAANMhpAIAAMB0CKkAAAAwHUIqAAAATIeQCgAAANMhpAIAAMB0CKkAAAAwHUIqAAAATIeQCgAAANMhpAIAAMB0CKkAAAAwHUIqAAAATIeQCgAAANMhpAIAAMB0CKkAAAAwHUIqAAAATIeQCgAAANMhpAIAAMB0CKkAAAAwHUIqAAAATIeQCgAAANMhpAIAAMB0CKkAAAAwHUIqAAAATIeQCgAAANMhpAIAAMB0CKkAAAAwHUIqAAAATIeQCgAAANMhpAIAAMB0CKkAAAAwHUIqAAAATIeQCgAAANMhpAIAAMB0CKkAAAAwHUIqAAAATIeQCgAAANMhpAIAAMB0CKkAAAAwHUIqAAAATIeQCgAAANMhpAIAAMB0CKkAAAAwHUIqAAAATIeQCgAAANMhpAIAAMB0CKkAAAAwHUIqAAAATKfMQ+rWrVvVs2dPBQcHy9/fXy1atNAHH3zgVrN+/Xp17NhR/v7+CgsL09ChQ5Wenl7WXQMAAIBJeZXlxH/++Wf16NFDbdq00WuvvabAwEBFRkYqOjraVbN9+3Z17dpVzZo10+TJkxUdHa2JEydq//79Wrp0aVl2DwAAACZVZiHVbrerX79+uvPOOzV37lx5eJR80Pbll19WtWrVtHr1agUFBUmS6tevryeeeEI///yzbrvttrLqIgAAAEyqzE73z549W8ePH9fbb78tDw8PZWRkyOFwuNXY7XYtX75cffv2dQVUSerXr58CAwM1Z86csuoeAAAATKzMQuqKFSsUFBSkmJgYNW3aVIGBgQoKCtJTTz2l7OxsSdKOHTuUn5+vdu3auY3r4+Oj1q1ba9u2baedfk5Ojux2u9sDAAAAl4YyC6n79+9Xfn6+7r77bt1+++36/vvv9dhjj+mTTz7RgAEDJEmxsbGSpPDw8GLjh4eH6+jRo6ed/rhx42Sz2VyPunXrls2MAAAAoNyV2TWp6enpyszM1JNPPum6m/++++5Tbm6uPv30U40ZM0ZZWVmSJKvVWmx8X19f1/CSjBw5UsOHD3c9t9vtBFUAAIBLRJkdSfXz85MkPfzww27tffr0kSRt2LDBVZOTk1Ns/OzsbNfwklitVgUFBbk9AAAAcGkosyOptWrV0q5du1SzZk239ho1akiSkpOT1ahRI0lFp/1PFhsbq1q1apVV9wAAqJQMw1BBQYHy8/MruiuAG29vb3l6el606ZVZSL3mmmu0fPly141ThQqvMw0NDVWLFi3k5eWlLVu2qHfv3q6a3Nxcbd++3a0NAIDLmWEYSklJUXx8vAoKCiq6O0CJqlatqrCwMFkslgueVpmF1N69e2v8+PGaNm2abr75Zlf7559/Li8vL914442y2Wy65ZZbNGvWLL322muqUqWKJGnmzJlKT09Xr169yqp7AABUKseOHVNKSorrEjcvL6+LEgSAi8EwDGVmZiouLk5SyTfFl1aZhdQ2bdroscce0/Tp05Wfn68uXbpo9erV+u677zRy5EjXqfy3335bHTp0UJcuXTRo0CBFR0dr0qRJuu2229StW7ey6h4AAJVGQUGBUlNTFRoaqpCQkIruDlCiwnuJ4uLiVKNGjQs+9V+mP4v6ySefKCIiQl988YXmzZunevXq6b333tOwYcNcNW3bttWKFSv00ksv6fnnn1eVKlX0+OOPa9y4cWXZNQAAKo28vDwZhqGAgICK7gpwRv7+/pKcy+yFhlSLYRjGxehURbPb7bLZbEpNTeVOfwDAJSU7O1uHDh1SgwYN5OvrW9HdAU7rbMtqafJamX0FFQAAAHC+CKkAAAAwHUIqAADAeXjjjTf4hoUyREgFAAAVasaMGbJYLK6Hl5eXateurUcffVQxMTEV2rfMzEy98cYbWr16dbm/dv369d3el9M9ZsyYccb6J598stz7fjGU6d39AAAA52rMmDFq0KCBsrOztXHjRs2YMUPr1q3Tzp07K+yGsczMTI0ePVqSdOONN7oNe/XVV/Xvf/+7zF57ypQpSk9Pdz1fsmSJvvnmG7333ntuX0XWoUMH1/9bt26tF154wW06TZo0KbM+liVCKgAAlzOHQ9q1S0pOlqpVk666SvKomBOt3bt3V7t27SRJAwcOVEhIiN555x0tXLjQlL9C6eXlJS+vsotS99xzj9vzY8eO6ZtvvtE999yj+vXrlzhO7dq11bdv3zLrU3nidD8AAJer9eulvn2lfv2kJ590/tu3r7PdBDp16iRJioyMdLXt2bNHDzzwgIKDg+Xr66t27dpp4cKFbuMlJSVpxIgRatmypQIDAxUUFKTu3bvrjz/+KPYa2dnZeuONN9SkSRP5+voqPDxc9913nyIjI3X48GGFhoZKkkaPHu06ff7GG29IKvma1Pz8fL355ptq1KiRrFar6tevr5dfflk5OTludfXr19ddd92ldevWqX379vL19VXDhg311VdfXfD7lpubq4yMjAueTkUjpAIAcDlav14aMULaulWqWlWqX9/577ZtznYTBNXDhw9LkqpVqyZJ2rVrl66//nrt3r1b//73vzVp0iQFBATonnvu0bx581zjHTx4UPPnz9ddd92lyZMn68UXX9SOHTvUpUsXHT161FVXUFCgu+66S6NHj9Y111yjSZMm6bnnnlNqaqp27typ0NBQffzxx5Kke++9VzNnztTMmTN13333nbbPAwcO1Ouvv662bdvqvffeU5cuXTRu3Dg99NBDxWoPHDigBx54QLfeeqsmTZqkatWq6dFHH9WuXbvO+z1buXKl/P39FRgYqPr16+v9998/72lVOOMSkZqaakgyUlNTK7orAABcVFlZWcZff/1lZGVluQ9wOAwjK6v0j4wMw+jd2zCaNDGM7t0N4447ih7duzvbH3zQWVea6Toc5zV/X3zxhSHJWLFihREfH29ERUUZc+fONUJDQw2r1WpERUUZhmEYXbt2NVq2bGlkZ2ef9BY4jA4dOhiNGzd2tWVnZxsFBQVur3Ho0CHDarUaY8aMcbVNnz7dkGRMnjy5WJ8cJ+YlPj7ekGSMGjWqWM2oUaOMk6PU9u3bDUnGwIED3epGjBhhSDJWrlzpaqtXr54hyVizZo2rLS4uzrBarcYLL7xQ4vs0YcIEQ5Jx6NChEof36NHDeOedd4z58+cb06ZNMzp16mRIMv71r3+VWF8WTrusnlCavMY1qQAAVFY5OVKvXqUfz26Xtm+XvLyklJTiw/PzpaVLpe7dpdL8iuN330kXcIPTLbfc4va8fv36mjVrlurUqaOkpCStXLlSY8aMUVpamtLS0lx1t99+u0aNGqWYmBjVrl1bVqvVNaygoEApKSkKDAxU06ZNtXXrVtew77//XiEhIXr22WeL9eV8vlpqyZIlkqThw4e7tb/wwguaOHGiFi9erJtuusnV3rx5c9clDZIUGhqqpk2b6uDBg6V+bUnFLnsYMGCAunfvrsmTJ+vZZ59VnTp1zmu6FYXT/QAAXG7y8pw3TJ3ut9U9PZ3D8/LKtVsffvihli9frrlz5+qOO+5QQkKCK3AeOHBAhmHotddeU2hoqNtj1KhRkqS4uDhJksPh0HvvvafGjRvLarUqJCREoaGh+vPPP5Wamup6vcjISDVt2vSi3fx05MgReXh46IorrnBrDwsLU9WqVXXkyBG39oiIiGLTqFatmpKTky9KfywWi55//nnl5+dXyFdoXSiOpAIAUFlZrc6jl6W1c6c0cKBks0mBgcWHp6dLqanSBx9ILVqUrj8XoH379q67+++55x517NhRffr00d69e+VwOCRJI0aM0O23317i+IXhcOzYsXrttdf02GOP6c0331RwcLA8PDw0bNgw13TK0rkehfU8zYcEwzAuWl/q1q0ryXkzWWVDSAUAoLKyWM7v9HrbtlLz5s6bpKpUcU6nkGFIcXHOmrZtK+zrqDw9PTVu3DjddNNNmjp1qh577DFJkre3d7HLAk41d+5c3XTTTZo2bZpbe0pKitv3izZq1Ei//fab8vLy5O3tXeK0SnPav169enI4HNq/f7+aNWvmaj9+/LhSUlJUr169c57WxVJ46UDhtxRUJpzuBwDgcuPhIT3zjPN7USMjnUdOCwqc/0ZGOtuffrrCAmqhG2+8Ue3bt9eUKVMUFBSkG2+8UZ9++qliY2OL1cbHx7v+7+npWexo5HfffVfs16vuv/9+JSQkaOrUqcWmVzi+v7+/JGfAPZs77rhDkvNL+E82efJkSdKdd9551mmcr6SkJBUUFLi15eXlafz48fLx8XG7Fray4EgqAACXow4dpIkTpalTpd27pePHnafr27Z1BtSTfsWoIr344ovq1auXZsyYoQ8//FAdO3ZUy5Yt9cQTT6hhw4Y6fvy4NmzYoOjoaNf3oN51110aM2aMBgwYoA4dOmjHjh36+uuv1bBhQ7dp9+vXT1999ZWGDx+uTZs2qVOnTsrIyNCKFSs0ZMgQ3X333fLz81Pz5s317bffqkmTJgoODlaLFi3UooTLIK6++mr1799fn332mVJSUtSlSxdt2rRJX375pe65554yDYoLFy7UW2+9pQceeEANGjRQUlKSZs+erZ07d2rs2LEKCwsrs9cuK4RUAAAuVx06SNdfb5pfnCrJfffdp0aNGmnixIl64okntGXLFo0ePVozZsxQYmKiatSooTZt2uj11193jfPyyy8rIyNDs2fP1rfffqu2bdtq8eLFxX7C1NPTU0uWLNHbb7+t2bNn6/vvv1f16tVdQbjQ559/rmeffVbPP/+8cnNzNWrUqBJDamFtw4YNNWPGDM2bN09hYWEaOXKk6+austKyZUs1b95cs2bNUnx8vHx8fNS6dWvNmTNHvc7nGyBMwGJczKtzK5DdbpfNZlNqaqqCSvN1GQAAmFx2drYOHTqkBg0aVNhv2APn4mzLamnymnk+KgEAAAAnEFIBAABgOoRUAAAAmA4hFQAAAKZDSAUAAIDpEFIBAABgOoRUAAAAmA4hFQAAAKZDSAUAAIDpEFIBAABgOoRUAAAAmA4hFQAA4BJjsVj0xhtvVHQ3LgghFQAAmEJkZKQGDx6shg0bytfXV0FBQfrHP/6h999/X1lZWRXdvUrLYrGc02P16tVnrB8/fny59turXF8NAACgBIsXL1avXr1ktVrVr18/tWjRQrm5uVq3bp1efPFF7dq1S5999llFd7NSmjlzptvzr776SsuXLy/W3qxZM9f/b731VvXr189teJs2bcqukyUgpAIAcBmKj5fs9tMPDwqSQkPLpy+HDh3SQw89pHr16mnlypUKDw93DXv66ad14MABLV68uHw6cwnq27ev2/ONGzdq+fLlxdpP1qRJkzMOLw+c7gcA4DITHy/16SP16nX6R58+zrry8O677yo9PV3Tpk1zC6iFrrjiCj333HOSpPz8fL355ptq1KiRrFar6tevr5dfflk5OTlu49SvX1933XWXVq9erXbt2snPz08tW7Z0ndL+4Ycf1LJlS/n6+uqaa67Rtm3b3MZ/9NFHFRgYqIMHD+r2229XQECAatWqpTFjxsgwDLfajIwMvfDCC6pbt66sVquaNm2qiRMnutUdPnxYFotFM2bMKDZ/p14/+sYbb8hisejAgQN69NFHVbVqVdlsNg0YMECZmZlu4+bk5Oj5559XaGioqlSpop49eyo6Ovqs7/m5yMrKUnZ29kWZ1vkgpAIAcJmx26XERMlqlapWLf6wWp3Dz3Sk9WL68ccf1bBhQ3Xo0OGstQMHDtTrr7+utm3b6r333lOXLl00btw4PfTQQ8VqDxw4oD59+qhHjx4aN26ckpOT1aNHD3399dd6/vnn1bdvX40ePVqRkZHq3bu3HA6H2/gFBQXq1q2batasqXfffVfXXHONRo0apVGjRrlqDMNQz5499d5776lbt26aPHmymjZtqhdffFHDhw+/oPeld+/eSktL07hx49S7d2/NmDFDo0ePLvZ+TJkyRbfddpvGjx8vb29v3XnnnRf0upI0Y8YMBQQEyM/PT82bN9fs2bMveJqlxel+AAAquTMd7PLwkHx83GtzciSHwxlG/fyK10vOmpyc00/71OmeL7vdrpiYGN19991nrf3jjz/05ZdfauDAgfrPf/4jSRoyZIhq1KihiRMnatWqVbrppptc9Xv37tX69et1ww03SJKaN2+u22+/XU888YT27NmjiIgISVK1atU0ePBgrVmzRjfeeKNr/OzsbHXr1k0ffPCB67V69Oihd955R0OHDlVISIgWLlyolStX6q233tIrr7wiyXmJQq9evfT+++/rmWeeUaNGjc7rvWnTpo2mTZvmep6YmKhp06bpnXfecb0fs2bN0pAhQ/Thhx+6XvuRRx7Rn3/+eV6vKUkdOnRQ79691aBBAx09elQffvihHnnkEaWmpuqpp5467+mWFiEVAIBKrlev0w9r10466cCf+vaVkpKkQ4ckLy/no1CVKtJJ987o3/+WTjmz7dK4sTR58oX1W3KGVOdrVzlr7ZIlSySp2BHKF154QRMnTtTixYvdQmrz5s1dAVWSrrvuOknSzTff7AqoJ7cfPHjQLaRK0jPPPOP6v8Vi0TPPPKPFixdrxYoVeuihh7RkyRJ5enpq6NChxfo0d+5cLV261G0apfHkk0+6Pe/UqZPmzZsnu92uoKAg1/tx6msPGzbsgo58/vrrr27PH3vsMV1zzTV6+eWX9eijj8rv1E82ZYTT/QAAoMIEBQVJktLS0s5ae+TIEXl4eOiKK65waw8LC1PVqlV15MgRt/aTg6gk2Ww2SVLdunVLbE9OTnZr9/DwUMOGDd3amjRpIsl5jWlhn2rVqlUsZBfeKX9qn0rj1P5Xq1bNrZ+F78epR2qbNm163q9ZEh8fHz3zzDNKSUnR77//flGnfSYcSQUAoJL77rvTD/M45XDUrFnSwYPOG6OqVpUCAk4/7vjx0ikZ7bTTPV9BQUGqVauWdu7cec7jWCyWc6rz9PQsVfupN0RdTKfrc0FBwWnHqYh+nk5hsE9KSiq31+RIKgAAlZyv7+kfp1436uvrvBbVw6Pkx8ms1nOf7oW46667FBkZqQ0bNpyxrl69enI4HNq/f79b+/Hjx5WSkqJ69epdvE5JcjgcOnjwoFvbvn37JDm/PaCwT0ePHi12JHjPnj2u4VLRUdCUlBS3ugs50lr4fkRGRrq1792797yneTqF70NoeX0vmQipAABctrKypIyM4o/y/nGnf/3rXwoICNDAgQN1/PjxYsMjIyP1/vvv64477pAkTZkyxW345BMXx16Mu9pPNXXqVNf/DcPQ1KlT5e3tra5du0qS7rjjDhUUFLjVSdJ7770ni8Wi7t27S3IeMQ4JCdGaNWvc6j766KPz7lvhtAtv7Cp06vtTGvElfO9YWlqapkyZopCQEF1zzTXnPe3S4nQ/AACXmaAgqXp159dMnfL1oi7VqzvrykOjRo00e/ZsPfjgg2rWrJnbL06tX79e3333nR599FE999xz6t+/vz777DOlpKSoS5cu2rRpk7788kvdc889bjdNXQy+vr766aef1L9/f1133XVaunSpFi9erJdfftl1RLFHjx666aab9Morr+jw4cO6+uqr9fPPP2vBggUaNmyY2/WiAwcO1Pjx4zVw4EC1a9dOa9ascR2ZPR+tW7fWww8/rI8++kipqanq0KGDfvnlFx04cOC8p/nhhx9q/vz56tGjhyIiIhQbG6vp06fr77//1syZM+VzMQ+hnwUhFQCAy0xoqDR7tnl+cUqSevbsqT///FMTJkzQggUL9PHHH8tqtapVq1aaNGmSnnjiCUnS559/roYNG2rGjBmaN2+ewsLCNHLkSLfvLr1YPD099dNPP+mpp57Siy++qCpVqmjUqFF6/fXXXTUeHh5auHChXn/9dX377bf64osvVL9+fU2YMEEvvPCC2/Ref/11xcfHa+7cuZozZ466d++upUuXqkaNGufdx+nTpys0NFRff/215s+fr5tvvlmLFy8udnPYufrHP/6h9evX6/PPP1diYqICAgLUvn17TZ8+XTfffPN59/N8WIyKuPq2DNjtdtlsNqWmprruFAQA4FKQnZ2tQ4cOqUGDBvL19a3o7lwWHn30Uc2dO1fp6ekV3ZVK5WzLamnyGtekAgAAwHQIqQAAADAdQioAAABMh5AKAABwihkzZnA9agUjpAIAAMB0CKkAAAAwHUIqAAAATIeQCgAAANMhpAIAAMB0CKkAAAAwHUIqAAAATIeQCgAAANMhpAIAgMva6tWrZbFYNHfu3IruCk5CSAUAABXGYrGc02P16tVnrB8/fnyJ01+9erXuu+8+hYWFycfHRzVq1FCPHj30ww8/lONc4nx4VXQHAABA+UvITFBqdupph9t8bQrxDynzfsycOdPt+VdffaXly5cXa2/WrJnr/7feeqv69evnNrxNmzbFpj1q1CiNGTNGjRs31uDBg1WvXj0lJiZqyZIluv/++/X111+rT58+F3FucDERUgEAuMzkFeTpyUVP6mDywdPWNKzWUN/c/428Pb3LtC99+/Z1e75x40YtX768WPvJmjRpcsbhkjR37lyNGTNGDzzwgGbPni1v76L5ePHFF7Vs2TLl5eVdWOdRpsrtdP/bb78ti8WiFi1aFBu2fv16dezYUf7+/goLC9PQoUOVnp5eXl0DAOCy4uXhpdpVais1J1VVfasWe6TmpKp2ldry8jDvsaysrCxlZ2efdvhrr72m4OBgTZ8+3S2gFrr99tt11113ubU5HA69/fbbqlOnjnx9fdW1a1cdOHCg2Li//fabunXrJpvNJn9/f3Xp0kW//vqrW80bb7whi8Wiffv2qW/fvrLZbAoNDdVrr70mwzAUFRWlu+++W0FBQQoLC9OkSZPO8524dJVLSI2OjtbYsWMVEBBQbNj27dvVtWtXZWZmavLkyRo4cKA+++wz9erVqzy6BgBApZedn63s/GwZhuFqy3fkKzs/W3kFecVqcwpy1O/qfrJZbcp35Mvf219+Xn7y8/ZTviNfNqtN/Vv3V05Bzmmnm1uQW27zd6oZM2YoICBAfn5+at68uWbPnu02fP/+/dqzZ4/uueceValS5ZynO378eM2bN08jRozQyJEjtXHjRj3yyCNuNStXrlTnzp1lt9s1atQojR07VikpKbr55pu1adOmYtN88MEH5XA4NH78eF133XV66623NGXKFN16662qXbu23nnnHV1xxRUaMWKE1qxZc35vyCWqXD4ijRgxQtdff70KCgqUkJDgNuzll19WtWrVtHr1agUFBUmS6tevryeeeEI///yzbrvttvLoIgAAlVav75wHdmbdO0s2X5sk6YfdP2jmnzN1W8Pb9Ox1z7pq+/7QVzkFOfq8x+fqFNFJyyKXKTMvU1H2KAX7BsuQodsb3a42YW3Ud15f2XPs+vCODxVhi5Ak/XLwF03dPFXX1b5Or3Z+tdzntUOHDurdu7caNGigo0eP6sMPP9Qjjzyi1NRUPfXUU5Kk3bt3S5JatmxZqmlnZ2dr+/bt8vHxkSRVq1ZNzz33nHbu3KkWLVrIMAw9+eSTuummm7R06VJZLBZJ0uDBg3XVVVfp1Vdf1c8//+w2zfbt2+vTTz+VJA0aNEj169fXCy+8oHHjxumll16SJD388MOqVauWpk+frs6dO5//m3OJKfMjqWvWrNHcuXM1ZcqUYsPsdrvrupPCgCpJ/fr1U2BgoObMmVPW3QMA4LJksVjUv3V/+Xv7KzMvU5KUXZAtf29/9W/d3xXAzObXX3/Vc889p549e+rJJ5/U77//rhYtWujll19WVlaWJGe+kFSqo6iSNGDAAFdAlaROnTpJkg4edF67u337du3fv199+vRRYmKiEhISlJCQoIyMDHXt2lVr1qyRw+Fwm+bAgQNd//f09FS7du1kGIYef/xxV3vVqlXVtGlT1+vAqUyPpBYUFOjZZ5/VwIEDS/w0s2PHDuXn56tdu3Zu7T4+PmrdurW2bdt22mnn5OQoJyfH9bxwgQQA4HLzXa/vJElWT6ur7b5m96ln057ytHi61c66b5arNtQ/VJ0iOumnAz/pmrBrFJkcqU4RndQmzHmn/LSe04pNt2vDrupSv4s8LOb4FksfHx8988wzrsDasWNH14GvtLS0Uk0rIiLC7Xm1atUkScnJyZKclxFIUv/+/U87jdTUVNd4JU3TZrPJ19dXISEhxdoTExNL1d9LXZmG1E8++URHjhzRihUrShweGxsrSQoPDy82LDw8XGvXrj3ttMeNG6fRo0dfnI4CAFCJ+Xr5Fmvz8vAq8canU2v7t+6vtX+vVUx6jAJ8AtyOopZmuhWpbt26kqSkpCRJ0pVXXinJeTCsNDw9PUtsL7wmt/Ao6YQJE9S6desSawMDA886zbO9DpzKbClLTEzU66+/rtdee02hoaEl1hQelrdarcWG+fr6uoaXZOTIkRo+fLjrud1udy2kAADg3LQJa6NOEZ00b8883Xvlva6jqJVJ4WnywrzRpEkTNW3aVAsWLND7779fLDier0aNGkmSgoKCdMstt1yUaeL0yuxY/auvvqrg4GA9++yzp63x8/OTJLfT9oWys7Ndw0titVoVFBTk9gAAAKVjsVg0oM0A3dboNg1oM8C016JKUnx8fLG2tLQ0TZkyRSEhIbrmmmtc7aNHj1ZiYqIGDhyo/Pz8YuP9/PPPWrRoUale/5prrlGjRo00ceLEEr8qs6T+4fyVyZHU/fv367PPPtOUKVN09OhRV3t2drby8vJ0+PBhBQUFuU7zF572P1lsbKxq1apVFt0DAAAnaR3WWp/3/Lyiu3FWH374oebPn68ePXooIiJCsbGxmj59uv7++2/NnDnT7aanBx98UDt27NDbb7+tbdu26eGHH3b94tRPP/2kX375pdhXV52Nh4eHPv/8c3Xv3l1XXXWVBgwYoNq1aysmJkarVq1SUFCQfvzxx4s925etMgmpMTExcjgcGjp0qIYOHVpseIMGDfTcc89p9OjR8vLy0pYtW9S7d2/X8NzcXG3fvt2tDQAAXN7+8Y9/aP369fr888+VmJiogIAAtW/fXtOnT9fNN99crP6tt97SzTffrA8++EAff/yxkpKSVK1aNV1//fVasGCBevbsWeo+3HjjjdqwYYPefPNNTZ06Venp6QoLC9N1112nwYMHX4zZxAkWowyu0k1ISNC6deuKtb/66qtKS0vT+++/r0aNGqlly5bq3r27/vjjD+3du9f1VRHTpk3TwIEDtXTpUnXr1u2cXtNut8tmsyk1NZVT/wCAS0p2drYOHTqkBg0ayNe3+M1MgFmcbVktTV4rkyOpISEhuueee4q1F35X6snD3n77bXXo0EFdunTRoEGDFB0drUmTJum2224754AKAACAS0uFf8lZ27ZttWLFCvn5+en555/XZ599pscff1xz586t6K4BAACggpTrF52tXr26xPaOHTvq119/Lc+uAAAAwMQq/EgqAAAAcCpCKgAAAEyHkAoAQCXBz2bC7C7mMkpIBQDA5Ap/6z0vL6+CewKcWeGve3l5XfhtT4RUAABMztvbW1arVampqRxNhanZ7XZ5enq6PlhdiHK9ux8AAJyfkJAQxcTEKDo6WjabTd7e3rJYLBXdLUCS8zR/RkaG7Ha7wsPDL8qySUgFAKASKPx1noSEBMXExFRwb4DiLBaLqlatKpvNdlGmR0gFAKCSCAoKUlBQkPLy8lRQUFDR3QHceHt7X5TT/IUIqQAAVDLe3t7y9vau6G4AZYobpwAAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYTpmF1M2bN+uZZ57RVVddpYCAAEVERKh3797at29fsdrdu3erW7duCgwMVHBwsP75z38qPj6+rLoGAAAAk/Mqqwm/8847+vXXX9WrVy+1atVKx44d09SpU9W2bVtt3LhRLVq0kCRFR0erc+fOstlsGjt2rNLT0zVx4kTt2LFDmzZtko+PT1l1EQAAACZVZiF1+PDhmj17tlvIfPDBB9WyZUuNHz9es2bNkiSNHTtWGRkZ+v333xURESFJat++vW699VbNmDFDgwYNKqsuAgAAwKQshmEY5fmC11xzjSTp999/lyTVrFlTXbp00Zw5c9zqmjZtqrp162rFihXnNF273S6bzabU1FQFBQVd3E4DAADggpUmr5XrjVOGYej48eMKCQmRJMXExCguLk7t2rUrVtu+fXtt27atPLsHAAAAkyjXkPr1118rJiZGDz74oCQpNjZWkhQeHl6sNjw8XElJScrJySlxWjk5ObLb7W4PAAAAXBrKLaTu2bNHTz/9tG644Qb1799fkpSVlSVJslqtxep9fX3dak41btw42Ww216Nu3bpl1HMAAACUt3IJqceOHdOdd94pm82muXPnytPTU5Lk5+cnSSUeLc3OznarOdXIkSOVmprqekRFRZVR7wEAAFDeyuzu/kKpqanq3r27UlJStHbtWtWqVcs1rPA0f+Fp/5PFxsYqODi4xKOskvPo6+mGAQAAoHIr05CanZ2tHj16aN++fVqxYoWaN2/uNrx27doKDQ3Vli1bio27adMmtW7duiy7BwAAAJMqs9P9BQUFevDBB7VhwwZ99913uuGGG0qsu//++7Vo0SK30/W//PKL9u3bp169epVV9wAAAGBiZfY9qcOGDdP777+vHj16qHfv3sWG9+3bV5IUFRWlNm3aqGrVqnruueeUnp6uCRMmqE6dOtq8efM5n9Lne1IBAADMrTR5rcxC6o033qj//e9/px1+8svu2rVLw4cP17p16+Tj46M777xTkyZNUs2aNc/59QipAAAA5maKkFreCKkAAADmZtpfnAIAAADOBSEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApuNV0R2obBIyE5SanXra4TZfm0L8Q8qxRwAAAOeusmQZQmop5BXk6clFT+pg8sHT1jSs1lDf3P+NvD29y7FnuFCVZYVF5cEyBcCMKlOWMUVIzcnJ0euvv66ZM2cqOTlZrVq10ltvvaVbb721orvmxsvDS9W8aishfZtqBdST5aRhhqSjGUd0TWhteXmY4m29YHv+TtDRpNPvZGsF23RlROXfyeYV5GnA92dfYX/oU/ErLCoHlikAZlWZskzF90DSo48+qrlz52rYsGFq3LixZsyYoTvuuEOrVq1Sx44dK7p7LgkJFv35VX8lNFirpPx8eebbXMMKvFLl8LLpz7X9ldDJotDQCuzoRXD0WJ46v/uk0r1Pv5MNzGuo7a9+o1phlXsnm5zopT/W1tZx2zb5ZNUrNjzX74jSdtZW8u1eqlGjAjqISodlCoBZVaYsU+E3Tm3atEn//e9/NW7cOE2YMEGDBg3SypUrVa9ePf3rX/+q6O65sdul3L/bKCilkxz+cZJvqgoCo2T4JcrhH6eglE7K/buN5v31oxbsWaCM3AzXuDH2GK05skZ7Eva4TXN3/G79Ff+XsvOzXW0ZuRk6mnZUKdkpbrU5+TnKK8iTYRhlOp+SlJnuJcNeWw6fVFmNqsUeDp9UGfbaykw3xeecC5KWZpH//v7yKrDJ0ztfVo8A18PTO19eBTb57++vtDTL2ScG6NJbphyGQwWOArdtj8NwKLcgV7kFuW61Ofk5yszLVL4j39WW78iXPceu9Nx0t9r03HQlZSUpJz/HrTY+I16JmYlutSnZKYpNi3XbruYV5CkqNUox9hi32oTMBB1OOex2uUW+I18Hkg4oMinSrfZ4+nHtTdirhMwEt9qdcTu1M26n2zwfTTuqP479odi0WFebYRjaGrtVvx/9XQWOAld7jD1Gm2I26XDKYbfX2xC1Qeuj1ru9b1GpUVpzZI32Jux1q117ZK1WHVqlrLwst9rlkcv15/E/3WpXHVqlnw78pLScNLfaH/f+qE0xm9xql0cu1/w985WUleRqi7ZHa+5fc7XmyBq32mUHlunbnd/qePpxt3n7+s+v9dOBn9xql+5fqi+3f6mo1ChXW2xarKZvm655u+e51S7Zv0Sf/f6ZDiQdcLUdTz+ujzZ/pFl/znKrXbRvkT747QPtitvlakvITNB7G97TZ79/Vqx2wq8T9PvR311tKdkpGrd2nCatn1Ss9q01b2l91HpXW3puut5Y/YZGrx5drL+vrXxNvxz8xdWWk5+jkStGauSKkcoryHOrffHnF/Xj3h9dbYZhaPiy4Rq+bLjbMrzswDINXTpU3+781u31nlv6nIYsHuL2N1pxcIUG/ThI07dNL1b7+ILH3ZbLVYdWqf/8/vrgtw+K1Q5c0ldpCVVdWcbwS5SnT7a8vA23LGO3q8JVeEidO3euPD09NWjQIFebr6+vHn/8cW3YsEFRUVFnGLv8WWRR3cT+8jb8VWCNV55PrPJ9jsvb8Fft+P4yHBZ9f+ArfbblcyWmF20stsZu1TtrJ+j7XQuVnS3XY8zqt/Xispf0d9IxV+36qPV6YsFgTf71A7faJ398Wvd8c592HtvnVtt7zkMas2qsW+2oX97Ukwuf1o6jRRu93fG79e/lr+rj3z53q52xdbYmrJ2i3ceLjprGZx2TV06wLBbJ4pkvHwXIRwGSV5byfGPl6fBT9aj+slgsisuI0xe/z9R3Oxa4TXfR7p/1yW/TtPPoftd0EzIT9H8bP9b0LTPdaufvWqKJ697X5r+LNrxJWUkat2aCJq1zfx/m7lioN1a+pf9FblTh/iM1O1Wv/jJKr/8yxq12zp/z9eKykVq65xdXbWZepoYtHa6hi4crM8uhnBzJmtxGvnm1lBG4Q3nWWHl6Sh6ehvKtxyXPfMVH/EepmVmu6f6wc5EGLxyimdu+1cmfGZ5e/KwGLxyi46mprtrFu5dr8MIhmr5lphyOotrnf3pBgxcO0d+Jca7a5fv+p8ELh+jj3z53q31p+UgNXjhEB+KiXbWrIzdo8MIhen/9Ryoo2jfq9ZVvaPDCIdp97KCrdv3h3zV44RBNWDvFrfat/43V4IVDtD16t6v296idGrxwiN5a/Y7yi3KG3l03UYMXDtHmv/901f4Zs1eDFw7RqF/edKt9f+MHGrxwiNYd2uyq3Xv8sAYvHKKRP7/mVvvR5k80eOEQ/XJgnav2UEKsBi8cohd+esmtdtrW6Rq8cIiW7V3lqo1OStDghUP07OLn3Wpn/fm1Bi8cooV//eSqjU9N0+CFQzR44RDlFe1TNGfXdxq8cIjm7ihaP1PTc1y1GdlFgWL+ngUavHCIZm+f66rNyjJctYlpzvXeL7WN/HIjlBG4Q7m+0fL0kDw8DOVb4ySPAsXV+0RxaUU7oOWRy/XoD48XWz+f+vFZ/fP7R3U48airduWhlXr4u76asHZKsdpe3z6kPSety2uOrNH9/+2lN1a+5VY7ZNFQ9Zx9r7ZFF+3410et152zeuiFn15yqx26eLh6zr5HG//e6qrdfmy77v3v/Xp+6Ytuta+sGKVe3z6oNQd/c9XuTdirh+c+omFLXnCrHb9mov75fX+tPLDOVXsk5Ygenf+Yhi11r/2/DR9r4IJBWr7/f67auIw4PbloSLHaaVu+0tOLntVP+1a61s+U7BQ9t/R5Pf/TCLfa//75vYb/NEKL9yx31WblZeml5SP10s8jlZnlcNUu/GupXl7xqn7c/ZOrNt+Rr9dXjtLrK99QSnq2q/bn/as1etWbbrWSNHbtOL39v3FKsKe7atce+k3vrJ2ghbuXuq33H/z2f5r462QdS01x1W6K2q4pGz4oVjtt2xf6v40fKiY5oWj9PLpHn2z+TIv2/OS23v935xz9Z8s0/Z143FW75/ghfbH1y2K1P+5dpK+2z9LhxFhX7cGEGM3+879asvdnt3Vu+cFfNGfnXB1KPFq0fibH6/td87Rs/wq32nVHftWC3T/qUEKMq/Z4aooW712qFQdWudVuPbpNy/Yv16HEotrEtHStiFypVQfXuNXuivtLqw+tcatNSc/WuiPr9evfG91qI5MOasPfv7nVpmXmaXP079oUvcVtGxGVGq2tR7frSFLRvGVmOfTnsZ3689hO5eYV/ZHjMuL1V9weRaccd1vW9sbv1974/crKKXqDk7OTFZl4SLH2BLfawylROpIcpczsog6n56YrJjVWcWnJbrXH0uJ1LC1OmTlFtdn52UpIT1JyRppbbXKmXak5qZIKXFnG8E6Xp6chw8cub8NfdRP7yyJzfICu8MNg27ZtU5MmTRQUFOTW3r59e0nS9u3bVbdu3WLj5eTkKCen6NO3vRwjf2B6GwWnd9LxqguVbw9Tvk+ydKiTDm1oo4J8KXN1J3lZc/RxpJ/GjXKOU92/uiI3tNKxn+tp48Siaf1VJ1z5ngH6v11W/d84Z5uHxUM7t/vp4P98tW1KUe32+gXK9ZImb/fQFyemkZOfo41bM7Q9OVubJxfV7og4qiyfaCUvz9O3/+dsS85O1n9X/SHP5FwtiS6q3VV3szKsB7Quu6MWfNxQkhSXFatEv80qSK+iPK84eaQGySKL8qrFyfA6Lp+Y7vJLbSPJGTzfXjBHeYm19dXfd7umu7fWeqX6/66mSfW15svGkiR7jl1Tly1RZkJ1zTv8T1ftgbA/lBS4XnPiG2vTrFaSnDuKr/63RvaEAK1+Z6ir9mDNSCVU+U2LEq7Suv9Ivr5SniNP8zZuVWK8l9t7djg0VnG2nfolqaU6fCDZbM4jQL9s36+4OCnyE4eyMjx0+JBFubWuUX7rtUrPs8vHCFeBt12eDn/lZlaR3YjW8887FOTnnG50cKqOBkdpfWqSurwhRUQ423/bHaWomAIdnl4gnxPbodiqdkWFRGmdvYk6/Vtq7HwrtGlPtA4cydTBGfnyPbExPG5L15HQKK1Jr6eO/lLLls72LXuPamdkkg58mSf/E3kpvkqGDtWM0uqMmurgLV17rbP9931HtWVvrPbOzFGVEwfokwKzdCAsSkFZVXWDRSq8imbbgVit2xml3bNyZDtxsCbFP0v7akUpINtXNxRIXbueWKYOHtfP26K0a3aWqp04EGD3y9Ge2lHyy5XaZ0t33uls/+twvOb/FqUd32Sq+omDZ+nWXP1VN0rWvBxdmy7dd5+zfX90guavidL2bzNU48RqnOWTpx0RUfIusOmaJKlPH2d7ZGyi5q+K0tY5aQo7cZAsx6tAf9SPkofDqjbHpccec7b/HZ+k+auitGWuXf9JdrbleTq0rYHzg2/rGOmpp078jZKTNX9VlDZ9n6ovT+TGAouh3xs5axtHGhrxvLM9MT1V81dFaWNKir45cfDNkLT5Cmdt+C6H/tnb+YHWlna9EoNWKSMvTTmpksPHLuX7S9lVleuI1fQZ+brhxPKakZehZb/GaVNqipYUHbTS1gaJyvdMU9CWAn14YhuRW5CrNZtStS01Q2veLardXj9duV4ZemdzgWsbUeAo0Obt2dqRnKPf3yuq3RGRpyyffI3fYLi2EZK06y9pT7KhfR+e1FZXyrBK4381tODjovbdu6W/kqS/Pz2prbZFaX7S+FXSzdOcbRaLRXv3SjuTpF7Timr3hXsoNcCiccul7l8W1R484K0d8d7qNaOoNrKmj1ICfDVuqafu+drZ5mHxUPTBQCUfD1Cvkw6+HQkJUFIVmyb9ZNXdnzu3ER4WDyUcCVHsUU/1+qaoNjrYpsQqNRX3U4Du+dC5jfCweMgeXVt//y09+K1cu+vYqsFKCIrQ8WVVdc8kqUYNZ3+zjzVUZKT0zzke8jyRVeKCqishqImOL6uhu2sXbSMsCc21Z7dDj831lPeJbURiYA3F21rq2M91dE9Y0TbCO+lq7d+Woyfn+ri2Jyn+YYqzXatj2Y10d/WibYR/yjXasjFdQ7/3l/VEVrH71VScraOO5TRSj6CibUTV9Ou0YXWqRvwQJL8T2550a03F2brqWF4dbQgo2kaEZP1Dq1c21Ss/VHfVZvrUUJztDh3Pq6H/+RZtI2rnddYvv1yp0fNqurZT2d6hig+6T3EF1bTMs2gbUV83aunyKzV+XoQ+OFGb6xmieNvDiisI1EIVbSMae9+kBT831eR5V+jTE7V5HsFKCBqgeMNXc/KKthFX+d+sucua6qN5LfTlidoCjyAlVHlSFsNTX2UWbSNa227W7KVNNW1eY/33RK3DEqCkwGGSpM/TirYR14bcpBlLmujrefU070StIR+lBL4kSfo03lPDT2wjOtS6SZ+Pb6K5eeFa4qqV7P6vO2uj/PXKSGd7l3pd9NGYplqUH6JfTgrFdr/Rkgx9GllVb57IEZ0iOumjN5toRZZNv55Um2EdI0MOfbqrpt49sY34R8Q/9MnbTfVraoB6nbTeZ3m/qdysAsUfCFfN5t4KTu+kuKClshg+yvWKVqj9dgWmt9Hp70YpXxUeUmNjYxUeHl6svbDt6NGjxYZJ0rhx4zR69OgSh5W1wqOpSYFrlasCWXJC5Lu36JNHnZihCgiQfE/K1h3qdlCL4x10Uq6WJDWPniBJCmxR1Na1YVd1jO1a7FB7qyOfymHJk62Rr6vt+jrXq0PSJ0qK93GrbXB8uBweWQoKre9qa1K9iVpnjFBSUhW32popPZXrlSh/ax1XW7BvqKyH7lGBd7LyrvxGho9dllybPAqsMrKryXffI675DfYLVmNHD8Wl2dymG5z+D/nn1JdfXoSrrapvVbXQQzqS4u9WWz3tRvnnNFZgdlNXm83Xpms9ntDeBPd5C7F3VWBWM7faKj5VdKP1eW2Ncz85EGq/TUFZreSXU9QHXy9f3RHwutYflSzydLVbdz4ho9p+5dXcJKPAUK5XnELtt8nY+rgS4i3yqOd70nRvUVBWS3nnV3d7vQeqv6mlmwx5OYre4+D0zgrIaSzv/Gputb1qvKZ56x1u06iWcZ38cuvKq8D9vewV9m/lr8mTNa9oXbFlttWVMW/Lq8D9A16v2i8ofWWO/HKL5rlKVktdGfO2PAsC3Wrvr/usEpZlyz+ngastILuprox5Wx4OP7faeyMG6+9FmfLLLbrG0j+n4Ylaq1vtPfUHaM8P6W598Mur46w13P+ePev31dZveso3t2j588mroStj3pbFcN9E9WjwoH6NuV2+ebVcbd4F1XRlzNuS4SFdXVR7R4N7tSymi6x5Ya42z4IAZ60kNSmqvb3BXfoh5nr55BVdIOph+LhqPSKKrru+uf6tujKmtXzy3W8aLKz1rhHgags7NkAx+X8qr+ZvMmTI8IuTz9+3y/vPwSookKw9q7pqO9frrFaxzeTIDio2XcPiUEDjmq62f9T9h9rFXaksu/8ptWNlyKGg+kUXkl1f53r9I/E/xbYRTWPeliwOVQ0vWtba1Wqnrqlf6ehRr+K1cig4pGiZuLrm1borc44ORLuvc02PjpFkyFalaBrNQprpgbwF2nnE/ehMk1jnTtt60uLTsFpDPaQftOWIW6kaHX9BpwqvEq6Hvb/Rr6fU1ksYrHoJg93agv2C9bD/F/rllNo6SY+oTtIjbm0BPgHqE/SJlvx9yuul3KvwlHvd2rw8vNSn2vuad8pJvxr2bqph71aszw+FjNc30e5t1dM7q3p652K1D9R4RWmn7AKrZl6rqpnXFqu9u+Zzio11bwvKaqWgrFbFau8IG6h9x93bAnOaKDCuSbHaW8Ie0u/x7m3+ufVVP/6pYrWda/bU6gT3Nt+8cNVNHFCs9oaat+rHJPc2n4Lqqp3Up1htuxodVSvZvc3bEaTwlPuK1bYKaaewlHZubZ4Of9VMvbNYbbPqLVTD3sKtzcPwUUha12K1jao1VkhaY7c2izwVnO5M8x4nLdoRQRGqlhFxSq3F9XfzPKk2vEq4bFnFM1BQlvPTx8lrYnX/6grKqa6cPPfagJwrnNM9eXxrkALzgnTSVTeSnNthR45kcTg/fBVmmWyfGHk6zHUUVZIsRnlc4HgGjRo1UtOmTbVkyRK39oMHD6pRo0Z67733NGzYsGLjlXQktW7dukpNTS12VPZiiYyUevWSqlaV/AMM7ao7TMeqzlPN5HvV/O8pysywKCVFmj1bathQ8vCQfE7aL2Rnn37aF1KbkyOd7q9osbjvAM61NjJSeuABKchm6EizYUqwLZN/9hXK9D2gkNTbVW/3FKXZLfruO6lRIyk3V26nnk7lW5TvLmqt1erstyTl5cntNNW51h486PwUXrWqlB+6VTsaDJRhKZDF8NTVRz6XZ1xbJScX/V3PNN38fLmdTjqVj4/z73exa729JU/P0tcWFMjtlNapvLycj9LWOhzOv93FrjUMFfugd761np7O9+Ji13p4SFFRRduK3OonlikVyCJPtTz0ubzi2yolRfrmG6lZs6JxK9M2Qqq49b68txFnqz3bOsc2onjt5b6NKI9scLragwelRx6RqlVzzzJhKffqqqiiLFO4f7/Y7Ha7bDbbOeW1Cj+S6ufn5xY2C2WfeHf9/PyKDZMkq9Uqq9Va4rDyYJFFdRMGqMAjQxGJA+TpYZGHh3MhsVrdN6KFSmo7ndLUluZtKE2txSJ5elgUkdRfyVXWKtsaJU+HvyKS+svTw/2Tlo/PaSZSgrKq9fYu2kiUptZqletvVzXbeSlH4QoblNlGmZYz/11PdvIG8GzMUOvpWbQzupi1Hh7nvgyXptZiqVy1UvFlqmp2G2V6FN+hSJVvG2GG9b48thFnY4Z1mW2EkxnW+9JuI8q79uQPTSdnmboJA0x1FFUyQUgNDw9XTExMsfbYE+ctatWqVWxYRcs6ce2eV0ZrXZHwuSQp46T2S0lWluQb30ZBQZ0UHzJPoQn3yjO+zSU7r5JFIVEDlOPIUEjMAGVmWC7JeUX5YJkCYFaVIctUeEht3bq1Vq1aJbvd7nbY97fffnMNN4ugIKl6dSkx8fSH8qtXd9ZVdu7zalHA/gHKzM9QwKEBSrU7P2ldmvMqKaW1wmI+V4GklBM1l8q8onywTAEwq8qUZSr8mtTffvtN119/vSZMmKARI0ZIcl5v2qJFC1WvXl0bN248p+mU5hqHCxEfrzN+d1hQkCr8y28vFua1yKU0rygfLFMAzKoit0+V6prU6667Tr169dLIkSMVFxenK664Ql9++aUOHz6sadOmnX0C5Sw09PLZsTCvwPljmQJgVpVl+1ThIVWSvvrqK7322muaOXOmkpOT1apVKy1atEidOxf/Sg4AAABc+ir8dP/FUl6n+wEAAHB+SpPXKvxnUQEAAIBTEVIBAABgOoRUAAAAmA4hFQAAAKZDSAUAAIDpEFIBAABgOoRUAAAAmA4hFQAAAKZjil+cuhgKf5PAfqYfowUAAECFKcxp5/JbUpdMSE1LS5Mk1a1bt4J7AgAAgDNJS0uTzWY7Y80l87OoDodDR48eVZUqVWSxWMrlNe12u+rWrauoqKhL/qdYmVfg/LFMATCr8t4+GYahtLQ01apVSx4eZ77q9JI5kurh4aE6depUyGsHBQVdNjse5hU4fyxTAMyqPLdPZzuCWogbpwAAAGA6hFQAAACYDiH1AlitVo0aNUpWq7Wiu1LmmFfg/LFMATArM2+fLpkbpwAAAHDp4EgqAAAATIeQCgAAANMhpAIAAMB0CKkAAAAwHUIqAAAATIeQegbp6ekaNWqUunXrpuDgYFksFs2YMaPE2t27d6tbt24KDAxUcHCw/vnPfyo+Pr58O3wBdu3apV69eqlhw4by9/dXSEiIOnfurB9//LFYbWWf19WrV8tisZT42Lhxo1vt+vXr1bFjR/n7+yssLExDhw5Venp6BfUcZlAW2wWHw6F3331XDRo0kK+vr1q1aqVvvvmmjOcEwKVm8+bNeuaZZ3TVVVcpICBAERER6t27t/bt21estjJsny6Zn0UtCwkJCRozZowiIiJ09dVXa/Xq1SXWRUdHq3PnzrLZbBo7dqzS09M1ceJE7dixQ5s2bZKPj0/5dvw8HDlyRGlpaerfv79q1aqlzMxMff/99+rZs6c+/fRTDRo0SNKlMa+Fhg4dqmuvvdat7YorrnD9f/v27eratauaNWumyZMnKzo6WhMnTtT+/fu1dOnS8u4uTKIstguvvPKKxo8fryeeeELXXnutFixYoD59+shiseihhx4qpzkDUNm98847+vXXX9WrVy+1atVKx44d09SpU9W2bVtt3LhRLVq0kFSJtk8GTis7O9uIjY01DMMwNm/ebEgyvvjii2J1Tz31lOHn52ccOXLE1bZ8+XJDkvHpp5+WV3cvuvz8fOPqq682mjZt6mq7FOZ11apVhiTju+++O2Nd9+7djfDwcCM1NdXV9p///MeQZCxbtqysuwmTutjbhejoaMPb29t4+umnXW0Oh8Po1KmTUadOHSM/P7/sZgbAJeXXX381cnJy3Nr27dtnWK1W45FHHnG1VZbtE6f7z8BqtSosLOysdd9//73uuusuRUREuNpuueUWNWnSRHPmzCnLLpYpT09P1a1bVykpKa62S21e09LSlJ+fX6zdbrdr+fLl6tu3r4KCglzt/fr1U2BgYKWcV1wcF3u7sGDBAuXl5WnIkCGuNovFoqeeekrR0dHasGHDxZ0BAJesDh06FDuj2bhxY1111VXavXu3q62ybJ8IqRcoJiZGcXFxateuXbFh7du317Zt2yqgV+cvIyNDCQkJioyM1HvvvaelS5eqa9euki69eR0wYICCgoLk6+urm266SVu2bHEN27Fjh/Lz84vNq4+Pj1q3bl3p5hXlqzTryrZt2xQQEKBmzZoVqyscDgDnyzAMHT9+XCEhIZIq1/aJa1IvUGxsrCQpPDy82LDw8HAlJSUpJyfHlL+JW5IXXnhBn376qSTJw8ND9913n6ZOnSrp0plXHx8f3X///brjjjsUEhKiv/76SxMnTlSnTp20fv16tWnT5qzzunbt2vLuNiqR0qwrsbGxqlmzpiwWS7E6STp69GjZdxjAJevrr79WTEyMxowZI6lybZ8IqRcoKytLkkoMZr6+vq4aswe3QsOGDdMDDzygo0ePas6cOSooKFBubq6kS2deO3TooA4dOrie9+zZUw888IBatWqlkSNH6qeffjrrvBYOB0pSmnXldOvMyXUAcD727Nmjp59+WjfccIP69+8vqXJtnzjdf4H8/PwkSTk5OcWGZWdnu9VUBldeeaVuueUW9evXT4sWLVJ6erp69OghwzAuuXk92RVXXKG7775bq1atUkFBwVnntbLOJ8pHadYVPz+/S3KdAlCxjh07pjvvvFM2m01z586Vp6enpMq1fSKkXqDCQ96Fh89PFhsbq+DgYNMfWTyTBx54QJs3b9a+ffsu+XmtW7eucnNzlZGRcdZ5rVWrVnl3D5VIadaV8PBwHTt2TIZhFKuTxLIGoNRSU1PVvXt3paSk6KeffnLbjlSm7RMh9QLVrl1boaGhbjfdFNq0aZNat25d/p26iAoP5aempl7y83rw4EH5+voqMDBQLVq0kJeXV7F5zc3N1fbt2yv9vKJslWZdad26tTIzM93uvJWk3377zTUcAM5Vdna2evTooX379mnRokVq3ry52/DKtH0ipF4E999/vxYtWqSoqChX2y+//KJ9+/apV69eFdizcxcXF1esLS8vT1999ZX8/PxcC/mlMK8l/aLGH3/8oYULF+q2226Th4eHbDabbrnlFs2aNUtpaWmuupkzZyo9Pb3SzCsqzrmuK3fffbe8vb310UcfudoMw9Ann3yi2rVru10/DQBnUlBQoAcffFAbNmzQd999pxtuuKHEusqyfbIYpx7DhZupU6cqJSVFR48e1ccff6z77rtPbdq0kSQ9++yzstlsioqKUps2bVS1alU999xzSk9P14QJE1SnTh1t3ry5UpwCv/fee2W329W5c2fVrl1bx44d09dff609e/Zo0qRJGj58uCRdEvN68803y8/PTx06dFCNGjX0119/6bPPPpO3t7c2bNjg+qqNrVu3qkOHDmrevLkGDRqk6OhoTZo0SZ07d9ayZcsqeC5QkS72duFf//qXJkyYoEGDBunaa6/V/PnztXjxYn399dfq06dPRc0mgEpm2LBhev/999WjRw/17t272PC+fftKKt2+vEK3T2X6UwGXgHr16hmSSnwcOnTIVbdz507jtttuM/z9/Y2qVasajzzyiHHs2LGK63gpffPNN8Ytt9xi1KxZ0/Dy8jKqVatm3HLLLcaCBQuK1Vb2eX3//feN9u3bG8HBwYaXl5cRHh5u9O3b19i/f3+x2rVr1xodOnQwfH19jdDQUOPpp5827HZ7BfQaZnKxtwsFBQXG2LFjjXr16hk+Pj7GVVddZcyaNasc5wjApaBLly6n3TadGvkqw/aJI6kAAAAwHa5JBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApkNIBQAAgOkQUgEAAGA6hFQAAACYDiEVAAAApvP/VBMLpeFdQ3wAAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 800x500 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# top1 accuracy\n",
+    "CompoundT5 = [0, 0, 0, 0, 0]\n",
+    "ReactionT5 = [92.8, 92.8, 92.9, 93.0, 93.2]\n",
+    "T5Chem = [0.5, 0.2, 0.2, 0.1, 0.4][::-1]\n",
+    "\n",
+    "\n",
+    "# plot\n",
+    "import matplotlib.pyplot as plt\n",
+    "fig, ax = plt.subplots(1, figsize=(8, 5))\n",
+    "\n",
+    "\n",
+    "ax.plot([10,30,50,100,200], ReactionT5, \"o-\", label='ReactionT5', color='red', alpha=0.7)\n",
+    "ax.plot([10,30,50,100,200], CompoundT5, \"s--\", label='CompoundT5', color='blue', alpha=0.7)\n",
+    "ax.plot([10,30,50,100,200], T5Chem, \"v:\", label='T5Chem', color='green', alpha=0.7)\n",
+    "\n",
+    "\n",
+    "plt.ylim(-5, 100)\n",
+    "ax.set_xticks([10,30,50,100,200])\n",
+    "ax.set_xticklabels([10,30,50,100,200], fontsize=12)\n",
+    "# ax.set_yticks([10,20,30,40,50,60])\n",
+    "ax.set_yticklabels([int(i) for i in ax.get_yticks()], fontsize=12)\n",
+    "# plt.tight_layout()\n",
+    "ax.legend(loc=\"best\", fontsize=12)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "818bcb61",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/tmp/ipykernel_2056154/1623126519.py:21: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
+      "  ax.set_yticklabels([int(i) for i in ax.get_yticks()], fontsize=12)\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "<matplotlib.legend.Legend at 0x7f69c90f7810>"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAp4AAAGwCAYAAAAAOGVIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB4vklEQVR4nO3dd3xT5f4H8M9J2ySd6aaDltKyKdiyFGTIFJnK0ovIUEQcgKBehSsyVEAZghcHXESUoQIORESm/ECGyJIhCJRZWuiiSVfapjm/Px6TNqSFziRtP+/XKy+S5zw5+Z7SJt88U5JlWQYRERERURVT2DsAIiIiIqodmHgSERERkU0w8SQiIiIim2DiSUREREQ2wcSTiIiIiGyCiScRERER2QQTTyIiIiKyCWd7B3AvRqMRCQkJ8PT0hCRJ9g6HiIiIiO4gyzIyMjIQEhIChaLkdk2HTzwTEhIQFhZm7zCIiIiI6B6uX7+OunXrlnjc4RNPT09PAOJCvLy87BwNEREREd1Jp9MhLCzMnLeVxOETT1P3upeXFxNPIiIiIgd2r2GRnFxERERERDbBxJOIiIiIbIKJJxERERHZBBNPIiIiIrIJJp5EREREZBMOP6udiIioNigoKEB+fr69wyCy4OzsDCcnp0rbxIeJJxERkR3JsoybN28iPT3d3qEQFcvJyQmBgYHQaDQVTkCZeBIREdmRKekMDAyEm5sbt4cmhyHLMgwGA3Q6HRITE5GTk4Pg4OAKnZOJJxERkZ0UFBSYk04/Pz97h0NULE9PT6hUKqSkpCAwMBBOTk7lPhcnFxEREdmJaUynm5ubnSMhujt3d3fIslzhcchs8QSQkp0CrV5b4nGNWgN/N38bRkRERLUJu9fJ0XFyUSXJL8jH+J/G49LtSyXWifSJxFeDv4KLk4sNIyMiIiKqWWp9V7uzwhmhnqHQ5mrhrfa2umlztQj1DIWzotbn6EREREQVUusTT0mSMCpmFDQqDQxGA9yV7lA7q+GudIfBaIBGpcGomFHsBiEiIqolZs6cyc/9KlLrE08AiA2KRafwTkjKSoJslHEm6Qz+SvoLCRkJ6BTeCbFBsfYOkYiIqNpZtWoVJEky35ydnREaGorRo0fjxo0bdo0tOzsbM2fOxJ49e2z+2hERERY/l5Juq1atumv98ePH2zz2imL/MQpbPfdd24ekrCTkFuQiOz8boV6h5tbOAmMBnBTlXz6AiIjIZoxG4MwZ4PZtwMcHaN4cUNivrWn27NmoX78+9Ho9Dh06hFWrVuG3337D6dOnoVar7RJTdnY2Zs2aBQB46KGHLI69+eabeOONN6rstRcvXozMzEzz459//hlfffUVPvjgA/j7F05m7tChg/l+TEwMXnnlFYvzNGrUqMpirCpMPP9havXcFrcNLeu0xPnU8+hSr4u5tfPdfe8CAEbHjEa4JtyeoRIREZXswAFg6VLg7FkgNxdQqYCmTYGXXgKKJDK29Mgjj6BNmzYAgLFjx8Lf3x/vvfcefvzxRwwbNswuMd2Ns7MznJ2rLkV69NFHLR7fvHkTX331FR599FFEREQU+5zQ0FCMGDGiymKyFXa1/8PU6unm4oZbWbfg6+prbu1My0nDkYQj+CPhD7goOLOdiIgc1IEDwKuvAseOAd7eQESE+Pf4cVF+4ICdAxQ6deoEAIiLizOXnTt3DkOGDIGvry/UajXatGmDH3/80eJ5aWlpePXVV9GiRQt4eHjAy8sLjzzyCP7880+r19Dr9Zg5cyYaNWoEtVqN4OBgDBo0CHFxcbhy5QoCAgIAALNmzTJ3Xc+cORNA8WM8DQYD3n77bURFRUGlUiEiIgLTpk1Dbm6uRb2IiAj069cPv/32G9q1awe1Wo3IyEh8+eWXFf655eXlISsrq8LnsScmnkWYWj1v59y2GNvp6+qLT/p+ghfavIBgz8Ktoj4//jnm75+PK+lX7BQxERHVOLIM6PVlv2VnA0uWAKmpQGQk4OYGSJL4t359Uf7hh6JeWc8ty5V6iVeuXAEA+Pj4AADOnDmDBx54AGfPnsUbb7yBhQsXwt3dHY8++ii+//578/MuXbqEH374Af369cOiRYvw2muv4dSpU+jSpQsSEhLM9QoKCtCvXz/MmjULrVu3xsKFCzFp0iRotVqcPn0aAQEB+OSTTwAAjz32GFavXo3Vq1dj0KBBJcY8duxYvPXWW2jVqhU++OADdOnSBXPnzsUTTzxhVffixYsYMmQIevbsiYULF8LHxwejR4/GmTNnyv0z2717N9zc3ODh4YGIiAgsWbKk3OeyJ3a1FyFJEsbEjkFWfhbGxI6x+LYT6hWKUK9Q8+O8gjxsi9uGrPwsdI/sjgjvCDtETERENU5uLjB0aNmfp9MBJ04Azs5Aerr1cYMB2LoVeOQRwMurbOfesAGowFhMrVaLlJQU6PV6/P7775g1axZUKhX69esHAJg0aRLCw8Pxxx9/QKVSAQBeeOEFdOzYEa+//joee+wxAECLFi1w/vx5KIqMV33qqafQpEkTfPbZZ5g+fToA4Msvv8SuXbuwaNEiTJ482Vz3jTfegCzLkCQJQ4YMwfPPP4+WLVveswv7zz//xBdffIGxY8fif//7nzm+wMBALFiwAL/++iu6du1qrv/3339j79695pbdYcOGISwsDJ9//jkWLFhQ5p9fy5Yt0bFjRzRu3BipqalYtWoVXn75ZSQkJOC9994r8/nsiYnnHWKCYrBiwIp71lM6KTGn+xzsvbrXYtb7jrgd+Dv1b/Rr1I/JKBER2U5+vphUVNI+2k5OQF6eqGdjPXr0sHgcERGBNWvWoG7dukhLS8Pu3bsxe/ZsZGRkICMjw1zv4YcfxowZM3Djxg2Ehoaak1KgcJ97Dw8PNG7cGMeOHTMf+/bbb+Hv748JEyZYxVKeZZJ+/vlnAMCUKVMsyl955RUsWLAAW7ZssUg8mzVrZk46ASAgIACNGzfGpUslb1ZzN3cOORgzZgweeeQRLFq0CBMmTEDdunXLdV57YOJZAZE+kYj0iTQ/lmUZP5z7Add011BPU4+JJxERlZ1KJVoYy+r0aWDsWECjATw8rI9nZgJarehuj44ue0wV8NFHH6FRo0bQarVYuXIl9u7da04iL168CFmWMX36dHOL5Z2SkpIQGhoKo9GIJUuW4OOPP8bly5dRUFBgruPn52e+HxcXh8aNG1faBKGrV69CoVCgQYMGFuVBQUHw9vbG1atXLcrDw60nIfv4+OD27duVEo8kSZg8eTK2bduGPXv2VKtJR0w8K9kLbV/Atrht6Fa/m7ns1K1T+P3G73ikwSMW3fVERERWJKl83dqtWgHNmomJRJ6e4jwmsgwkJYk6rVrZfGmldu3amWe1P/roo+jYsSOGDx+Ov//+G0ajEQDw6quv4uGHHy72+aaEb86cOZg+fTqefvppvP322/D19YVCocDLL79sPk9VKm1rqVMJrc5yJY6VDQsLAyAmXFUnTDwrkSRJaB7YHM0Dm1uU//j3jzh04xCMshHjWo+zU3RERFSjKRRiyaRXXwXi4oCgIMDVFcjJAW7eFOt5vviiXdfzBERSNnfuXHTt2hVLly7F008/DQBwcXGx6pK/08aNG9G1a1d89tlnFuXp6ekW619GRUXh999/R35+Plxcil+Npixd7vXq1YPRaMSFCxfQtGlTc/mtW7eQnp6OevXqlfpclcXUbW+anV9dcFa7DfRu0BttQ9rikQaPmMtuZt7E2pNrkZKdYsfIiIioRunQAViwAIiNFROMrlwR/7ZqJcrttI7nnR566CG0a9cOixcvhpeXFx566CEsW7YMiYmJVnWTk5PN952cnKxaDTds2GC1C9LgwYORkpKCpUuXWp3P9Hw3NzcAImm9lz59+gAQC78XtWjRIgBA375973mO8kpLS7MYUgAA+fn5mDdvHpRKpcXY0uqALZ420DqkNVqHtLYo++XiL/j27Le4kHYBMx+aaZ/AiIio5unQAXjgAYfauag4r732GoYOHYpVq1bho48+QseOHdGiRQs8++yziIyMxK1bt3Dw4EHEx8eb1+ns168fZs+ejTFjxqBDhw44deoU1q5di8jISItzjxw5El9++SWmTJmCw4cPo1OnTsjKysLOnTvxwgsvYODAgXB1dUWzZs3wzTffoFGjRvD19UV0dDSiixn/et9992HUqFFYvnw50tPT0aVLFxw+fBhffPEFHn300SpN/n788Ue88847GDJkCOrXr4+0tDSsW7cOp0+fxpw5cxAUFFRlr10VmHjaSbOAZjifet6iFVRv0GPrha3oHtkdXqoyLnVBRERkolAALVrYO4q7GjRoEKKiorBgwQI8++yzOHLkCGbNmoVVq1YhNTUVgYGBiI2NxVtvvWV+zrRp05CVlYV169bhm2++QatWrbBlyxar7S2dnJzw888/491338W6devw7bffws/Pz5zcmqxYsQITJkzA5MmTkZeXhxkzZhSbeJrqRkZGYtWqVfj+++8RFBSEqVOnYsaMGVXzA/pHixYt0KxZM6xZswbJyclQKpWIiYnB+vXrMbQ8y27ZmSRX5kjXKqDT6aDRaKDVauFV1nXHqpkdcTvw4eEPEe4VjqV9lpZryQciIqo+9Ho9Ll++jPr169ttz3Ki0rjX72pp8zXHanev5bzV3mjo2xA9InuYk05ZlvHr5V+RnZ9t5+iIiIiIKqbMiefo0aPNe5oWdys6wPfAgQPo2LEj3NzcEBQUhIkTJyIzM7NSL6AmaRvaFoseXoSBTQaay84kn8GiQ4sw/qfxMBgNdoyOiIiIqGLKPMbzueees1ruQJZljB8/HhEREQgNFetUnjhxAt27d0fTpk2xaNEixMfHY8GCBbhw4QK2bt1aOdHXUAqp8PtAriEXdT3ronlgczgrCv+7Tt46iSb+TaB0UtojRCIiIqIyK3Pi2b59e7Rv396i7LfffkN2djaefPJJc9m0adPg4+ODPXv2mPv6IyIi8Oyzz2L79u3o1atXBUOvHVqHtEar4FbILcg1l6Vkp+DN3W/CQ+mBZf2WwVPlaccIiYiIiEqnUsZ4rlu3DpIkYfjw4QDEANMdO3ZgxIgRFgNMR44cCQ8PD6xfv74yXrbWkCQJaufCgbw3M2/Cz80P9TT1LJLO69rrKDAWFHcKIiIiIrur8HJK+fn5WL9+PTp06ICIiAgAwKlTp2AwGMzbY5mYlgA4fvx4iefLzc1Fbm5h655Op6toiDVOdGA0PhvwGdL16eayvII8vL7zdSidlHi327vcmpOIiIgcToUTz23btiE1NdWim92080BwcLBV/eDgYOzbt6/E882dOxezZs2qaFg1nkJSwNfV1/w4XhcPhaSAJEkI8ihcTPZ2zm14q73Ns+RTslOg1WtLPK9GrYG/m3+Jx4mIiIjKq8KJ57p16+Di4oJhw4aZy3JycgAAKpXKqr5arTYfL87UqVMxZcoU82OdToewsLCKhlnjRfpE4vOBnyMxMxFOCicAYtLXzD0zoTfo8WqHVxHhHYHxP43HpduX7nqerwZ/BRen4ve2JSIiIiqvCiWemZmZ2LRpEx5++GH4+fmZy11dXQHAosvcRK/Xm48XR6VSFZuw0r25OLkgXBNufpyak4rEzEQYjAYEeQTBWeGMUM9QHEs8hghNBHDH+vRXtVcR6hlqMXueiIiIqLJUaHLRDz/8YDWbHSjsYjd1uReVmJiIkJCQirwslZK/mz++fOxLzHxoJjxVnpAkCaNiRiG3IBfnUs/BKBvhrnSHu9IdBqMBGpUGo2JGccckIiIiqhIVSjzXrl0LDw8PDBgwwKI8Ojoazs7OOHLkiEV5Xl4eTpw4gZiYmIq8LJWB2lmNlnVamh839W8Kdxd36HJ1UPzz3y/LMpKyktApvBNig2LtFSoRERHVcOVOPJOTk7Fz50489thjcHNzszim0WjQo0cPrFmzBhkZGeby1atXIzMzs1pual9TuLq44rMBnyHYIxgGWeyEpMvVwc3Fja2dREREDk6SJMycOdPeYZRbuRPPb775BgaDwaqb3eTdd99FWloaunTpgk8//RRvvvkmXnrpJfTq1Qu9e/cud8BUcR3DO6JPwz5IykqCLMtIyEhA6+DWbO0kIqIqERcXh+eeew6RkZFQq9Xw8vLCgw8+iCVLltx1wjHd3d22MC9627Nnz13rz5s3z2Yxl3sWydq1axEYGGi1faZJq1atsHPnTrz++uuYPHkyPD098cwzz2Du3LnlDpYqh2ms575r+3Am6Qxu62+bx4ASERFVpi1btmDo0KFQqVQYOXIkoqOjkZeXh99++w2vvfYazpw5g+XLl9s7zGpp9erVFo+//PJL7Nixw6q8adOm5vs9e/bEyJEjLY7Hxtqu4anciefBgwfvWadjx47Yv39/eV+CqlBsUCw6hXfChr82wMfVBx4uHjDKRot94omIqHpJTgbutu+KlxcQEGC7eC5fvownnngC9erVw+7duy3W937xxRdx8eJFbNmyxXYB1TAjRoyweHzo0CHzzpEladSo0V2PVzVmGbWUJEkYEzsGfRr2wad9P8XsbrOZdBIRVWPJycDw4cDQoSXfhg8X9Wzl/fffR2ZmJj777LNiN5Vp0KABJk2aBAAwGAx4++23ERUVBZVKhYiICEybNs1qacaIiAj069cPe/bsQZs2beDq6ooWLVqYu5O/++47tGjRAmq1Gq1bt7baLXH06NHw8PDApUuX8PDDD8Pd3R0hISGYPXs2ZFm2qJuVlYVXXnkFYWFhUKlUaNy4MRYsWGBR78qVK5AkCatWrbK6vjvHY86cOROSJOHixYsYPXo0vL29odFoMGbMGGRnZ1s8Nzc3F5MnT0ZAQAA8PT0xYMAAxMfH3/NnXho5OTnQ6/WVcq6yYqZRi8UExWDFgBV4uMHD9g6FiIgqSKcDUlMBlQrw9ra+qVTiuC13ot68eTMiIyPRoUOHe9YdO3Ys3nrrLbRq1QoffPABunTpgrlz5+KJJ56wqnvx4kUMHz4c/fv3x9y5c3H79m30798fa9euxeTJkzFixAjMmjULcXFxGDZsGIxGo8XzCwoK0Lt3b9SpUwfvv/8+WrdujRkzZmDGjBnmOrIsY8CAAfjggw/Qu3dvLFq0CI0bN8Zrr71msdFNeQwbNgwZGRmYO3cuhg0bhlWrVlnt2jh27FgsXrwYvXr1wrx58+Di4oK+fftW6HUBYNWqVXB3d4erqyuaNWuGdevWVficZSI7OK1WKwOQtVqtvUOp8XINufL5lPP2DoOIqNbIycmR//rrLzknJ6eYYyXfcnOt6545I8v33SfLXbrIcp8+lrd+/WS5a1dZjo0V9Up7Xr2+/Ndm+vweOHDgPeueOHFCBiCPHTvWovzVV1+VAci7d+82l9WrV08GIB84cMBctm3bNhmA7OrqKl+9etVcvmzZMhmA/Ouvv5rLRo0aJQOQJ0yYYC4zGo1y3759ZaVSKScnJ8uyLMs//PCDDEB+5513LGIaMmSILEmSfPHiRVmWZfny5csyAPnzzz+3ui4A8owZM8yPZ8yYIQOQn376aYt6jz32mOzn52f183jhhRcs6g0fPtzqnEW9+OKL8t1Suw4dOsiLFy+WN23aJH/yySdydHS0DED++OOPS3yOyd1+V2W59Pkat6ghAMAN3Q28sesNFBgLsHLgSqid1fYOiYioVrvbyoNt2gBFGucwYgSQlgZcvgw4O4ubiacnUGRuCd54A7ijR9msYUNg0aLCxy+8AHz2Wfni1/3TtOrp6XnPuj///DMAWLUkvvLKK1iwYAG2bNmCrl27msubNWuG9u3bmx/ff//9AIBu3bohPDzcqvzSpUt46KGHLM790ksvme9LkoSXXnoJW7Zswc6dO/HEE0/g559/hpOTEyZOnGgV08aNG7F161aLc5TF+PHjLR536tQJ33//PXQ6Hby8vMw/jztf++WXX65QC+Wd826efvpptG7dGtOmTcPo0aPvurNkZWFXOwEAgj2D4ersCrWzGgkZCfYOh4iIqjkvLy8AsFjPuyRXr16FQqFAgwYNLMqDgoLg7e2Nq1evWpQXTS4BsX44AISFhRVbfvv2bYtyhUKByMhIi7JGjRoBEGM2TTGFhIRYJc6mGeJ3xlQWd8bv4+NjEafp5xEVFWVRr3HjxuV+zeIolUq89NJLSE9Px9GjRyv13CVhiycBABSSArMemoUA9wDu1U5E5AA2bCj5mOKOZqM1a4BLl8TkIW9vwN295OfOmwfckXOVeN6PPy5VqMXy8vJCSEgITp8+XernlHZZPycnpzKVyyU18VaCkmIuKCgo8Tn2iLMkpmQ9LS3NJq/HFk8yC/YMZtJJROQg1OqSb0qldV2VSiSOxd2KUqlKf16VqmLX0K9fP8TFxd1zCcZ69erBaDTiwoULFuW3bt1Ceno66tWrV7FA7mA0GnHp0iWLsvPnzwMQs+ZNMSUkJFi12J47d858HChsrUxPT7eoV5EWUdPPIy4uzqL877//Lvc5S2L6OQTYaJ0tJp5kRZZlnE0+a5dvXkREVDE5OUBWlvXNHhsE/fvf/4a7uzvGjh2LW7duWR2Pi4vDkiVL0KdPHwDA4sWLLY4v+mfAaWXM5r7T0qVLzfdlWcbSpUvh4uKC7t27AwD69OmDgoICi3oA8MEHH0CSJDzyyCMARMuuv78/9u7da1Hv4wo0F5vO/eGHH1qU3/nzKYvkYtbRysjIwOLFi+Hv74/WrVuX+9xlweYtsiDLMmbumYljN49heufpaBfazt4hERFRKXh5AX5+YsmkO5a+NPPzE/VsJSoqCuvWrcPjjz+Opk2bWuxcdODAAWzYsAGjR4/GpEmTMGrUKCxfvhzp6eno0qULDh8+jC+++AKPPvqoxcSiyqBWq/HLL79g1KhRuP/++7F161Zs2bIF06ZNM7f89e/fH127dsV//vMfXLlyBffddx+2b9+OTZs24eWXX7YYfzl27FjMmzcPY8eORZs2bbB3715zC2p5xMTE4F//+hc+/vhjaLVadOjQAbt27cLFixfLfc6PPvoIP/zwA/r374/w8HAkJiZi5cqVuHbtGlavXg3lnc3dVYSJJ1mQJAn1ferjVNIpJGYk2jscIiIqpYAAYN06x9q5CAAGDBiAkydPYv78+di0aRM++eQTqFQqtGzZEgsXLsSzzz4LAFixYgUiIyOxatUqfP/99wgKCsLUqVMt1tasLE5OTvjll1/w/PPP47XXXoOnpydmzJiBt956y1xHoVDgxx9/xFtvvYVvvvkGn3/+OSIiIjB//ny88sorFud76623kJycjI0bN2L9+vV45JFHsHXrVgQGBpY7xpUrVyIgIABr167FDz/8gG7dumHLli1WE6hK68EHH8SBAwewYsUKpKamwt3dHe3atcPKlSvRrVu3csdZVpLs4P2pOp0OGo0GWq3WPEOOqlZGbgYK5AJ4q73tHQoRUY2m1+tx+fJl1K9fH2o1l7GzhdGjR2Pjxo3IzMy0dyjVyr1+V0ubr7HFk6x4qu695hoRERFRWXFyEd1VSnYKkrNsuLEvERER1VhMPKlEP1/4GWN/HIvVJ1fbOxQiIiKqAZh4Uoka+jZEgVyA2zm3YZSN9g6HiIiowlatWsXxnXbEMZ5UooZ+DfFRn48Qrgm/d2UiIiKie2CLJ90Vk04iIiKqLEw8qVTyCvIQlxZ374pEREREJWDiSfd0Q3cDY38ci+m/TofeoLd3OERERFRNMfGkewr2DIbaWQ2VswoJGQn2DoeIiIiqKU4uontSSArM6DIDdTzqwFnBXxkiIiIqH2YRVCqhXqH2DoGIiIiqOXa1U5n9nfI3ZFm2dxhERERUzTDxpFKTZRkz98zEqztexZGEI/YOh4iIqER79uyBJEnYuHGjvUOhIph4UqlJkoR6mnpwVjhzkhEREZVIkqRS3fbs2XPX+vPmzSv2/Hv27MGgQYMQFBQEpVKJwMBA9O/fH999950Nr5LKg2M8qUwGNxuMgU0GwtfV196hEBHRHVKyU6DVa0s8rlFr4O/mX+VxrF692uLxl19+iR07dliVN23a1Hy/Z8+eGDlypMXx2NhYq3PPmDEDs2fPRsOGDfHcc8+hXr16SE1Nxc8//4zBgwdj7dq1GD58eCVeDVUmJp5UJl4qL3uHQERExcgvyMf4n8bj0u1LJdaJ9InEV4O/gouTS5XGMmLECIvHhw4dwo4dO6zKi2rUqNFdjwPAxo0bMXv2bAwZMgTr1q2Di0vhdbz22mvYtm0b8vPzKxY8VSl2tVO5pWanIiU7xd5hEBERAGeFM0I9Q6HN1cJb7W110+ZqEeoZ6tDL4uXk5ECvL3mjkunTp8PX1xcrV660SDpNHn74YfTr18+izGg04t1330XdunWhVqvRvXt3XLx40eq5v//+O3r37g2NRgM3Nzd06dIF+/fvt6gzc+ZMSJKE8+fPY8SIEdBoNAgICMD06dMhyzKuX7+OgQMHwsvLC0FBQVi4cGE5fxI1FxNPKpdfLv6CsZvHYvWfq+9dmYiIykxv0ENv0FusImIwGqA36JFfkG9VN7cgFyPvGwmNSgOD0QA3Fze4OrvC1cUVBqMBGpUGo2JGIbcgt8Tz5hXkWZw315BbtRdZxKpVq+Du7g5XV1c0a9YM69atszh+4cIFnDt3Do8++ig8PT1Lfd558+bh+++/x6uvvoqpU6fi0KFDePLJJy3q7N69G507d4ZOp8OMGTMwZ84cpKeno1u3bjh8+LDVOR9//HEYjUbMmzcP999/P9555x0sXrwYPXv2RGhoKN577z00aNAAr776Kvbu3Vu+H0gN5bhfe8ihRflEwWA0ICU7BUbZCIXE7zBERJVp6IahAIA1j62BRq0BAHx39jusPrkavSJ7YcL9E8x1R3w3ArkFuVjRfwU6hXfCtrhtyM7PxnXddfiqfSFDxsNRDyM2KBYjvh8BXa4OH/X5COGacADArku7sPSPpbg/9H682flN83lf2PICPhv4WZVfa4cOHTBs2DDUr18fCQkJ+Oijj/Dkk09Cq9Xi+eefBwCcPXsWANCiRYsynVuv1+PEiRNQKpUAAB8fH0yaNAmnT59GdHQ0ZFnG+PHj0bVrV2zduhWSJAEAnnvuOTRv3hxvvvkmtm/fbnHOdu3aYdmyZQCAcePGISIiAq+88grmzp2L119/HQDwr3/9CyEhIVi5ciU6d+5c/h9ODcPEk8qloV9D/PeR/yLCO8LeoRAR0T8kScKomFHYd20fsvOzAQD6Aj181D4YFTPKnFQ5mju7tJ9++mm0bt0a06ZNw+jRo+Hq6gqdTgcAZWrtBIAxY8aYk04A6NSpEwDg0qVLiI6OxokTJ3DhwgW8+eabSE1NtXhu9+7dsXr1ahiNRigUhQ0sY8eONd93cnJCmzZtEB8fj2eeecZc7u3tjcaNG+PSpZLH3NZGTDyp3Jh0EhFVnQ1DNwAAVE4qc9mgpoMwoPEAOElOFnXXDFpjrhvgFoBO4Z3wy8Vf0DqoNeJux6FTeCfEBokZ4p8N+MzqvN0ju6NLRBer3quP+35c+RdWCkqlEi+99BLGjx+Po0ePomPHjvDyEpNbMzIyynSu8PBwi8c+Pj4AgNu3bwMQXfgAMGrUqBLPodVqzc8r7pwajQZqtRr+/v5W5Xcms7UdE0+qsPyCfFzXXUekT6S9QyEiqjHUzmqrMmeFc7GTg+6sa2r1vJF5A+5Kd4vWzrKcV+WssiqzlbCwMABAWloaAKBJkyYAgFOnTpXpPE5OTsWWm8a4Go1GAMD8+fMRExNTbF0PD497nvNer0MCE0+qkBu6G5i2exryC/KxcuDKYt/QiIjItmKDYtEpvBO+P/c9HmvymLm1szoxdVEHBAQAEMstNW7cGJs2bcKSJUusksHyioqKAgB4eXmhR48elXJOKhlnhFCFBHkEQeWkgouTC3czIiJyEJIkYUzsGPSK6oUxsWMcdmwnACQnJ1uVZWRkYPHixfD390fr1q3N5bNmzUJqairGjh0Lg8Fg9bzt27fjp59+KtPrt27dGlFRUViwYAEyMzNLFR+VH1s8qUKcFE6Y0WUGAt0Dq3xBYiIiKr2YoBisGLDC3mHc00cffYQffvgB/fv3R3h4OBITE7Fy5Upcu3YNq1evtpgY9Pjjj+PUqVN49913cfz4cfzrX/8y71z0yy+/YNeuXVbLMN2LQqHAihUr8Mgjj6B58+YYM2YMQkNDcePGDfz666/w8vLC5s2bK/uyay0mnlRhoV6h9g6BiIiqqQcffBAHDhzAihUrkJqaCnd3d7Rr1w4rV65Et27drOq/88476NatGz788EN88sknSEtLg4+PDx544AFs2rQJAwYMKHMMDz30EA4ePIi3334bS5cuRWZmJoKCgnD//ffjueeeq4zLpH9IsoOPetXpdNBoNNBqteYZbeS4LqReQAPfBg7drUNE5Cj0ej0uX76M+vXrQ63mGHlyXPf6XS1tvsYxnlQpZFnG7P+bjSnbp+BIwhF7h0NEREQOiIknVQpJklDXqy6cFc6I18XbOxwiIiJyQBzjSZVmSLMhGNh4IPzc/OwdChERETmgcrV4Hjt2DAMGDICvry/c3NwQHR2NDz/80KLOgQMH0LFjR7i5uSEoKAgTJ04sdpkCqjm8VF5MOomIiKhEZW7x3L59O/r374/Y2FhMnz4dHh4eiIuLQ3x8YffqiRMn0L17dzRt2hSLFi1CfHw8FixYgAsXLmDr1q2VegHkmNJy0mCUjfB38793ZSIiIqoVypR46nQ6jBw5En379sXGjRuhUBTfYDpt2jT4+Phgz5495plNERERePbZZ7F9+3b06tWr4pGTw/rl4i9YdnQZutTrgpcfeNne4RAROTwHX2CGqNJ+R8vU1b5u3TrcunUL7777LhQKBbKyssx7nJrodDrs2LEDI0aMsJhOP3LkSHh4eGD9+vWVEjg5rkifSBiMBiRlJcEoG+/9BCKiWsrZWbT/FLcLD5Ejyc/PB1DynvSlVabEc+fOnfDy8sKNGzfQuHFjeHh4wMvLC88//zz0ej0A4NSpUzAYDGjTpo3Fc5VKJWJiYnD8+PEKBUyOr5FfI3zY+0PM6T4HCokLJxARlcTJyQlOTk7Q6XT2DoWoRLIsQ6vVQqVSwcWlYrsUlqmr/cKFCzAYDBg4cCCeeeYZzJ07F3v27MF///tfpKen46uvvkJiYiIAIDg42Or5wcHB2Ldv311fIzc3F7m5uebH/GOsnur71Ld3CEREDk+SJAQGBiIxMREqlQru7u7cgIMchizLyM/Ph1arRWZmJkJDK75TYZkSz8zMTGRnZ2P8+PHmWeyDBg1CXl4eli1bhtmzZyMnJwcAoFKprJ6vVqvNx0syd+5czJo1qyxhkQMzGA2I18UjwjvC3qEQETkkjUaDnJwcpKSkIDk52d7hEFlRqVQIDQ2tlB0ky5R4urq6AgD+9a9/WZQPHz4cy5Ytw8GDB+Hm5gYAFq2WJnq93nyOkkydOhVTpkwxP9bpdAgLCytLmOQgEjIS8J/d/0FeQR5WDlgJlbP1lxEiotpOkiQEBwcjMDDQPI6OyFE4OTlVuHu9qDIlniEhIThz5gzq1KljUR4YGAgAuH37NqKiogDA3OVeVGJiIkJCQu76GiqVqtjWUqp+6rjXgZPkBCfJCfG6eET5Rtk7JCIih2Ua70lUk5Vp5kfr1q0BADdu3LAoT0hIAAAEBAQgOjoazs7OOHLEcr/uvLw8nDhxAjExMRUIl6oTJ4UTZnSZgRUDVjDpJCIiorIlnsOGDQMAfPbZZxblK1asgLOzMx566CFoNBr06NEDa9asQUZGhrnO6tWrkZmZiaFDh1ZC2FRdhGnCoHRS2jsMIiIicgBl6mqPjY3F008/jZUrV8JgMKBLly7Ys2cPNmzYgKlTp5q70d9991106NABXbp0wbhx4xAfH4+FCxeiV69e6N27d5VcCDm+uLQ4RPpEcsYmERFRLSXJZVyKPj8/H3PmzMHnn3+OhIQE1KtXDy+++CJefvlli3q//fYbXn/9dRw7dgyenp4YNmwY5s6dC09PzzIFqNPpoNFooNVqK2U2FdmeLMt4Z+87OJxwGDO6zECbkDb3fhIRERFVG6XN18qceNoaE8+aYeXxldh8fjOeavkUBjUdZO9wiIiIqBIx8SSHotVrkW/Mh7+bv71DISIiokpW2nytTGM8icpLo9bYOwQiIiKyM26kTTaXrk9HanaqvcMgIiIiG2PiSTa17eI2PL3paaw+udreoRAREZGNMfEkm4rwjkC+MR+3Mm/BKBvtHQ4RERHZEMd4kk019m+MJb2XoL53fa7nSUREVMsw8SSbi/SJtHcIREREZAfsaie7MRgNuJp+1d5hEBERkY0w8SS7SMhIwLjN4zBt9zTkGnLtHQ4RERHZABNPsos67nWgkBRQSApc1123dzhERERkAxzjSXbhpHDCW13eQpBHEJROSnuHQ0RERDbAxJPsJlwTbu8QiIiIyIbY1U4O4dLtS5Bl2d5hEBERURVi4kl2Jcsy3tn7Dib9MglHE4/aOxwiIiKqQkw8ya4kSUKIZwicJCcurURERFTDcYwn2d3gpoPRv1F/BLgH2DsUIiIiqkJMPMnuNGqNvUMgIiIiG2BXOzkUrV6LtJw0e4dBREREVYCJJzmMHXE78PSPT2P1n6vtHQoRERFVASae5DDCNeHIK8jDjYwbMMpGe4dDRERElYxjPMlhNPZvjEW9FqGBbwNIkmTvcIiIiKiSMfEkh9LQr6G9QyAiIqIqwq52ckgFxgJc016zdxhERERUiZh4ksNJyEjAuM3jMG3XNOQacu0dDhEREVUSJp7kcOq41zHfv667bsdIiIiIqDJxjCc5HCeFE97q8haCPYOhdFLaOxwiIiKqJEw8ySHV865n7xCIiIiokrGrnRzelfQrkGXZ3mEQERFRBTHxJIclyzLm7puLCVsn4FjiMXuHQ0RERBXExJMcliRJCHQPhEJS4HL6ZXuHQ0RERBXEMZ7k0IY0G4L+jfsj0D3Q3qEQERFRBTHxJIemUWvsHQIRERFVEna1U7Why9UhLSfN3mEQERFROTHxpGphR9wOjNk0BmtOrrF3KERERFROTDypWgjThCGvIA/XtNdglI32DoeIiIjKgWM8qVpo4t8EC3ouQCO/RpAkyd7hEBERUTkw8aRqo7F/Y3uHQERERBXArnaqdgqMBbiuvW7vMIiIiKiMmHhStZKYkYjxP43H1F1TkVeQZ+9wiIiIqAyYeFK1EugeCKNshFE24mr6VXuHQ0RERGXAMZ5UrTgpnDC9y3QEewRD5ayydzhERERUBkw8qdqJ8I6wdwhERERUDmXqat+zZw8kSSr2dujQIYu6Bw4cQMeOHeHm5oagoCBMnDgRmZmZlRo80XXtdciybO8wiIiIqBTK1eI5ceJEtG3b1qKsQYMG5vsnTpxA9+7d0bRpUyxatAjx8fFYsGABLly4gK1bt1YsYiIAsizjvf3vYf/1/Zj10Cy0Cm5l75CIiIjoHsqVeHbq1AlDhgwp8fi0adPg4+ODPXv2wMvLCwAQERGBZ599Ftu3b0evXr3KFy3RPyRJgr+bPyRIiEuLY+JJRERUDZR7VntGRgYMBoNVuU6nw44dOzBixAhz0gkAI0eOhIeHB9avX1/elySyMKTZECzvvxxDmw+1dyhERERUCuVKPMeMGQMvLy+o1Wp07doVR44cMR87deoUDAYD2rRpY/EcpVKJmJgYHD9+/K7nzs3NhU6ns7gRFcdb7Y0gjyB7h0FERESlVKbEU6lUYvDgwViyZAk2bdqEd955B6dOnUKnTp3MCWViYiIAIDg42Or5wcHBSEhIuOtrzJ07FxqNxnwLCwsrS4hUS2XkZiAtJ83eYRAREdFdlCnx7NChAzZu3Iinn34aAwYMwBtvvIFDhw5BkiRMnToVAJCTkwMAUKms11hUq9Xm4yWZOnUqtFqt+Xb9OrdGpLvbdWkXnv7xaaw5ucbeoRAREdFdVHgdzwYNGmDgwIH47rvvUFBQAFdXVwCiy/xOer3efLwkKpWq2KSVqCShXqHQG/S4mn4VRtkIhcQNuYiIiBxRpSwgHxYWhry8PGRlZZm72E1d7kUlJiYiJCSkMl6SyKyJfxPM7zkfjf0aQ5Ike4dDREREJaiUpqFLly5BrVbDw8MD0dHRcHZ2tphwBAB5eXk4ceIEYmJiKuMliSw08W/CpJOIiMjBlSnxTE5Otir7888/8eOPP6JXr15QKBTQaDTo0aMH1qxZg4yMDHO91atXIzMzE0OHcukbqjpG2Ygbuhv2DoOIiIiKIcll2G+wW7ducHV1RYcOHRAYGIi//voLy5cvh4uLCw4ePIimTZsCAI4dO4YOHTqgWbNmGDduHOLj47Fw4UJ07twZ27ZtK1OAOp0OGo0GWq3WYl1QojslZiRixp4ZyM7PxsqBK6F0Uto7JCIiolqhtPlamVo8H330UaSkpGDRokV44YUX8M0332DQoEE4cuSIOekEgFatWmHnzp1wdXXF5MmTsXz5cjzzzDPYuHFj+a+I6B4C3ANQYCyAUTbiavpVe4dDREREdyhTi6c9sMWTyuLy7csI9gyG2llt71CIiIhqjdLma5Uyq53IUdT3qW/vEIiIiKgEXPCQaqx4XTwcvEGfiIioVmHiSTWOLMt477f38PyW53Hi5gl7h0NERET/YOJJNY4kSfBz84MECRfSLtg7HCIiIvoHx3hSjTS46WD0bdgXwZ7B9g6FiIiI/sHEk2okH1cfe4dAREREd2BXO9V4mXmZuJ1z295hEBER1Xps8aQa7bu/vsP/jv8PbUPaYtR9o6yOa9Qa+Lv52yEyIiKi2oeJJ9VY+QX5+PTop/j9xu84mnAUm89vhgTJok6kTyS+GvwVXJxc7BQlERFR7cHEk2osZ4Uzmvo3xbmUc2jg0wB35Jy4qr2KUM9QOCv4Z0BERGQLHONJNZYkSRgVMwr+bv4wyAa4K93NN4PRAI1Kg1ExoyBJ0r1PRkRERBXGxJNqtNigWHQK74SkrCQYjUbczLgJrV6LpKwkdArvhNigWHuHSEREVGsw8aQazdTqqXRS4tjNY7imu4bEjES4ubiZWzsv3b4Eo2y0d6hEREQ1HhNPqvFig2LRNaIrVE4qBLgFwGA0mFs7EzMSMemXSRj1wygYjAZ7h0pERFSjMfGkGs/U6unr6gsnhRN8XH3MrZ0JGQlwc3GzmmS04tgKLD+6HDd0N+wYORERUc3C6bxUK5jGen5/7ns81uQx89jO1iGtsW7QOmhztea6BqMBOy7tQHZ+NrpGdDWXJ2clIyMvA/W963NCEhERUTkw8aRaQZIkjIkdg6z8LIyJHWORODopnODr6mtRf9L9k3A66TSifKPMZdvjtuPrM1+jZ2RPTLx/os1iJyIiqimYeFKtERMUgxUDVtyznrPCGR3COqBDWAeLcoPRALWzGk39m5rLMnIzMG3XNMQExWB0zGg4KZwqPW4iIqKagoknUSmNihmFJ1s+aTED/uStk7iivYICuQDPtHrGXH488Th8XH1QT1OP3fJERET/YOJJVAZ37nJ0X9B9eP3B1y2SUVmWsfTwUiRlJ2H2Q7MRGxxrLmcSSkREtRkTT6IK8FB6oGN4R4syvUGPcE04cgw5aBbQzFy+5cIWbI/bjv6N+qNnVE9bh0pERGR3TDyJKpmriytmPDQDBcYCizGfxxKP4XL6ZWTkZZjLDEYDtpzfglbBrVDXqy5bRImIqEZj4klURe6caDTp/kk4cfMEmvg3MZedTT6LFcdXwPusN7589EtzucFosOrWJyIiqu74yUZkIxq1Bl0iuliUOSmc0CqoFQLdAy1aO/+949+QZRkT7p+ASJ9IW4dKRERUJZh4EtlRs4BmmNV1lkVZZl4mLqZdhAzZYn3R00mncfn2ZbQJaYNgz2Bbh0pERFRhTDyJHIyH0gOrHl2FC6kX4K32NpfvvLQTuy7vQmpOKkbHjAYgZspn5WfBQ+lhn2CJiIjKgHu1EzkgX1df3F/3fouyJv5NcF+d+9A6uLW57Kr2KoZ/Oxz/2fUfyLJs6zCJiIjKhC2eRNVE7wa90btBb4uyC6kXIEOGi5OLxRjRdafWwVvtjY7hHeGl8rJ1qERERMVi4klUjfWM6olWwa2QnZ9tLssryMPGvzYi35iPFoEtzImnVq+Fs8IZ7kp3e4VLRES1HLvaiao5Pzc/hGnCzI8NRgP+Ff0vdKjbAXW96prLN/61EcO/G471Z9bbI0wiIiK2eBLVNG4ubhjafKhVeWJmIoyyEcEehTPi03LSsOzIMrQKboWHGzxsyzCJiKgWYuJJVEu82flNJGUlWYz5PJ54HAfiDyA5O9ki8YxLi0OIZwhcXVztESoREdVQTDyJapFA90CLx038m+DJFk/Cz9XPXCbLMmbsmYHMvEwsengRF7AnIqJKw8STqBYL9QrFE9FPWJSl5aTBzcUNuQW5CNeEm8t/vvAzTt06hYcbPIyYoBgbR0pERDUBE08isuDn5ofl/ZebZ8Gb/HbtN5xKOoUWdVqYE0+9QY/TSacRHRgNtbPaThETEVF1wcSTiIqlUWssHo+6bxSOJh5Fm5A25rJTt05h9t7ZCPUMxaf9PrV1iEREVM0w8SSiUmns3xiN/RtblOkNegS6BaJlnZYW5a/veB3+bv4YHTMaAe4BtgyTiIgcGBNPIiq3TvU6oWN4R+Qb881lSVlJ+CvlLygkBV5o+4K5/O+Uv5GVn4XmAc2hclbZI1wiIrIzJp5EVCGSJEHppDQ/9nX1xbzu83BNe81il6Tvz32P/df348kWT5onNJn2ly+63ScREdVc3LmIiCqVs8IZzQOb45GGj1iUB7gFwN/NH7FBseay86nnMeqHUVh+dLmtwyQiIjtgiycR2cQzrZ7B07FPW5Qdv3kct/W3kZaTZlH+0/mfUNerLpoHNIeLk4stwyQioirExJOIbObOLvXBTQejWUAzi6WYsvKy8L9j/4NRNuKzAZ+ZF73PNeRC6aRktzwRUTVW4a72d999F5IkITo62urYgQMH0LFjR7i5uSEoKAgTJ05EZmZmRV+SiGoIFycXtKzTEo38GpnLcgw56BrRFdEB0RY7La04tgJjNo3Bnit77BApERFVhgq1eMbHx2POnDlwd3e3OnbixAl0794dTZs2xaJFixAfH48FCxbgwoUL2Lp1a0VelohqMH83f7z8wMtW5WeSzyA1JxXuLoXvN0lZSfjl4i9oHdwazQOb2zBKIiIqjwolnq+++ioeeOABFBQUICUlxeLYtGnT4OPjgz179sDLywsAEBERgWeffRbbt29Hr169KvLSRFTLLO69GH8l/4Um/k3MZX/c+AMb/tqAcynnMKf7HHN5Wk4afNQ+7JYnInIw5e5q37t3LzZu3IjFixdbHdPpdNixYwdGjBhhTjoBYOTIkfDw8MD69evL+7JEVEspnZSICYqxGA8apglD14iu6Bje0VxmMBrw3E/P4Zkfn0FKdkpxpyIiIjspV4tnQUEBJkyYgLFjx6JFixZWx0+dOgWDwYA2bdpYlCuVSsTExOD48eMlnjs3Nxe5ubnmxzqdrjwhElEt0LJOS6tdk+J18TAYDcgtyIWfq5+5fHvcdiRlJaFLvS4I04TZOlQiIkI5E89PP/0UV69exc6dO4s9npiYCAAIDg62OhYcHIx9+/aVeO65c+di1qxZ5QmLiAgR3hH4evDXSMhIsOhq33ZxG86nnUewR7A58dQb9LidcxtBHkHslicisoEyJ56pqal46623MH36dAQEFL8Hc05ODgBApbLeFk+tVpuPF2fq1KmYMmWK+bFOp0NYGFsniKj0VM4q1Pepb1HWt1FfBCUEITa4cAH7IwlH8N7+99AmuA1mPDTjnudNyU6BVq8t8bhGrYG/m3/5AyciquHKnHi++eab8PX1xYQJE0qs4+rqCgAWXeYmer3efLw4KpWq2ISViKgiutXvhm71u1mU3cq8BWeFs0XXuyzLmLNvDiJ9IjGg8QDztp/5BfkY/9N4XLp9qcTXiPSJxFeDv+Ki90REJShT4nnhwgUsX74cixcvRkJCgrlcr9cjPz8fV65cgZeXl7mL3dTlXlRiYiJCQkIqGDYRUcUNbjYYfRv1Ra6h8EtyvC4eh24cwrGbxzC42WCLcm+1N7S5WtTT1LM611XtVYR6hsJZwX05iIhKUqZZ7Tdu3IDRaMTEiRNRv3598+3333/H+fPnUb9+fcyePRvR0dFwdnbGkSNHLJ6fl5eHEydOICYmpjKvgYio3NTOamjUGvNjH1cfTGg3AcOaDYPSSWku/+LPL3A+9TwkSDAYDXBXusPNxQ3uLu4wGA3QqDQYFTOKY0WJiO6iTF/No6Oj8f3331uVv/nmm8jIyMCSJUsQFRUFjUaDHj16YM2aNZg+fTo8PT0BAKtXr0ZmZiaGDh1aOdETEVUyD6UHekVZrjMsyzKMshHeam9E+UThYPxBeKm8kJaThri0OBhlI4a3GI7YIDF+9JeLvwAA2tdtb5HUEhHVdpIsy3JFT/LQQw8hJSUFp0+fNpcdO3YMHTp0QLNmzTBu3DjEx8dj4cKF6Ny5M7Zt21bqc+t0Omg0Gmi1Wos1QYmIbC0rLwvnUs7h2c3PwkPpgZz8HMTdjoOriys2/2szWgW3AgA89f1TSNenY0nvJYj0iQQA7L26F8uPLkf7uu3xYrsXzefce3UvnBXOaFmnJTyUHna5LiKiiiptvlbhvdpL0qpVK+zcuROurq6YPHkyli9fjmeeeQYbN26sqpckIqpS7kp3tApuhU7hnZCUlYRA90D4ufmhb8O+5tZOWZbRvm57tA1pa7GOaEp2CrS5WugNeotzLj+6HHN/m4ukrCRz2f5r+/HClhfw+fHPLeoeTTiK00mnrc5BRFRdVMoo+D179hRb3rFjR+zfv78yXoKIyCFIkoRRMaOw79o+xGfEw1PpifFtxpvHdkqShBfavmD1vIejHkZMUAxUToWrdsiyjBaBLZCSnWKRpN7KuoXruuto4NvA4hwfHPoA2lwtPuz9oXm5qN/jf8cP535AbHAshjUfZq57LuUc3FzcEOwRzFn2ROQwOP2SiKiMYoNi0Sm8E74/9z0ea/KYubXzbtyV7ohURlqUSZKE1zu+blX3oYiHEOkTCU+lp7lMlmWEa8KRmp0KX1dfc3m8Lh6nk08jwN1yXeW3974NXa4O/33kv4jwjgAgWky3xW1Dyzot0a9RP4tzeCg9oFFpODmKiKoUE08iojKSJAljYscgKz8LY2LHVHqy5uvqa5Fcml5zTvc5VnU7hHUwd/mbGGUjfNQ+5n9NLqdfxsH4gxb73QPA6ztft0pS/7z5J/Ze3YvowGh0rd/VXDddnw4PpQeXjSKicuE7BxFROcQExWDFgBX2DgPBnsEI9rTcnlghKbC0z1Kruq2DW8PV2dWivlE2wklyggTJItk9n3oe2y9tR4FcYJF4vrDlBWTkZeCjPh8hXBMOAPgr+S8cSTiCxn6NcX/d+811DUYDE1QissB3BCKiWqK+T32rrUQVkgJfPvYlCowFUEiF802jA6PxZIsnzS2ggEgkcwxiy2Nvtbe5/EzSGWz4awO61+9ukXiO/H4kjLIRix5ehBBPsXHIhdQLOJ10Gg18G6BFnRZVcJVE5MiYeBIREZwUThaPmwY0RdOAphZlzgpnfDfsO+hydRbjTxv6NUS/hv3QxL+JuSy/IB8ZeRkAYFH3+M3jWH1yNXrU72GReI7ZNAYuChe83fVt1PGoAwC4kn4FcWlxiPCOQJRvVOVdLBHZDRNPIiIqNUmSrBbFjwmKQUxQjEWZs8IZXw/+Grf1ty3WJw3zCkPn8M5oFtDMXJZXkIeU7BQAgJuLm7n88I3D5iR10gOTzOUTfp4ApZMSUztNhb+bPwDghu4GEjISEOoVam5dJSLHw8STiIgqnSRJcFe6w13pblHePqw92oe1tyhzUbhgeb/lSMtJs0hSA9wCEBsUa9HamVeQhyvaKwBgMUnqt2u/Yc2pNegZ2RMT759oLp+6cyrUzmpMvH8ifFzFRKukrCTczrmNOh51LIYMEFHVY+JJRER2JUlSsZOkutbvajGxCQCcJCcs6LkAt/W34e5SmNR6qjwR5ROFul51zWV5BXk4nSx21FM6Kc3luy/vxtpTa62S1Hf2vgM3FzeMbTUWXiqx80q6Ph05+TnwdfWFyrlwDVYiKh8mnkREVG04KZzQ2L+xVXmfhn3Qp2EfizKFpMBbnd9Cuj7dogvfReGCALcABLoHmstyDbn4/cbvAIDnWj9nLv/l4i9Ye2otekX2woT7J5jLlxxaAg+lBx6PftzcSpuVlwUZMtxd3LkeKlEJmHgSEVGN5KxwRtvQtlblg5sNxuBmgy3KJEnC5AcmWyWpBcYCqJxUFktN6Q167Ly8EwDwrxb/MpdvPr8Za0+tRe+o3nix3Yvm8s+OfQZPlSf6N+oPVxdXAGLylbPCmQkq1TpMPImIqNZTOinRrX43q/InWz6J4S2GwygbLcqfiX0GWr0Wrs6u5rLMvEwAlktN6Q16/PD3DwCA/o36m8s3/rUR35z5BgMbD8SY2DHm8vVn1sNL5YWuEV3ZtU81EhNPIiKiu5AkCU5S4XJTamc1Hm3yqFW9sa3GYuR9Iy2SVKNsxNBmQ6HL1ZlbOwEgLScNBXKBxdhTvUGP1SdXAxDbppp8ffpr/HT+J/Rr1A9PRD9hLv/5ws/wVnujTUgbi/MQOTImnkRERJXkzgTQzcUNI+8baVXvuTbP4fHox+GicDGXGYwG9I7qjYy8DIsZ+ynZKdDmaiHLsrksJz8Hnxz5BACwYegGc/m3f32LXZd34eGohzGwyUAAgCzL2H99P7zV3mji34S7SZFd8bePiIjIxpwVzuY1SE08lB4WY0NNRt03Cn0a9jHPtAfEjP0OdTsgMy/TIklNyEjAdd118w5TAJBjyMF7+98DIJJUU+K5+e/N2H99P7rX746eUT0BiCT15K2T8HH1QV2vuha7WRFVBiaeREREDsxT5QlPladFmUatwdROU63qPh79ODrX64wA9wBzmd6gR3RANLLzsy2S1CvpV3Am+YzF4v/Z+dl489c3AQAbh240jzPdHrcdRxKOoFN4J3Sq1wmASFIvp1+Gj9oH3mpvTpSiUmHiSUREVEMEugdaLBMFAL6uvpjbY65V3QGNByAmKAbhmnBzWY4hB+Fe4dAb9BaTm86lnMPB+IOI9Ik0l2XnZ2PSL2JHqW+HfWseZrDnyh6cTjqN+0PvN68qIMsyUnNS4a32Zld/Lcf/fSIiolqonnc91POuZ1Hm7+aPj/p+ZFW3Z2RPRPlEoZFfI3NZZl4mvNXeKDBaTpI6eeskdlzagQC3AHPimZWfhTGbxkCChI3DNprrH7h+ABdSL6BVcCu0qNMCgEhS8435nDBVQzHxJCIiortqGtAUTQOaWpTV8aiD1Y+ttlpq6sGwBxHoHoiWdVqay7R6LZwkJ6id1RYJ5R83/sDOyzvh6uJqTjwz8zIx/LvhcHdxx5pBa8wtpEcTjuKa9hqiA6PR0K8hAJgnXLGbv/pg4klERETlducEpNYhrdE6pLVFWahXKL5//Htk5WdZ1XV1cUUT/ybmstv62wBEMlm0W/63a79h5+WdeKrlU+bEMzMvE6M3jYafqx8+6fsJnBRi2avTSadxK/MWGvk1QpgmrPIuliqMiScRERFVOUmSzNuLmnQM74iO4R0tysK8wvD14K+hy9VZlDcNaIp8Yz6ifKLMZWk5acgryENmXqY56QSAnZd2YtflXXiq5VPmxDMjNwMTf5kIP1c/vN/zfXPCfCH1AtL16YjwjrCYlEVVg4knEREROQxJkuCudIe70t2ivFdUL/SK6mVRFuoViuX9llu1pNbT1ENMnRjU0xSOYU3NSUVKdgpyDbkWrbRbLmzBrsu7MLLlSAxtPhSASFLf3P0m/Nz8ML3zdHNX/jXtNeTk5yDEM8RqpQEqHSaeREREVC05K5wR7BlsVf5Y08fwWNPHLMpCPEOwsNdC5OTnWJQHuAUgyifK4jypOam4lH4JKTkpFuNHv/3rW+y+shuj7huFIc2GABBJ6vv734efmx8m3T/JXD8pKwkFxgL4ufnZbKJUSnYKtHpticc1ao3V+rG2xsSTiIiIajylk9JiVr7Jky2fxJMtn7QoC3QPxFud30K+Md+i3F3pjgC3AAS4FXbJp+ak4sStE/BSeVkkqWtOrsGvV37F6PtGY3CzwQBEkvrpkU/h5+aHMTFjzPW1ei2cFc5wc3Er90Sp/IJ8jP9pPC7dvlRinUifSHw1+Cu4OLmUWKeqMfEkIiIiKsLNxc28FFRR41qPw7jW4yzK/Fz9MPmBySgwFljVVzmp4Ovqa36ckp2Cvdf2QqPS4OnYp83lnx3/DL9e+RVPxzxtbqnNzMvEmpNr4O/mb25dBYBcQy6UTkqrBNVZ4YxQz1Acv3ncYoiByVXtVYR6htp9HVUmnkRERETl5KnyRLf63azKp7SfgskPTLYo06g1eCb2GfMyUCbZ+dkAAG+1t7ksKSsJWy5sgUalsUg8lx5ein3X9mFc63Ho07APACArLws//v0jGvs1xr5r+2AwGqBRa8zP0eq10Kg0GBUzyu5LTzHxJCIiIqoCdyZ5vq6+eLTJo1b13uz8JvIK8izKPJWeGNZsmNVyVbf1t1EgF8DV2dVcdivrFtadXgeNSoNO4Z2wLW6buetflmUkZSXh4aiHERsUW3kXV05MPAEkJwM6XcnHvbyAAK6wQERERFXkzglIAe4BeOq+p6zqzXxoJtL16XBzcTOXqZ3V6B3VG0onJVqHtMa+a/ugy9VBo9ZAl6uDm4ubQ7R2Akw8kZwMDB8OpKaWXMfPD1i3jsknERER2ZezwtlqZnqIZwhebPciALGbU9FWT0dq7QQAxb2r1Gw6nUg6VSrA29v6plKJ43drESUiIiJyBJIkYVTMKLi5uOG67rpDtXYCTDzNXF0Bd3frm6vrvZ9LRERE5Chig2LRKbwTbufcRqfwTg7T2gmwq71YRiOgYEpORERE1ZAkSRgTOwZZ+VkYEzvGYVo7ASaeVrKygIsXgeBgIDDQ3tEQERERlV1MUAxWDFhh7zCssF3vDjodkJsLXLsmklAiIiIiqhxMPO8QHCwmFRmNouWzwHojAiIiIiIqByae/8jJES2cWVlAUBDg5CTuX75s78iIiIiIaoZan3h6eYl1OnNzgfR0ccvIEK2eBQWAVitaP7287BwoERERUTVX6ycXBQSIxeGLW6dz927gq6/EWp5paVxAnoiIiKgian3iCYiEsrikMjJSJJy5uUBIiO3jIiIiIqpJmHjehSQBkycDLi7iPhERERGVX60f43kvSmVh0inLwKVL9o2HiIiIqLpi4llKBQXABx8AL78MnDxp72iIiIiIqp8yJZ5nzpzB0KFDERkZCTc3N/j7+6Nz587YvHmzVd2zZ8+id+/e8PDwgK+vL5566ikkJydXWuC25uQEODuLVs/588XYTyIiIiIqvTIlnlevXkVGRgZGjRqFJUuWYPr06QCAAQMGYPny5eZ68fHx6Ny5My5evIg5c+bg1VdfxZYtW9CzZ0/k5eVV7hXY0HPPARERYsml+fO5uDwRERFRWUiyLMsVOUFBQQFat24NvV6Pc+fOAQBeeOEFrFq1CufOnUN4eDgAYOfOnejZsyeWLVuGcePGlfr8Op0OGo0GWq0WXg6wmOaNG2LCUU4OMGQIMGqUvSMiIiIisq/S5msVHuPp5OSEsLAwpKenm8u+/fZb9OvXz5x0AkCPHj3QqFEjrF+/vqIvaVehocDEieL+xo3AH3/YNx4iIiKi6qJciWdWVhZSUlIQFxeHDz74AFu3bkX37t0BADdu3EBSUhLatGlj9bx27drh+PHjFYvYAXTsCPTvL+4vXgzo9XYNh4iIiKhaKNc6nq+88gqWLVsGAFAoFBg0aBCWLl0KAEhMTAQABAcHWz0vODgYaWlpyM3NhUqlKvbcubm5yM3NNT/WFbelkAMYMwa4eRMYMABQq+0dDREREZHjK1fi+fLLL2PIkCFISEjA+vXrUVBQYJ40lJOTAwDFJpbqfzK0nJycEhPPuXPnYtasWeUJy6ZcXIC33rJ3FERERETVR7m62ps0aYIePXpg5MiR+Omnn5CZmYn+/ftDlmW4uroCgEWrpYn+nz5pU53iTJ06FVqt1ny7fv16eUK0ucRE4OhRe0dBRERE5LgqZQH5IUOG4I8//sD58+fNXeymLveiEhMT4evrW2JrJyBaSr28vCxuju7KFbGw/HvviVnvRERERGStUhJPU/e6VqtFaGgoAgICcOTIEat6hw8fRkxMTGW8pEMJCwMiI8USS/PmAcU09hIRERHVemVKPJOSkqzK8vPz8eWXX8LV1RXNmjUDAAwePBg//fSTRTf5rl27cP78eQwdOrSCITseJyfgtdcAjUa0fv4z74qIiIiIiijTAvKPPfYYdDodOnfujNDQUNy8eRNr167FuXPnsHDhQkyZMgUAcP36dcTGxsLb2xuTJk1CZmYm5s+fj7p16+KPP/64a1f7nRxtAfm7OXkSePNNsa3myy8D/6wwRURERFSjlTZfK1Pi+fXXX+Ozzz7DqVOnkJqaCk9PT7Ru3RoTJkzAgAEDLOqeOXMGU6ZMwW+//QalUom+ffti4cKFqFOnTpVciKP4+mtg7VpAqQQWLhRbbBIRERHVZFWSeNpDdUs8ZRmYORM4dgx46CHglVfsHRERERFR1SptvlaudTypZJIkks3Nm4HHH7d3NERERESOg4lnFfDyAp580t5REBERETmWSllOiUpWUAB88QVw4YK9IyEiIiKyLyaeVeybb4CNG8X6npmZ9o6GiIiIyH6YeFaxgQOBoCAgKQn44AMx+YiIiIioNmLiWcXc3YGpUwEXF+DwYeC77+wdEREREZF9MPG0gchIYNw4cf/LL4EzZ+wbDxEREZE9MPG0kYcfBrp2BYxG4P33Aa3W3hERERER2RYTTxuRJOCFF4CwMCA7G7h82d4REREREdkW1/G0IbUamDZNTDAKC7N3NERERES2xcTTxurWtXwsy6I1lIiIiKimY1e7HZ0+DUycCKSk2DsSIiIioqrHxNNOZBlYuRK4ckVMNjIY7B0RERERUdVi4mknkgS89ppY5/PsWbHMEhEREVFNxsTTjoKDgUmTxP3vvwcOHbJvPERERERViYmnnbVvL7bVBIDFi4GbN+0aDhEREVGVYeLpAEaPBpo0AbKygLlzgbw8e0dEREREVPmYeDoAZ2fg9dcBT08gJETsbkRERERU03AdTwfh7w8sWSL+5bqeREREVBOxxdOBBAQUJp2yDGRk2DceIiIiosrExNMBZWUB8+aJ7ne93t7REBEREVUOJp4OKD8fOHcOuH4d+Phj0fpJREREVN0x8XRA3t5icXmFAvj1V2D7dntHRERERFRxTDwdVHQ08NRT4v6yZcClS/aNh4iIiKiimHg6sMGDgbZtRdf7vHli7CcRERFRdcXE04FJEjB5MhAYCCQmAh99ZO+IiIiIiMqPiaeD8/QUs9uDgoD+/e0dDREREVH5cQH5aqBRI+CTT8QOR0RERETVFVs8q4miSee1a4BOZ79YiIiIiMqDiWc1c/CgGPe5cCHX9yQiIqLqhYlnNRMcLP49dgzYsMG+sRARERGVBRPPaiYiAnj+eXF/zRrg5Em7hkNERERUakw8q6EePYDu3UVX+/z5QFqavSMiIiIiujcmntXU88+L1s/0dJF8FhTYOyIiIiKiu2PiWU2pVMAbbwBqNXD6NLBjh70jIiIiIro7rgxZjYWGApMmAZcvAz172jsaIiIiortj4lnNdewobkRERESOjolnDZKfD2zeDAwYwF2OiKpCcvLdN2/w8gICAmwXDxFRdcP0pAZ55x2xvmdKCjBunL2jIapZkpOB4cOB1NSS6/j5AevWMfkkIioJJxfVIP36iX83bwZ++82+sRDVNDqdSDpVKsDb2/qmUonj3M6WiKhkTDxrkLZtgSFDxP0PPwRu3LBvPEQ1kasr4O5ufXN1tXdkRESOj4lnDTNiBBAdDeTkAPPmAXl59o6IqGbJyQFycwsf5+eLlSVSUwG9nn9zRER3U6bE848//sBLL72E5s2bw93dHeHh4Rg2bBjOnz9vVffs2bPo3bs3PDw84Ovri6eeegrJycmVFjgVz8kJeO01QKMBrlwBPv3U3hERVX9pacC2bcD168D580BSUuGx7Gwx/jM+XvQyTJgAvPQS8MEHwI8/AgkJ9oubiMjRlGly0XvvvYf9+/dj6NChaNmyJW7evImlS5eiVatWOHToEKKjowEA8fHx6Ny5MzQaDebMmYPMzEwsWLAAp06dwuHDh6FUKqvkYkjw9RXJ5/TpwIEDYkKEv7+9oyKqXvLygN9/B3btEpP2MjNFmVIptqs1USqBkBCxi5hWCxiNwNWr4rZ7t+iGDwkRda9fBw4fBqKixM3T0y6XRkRkN2VKPKdMmYJ169ZZJI6PP/44WrRogXnz5mHNmjUAgDlz5iArKwtHjx5FeHg4AKBdu3bo2bMnVq1ahXGccl3l7rsPmDgRaNGCSSdRWcmyaLVMTCwsi4oCrl0TGzd4eRWWu7oCdesCPj6FW9gajUBcHHDxItCoUWHdEyeAVasKHwcEFCahUVFAs2YiUSUiqqkkWS763b18WrduDQA4evQoAKBOnTro0qUL1q9fb1GvcePGCAsLw86dO0t9bp1OB41GA61WC6+i7/ZERJUkJUX0DvTvD0iSKFu+HDh0COjWTdxycoChQ8Xs9eImEpnGfm7YIJLI4hw5IlpQ4+Isk1qT998HmjYV9y9dAm7dEucKCCiMi4jIEZU2X6vwOp6yLOPWrVto3rw5AODGjRtISkpCmzZtrOq2a9cOP//8813Pl5ubi9wiI/d1XJukUhw9Krr4xo/nBxgRIJLEgwdFIvjnn6KVs3590UsAACNHAs8+W/j3kpws1ulMTbWcXFSUn59la+id2rQRNwDIyhKTkuLiCm/16xfW3blTLI0GAB4eli2jUVGi+55/y0RU3VQ48Vy7di1u3LiB2bNnAwAS//kaHxwcbFU3ODgYaWlpyM3NhUqlKvZ8c+fOxaxZsyoaFhWRmioWlzcYgHr1gD597B0RkX3IMvDXXyLZ/O030UppEh0NKIpMt1SrLZ8bECAWh6+snYvc3cVr/jM03oqfHxAZKbr3MzNFcvznn4XHv/xSdO8DonVUoQDCwsQEQyIiR1WhxPPcuXN48cUX0b59e4waNQoAkPPPO3lxiaX6n3fynJycEhPPqVOnYsqUKebHOp0OYWFhFQmz1vPzA0aNAj77DPjf/4CGDcWNqLb5+2/gjTcKH9epA3TvLrrS69S59/MDAmy3K9HgweKWny+Sz0uXxJjRuDiR/JqSTgBYvVp04yuVQESEZctovXqAi4ttYiYiupdyJ543b95E3759odFosHHjRjj98zXb9Z/BT7nF9EXp9XqLOsVRqVQlJqVUfgMHAmfOAP/3f8C0aWLGe3GTGLjXNNUUej2wf79Y7qh/f1HWuLFoRYyMFAln8+aO313t4lKYRPbsKcruHJlvGneakyOWeyq6wp2nJ7B2beF1JiSIlS/ubNElIrKFciWeWq0WjzzyCNLT07Fv3z6EmNYKQWEXe2IxI+cTExPh6+vLxNIOJAl48kng449Ft93+/UBQkHU97jVN1ZksA6dOia70AwdE8unhAfTuLRI4SQIWL3b8ZPNe7oz/jTfEtScmWo4ZjYsDwsMt68+eLZLPunULE9rISPEvZ9QTUVUrc+Kp1+vRv39/nD9/Hjt37kSzZs0sjoeGhiIgIABHjhyxeu7hw4cRExNT7mCpYgoKxJ7Sptm3+fmWCWZOTuFe00w8qTpJTBTJ5u7dYhKQSUiIaNk0GAq7m6t70lkSSRLXGxICdOokymTZchyrwSD+9mVZrCl6/TqwZ0/h8bZtgbfeKnyclcVklIgqV5kSz4KCAjz++OM4ePAgNm3ahPbt2xdbb/Dgwfjiiy9w/fp18/jMXbt24fz585g8eXLFo6ZyU6nEBISbN8XjOz9USpqtS+TIduwQyxgB4ne6UyeRcDZuXHMTzdKQJMDNrfCxszPw+efA7dtizKipVfTiRbEbk0ZTWNdgEFvwentbtopGRYmekdr8cyWi8itT4vnKK6/gxx9/RP/+/ZGWlmZeMN5kxIgRAIBp06Zhw4YN6Nq1KyZNmoTMzEzMnz8fLVq0wJgxYyoveioXPz/xYcJlUam6MRrFzO6dO4EePYDYWFHevbtIpLp3B+6/X0yyoZL5+ACtW4ubSUaG5RfPxETRS5KSIm6//154TKMR42Yff1w8No05ZTJKRPdSpsTzxIkTAIDNmzdjs2mBuSJMiWdYWBj+7//+D1OmTMEbb7wBpVKJvn37YuHChRzf6SCKJp35+WLcJz+syVHFx4uu9F9/FcNBAJGEmhLP0FBg5ky7hVcjeHpabuEZFgZ88431WqPXromtQYsmmcnJYqe0oq2iUVHi/6XoElVERGVKPPcUHQx0D82bN8e2bdvKGg/ZwaVL4oNEo7HsliOyJ6MR+OUXMW7z778Lyz08gC5dROsmVS1XV7GNZ9Gh/Hl5Yh96b+/CsosXxXjQU6fEzUSlEoviDxkiWqKJiCq8gDxVb7IsxsTpdKIlKS1NfKiUtOUfUVWS5cKWNEkCtm4FrlwRrWZt2ohks21brktpT0ql9TrA7doBH35o2TJ66ZLouj93TvSqmJw8KfarL9o6GhHBHhei2oKJZy1UdJYrIMZ7ubiILrW8PGDePJGIDhzIMVtkG1eviq70Q4dEAqNWi9+9oUPFRJguXSxb2MixODuLls369cXYW0C0WCckiCT0nx2VAYg1Ri9cEDcThUIs+xQZCQwaJBa9J6KaiYlnLeLldfe9pkNCRHcZIHY5On4ceOUVTkKiqqHTiQ0Ndu0SyYnJgQNiJyEA6NzZPrFRxSkUYq3QunUty7t1A4KDLVtHtVrRsn3lCjBgQGHd334DDh60XG+06DhUIqp+mHjWIqXZa9rTEzh6FFixQsxqZZcmVbb4eLHP+B9/iCV7ANFi1rat6EovOtOaah5fX+DBB8UNEMMr0tIKl3UKDy+se+IEsHevuJkEBhYmon36MBElqm4kWb5z8zXHotPpoNFooNVq4cWmN5u5fl2My4qMFI9l2XIRbqLSkmXRwm7aojE5GXjmGVHeoIFoAevShS3rZO3vv8VkJVPL6J0b4n39deFaxDt2iLVITS2jAQEcKkRkS6XN19jiScX6Z91/s82bRZfoa69Zd50RFSc9XeyKs2sX4O8PzJghygMCgPHjxbg/juWju2ncWNxMsrIKF75PTrbcAOPXXy1n1Ht6Wi7t9OCDXNqJyBGwxZPuKS8PGDdOjA1VqYDnnhMTCNiaQHfKzwcOHxbJ5tGjYoIJIH5vvvySy3VR1dm5E/jrL5GUXr0qFr830WiA1asL37N27BDDO0xrjTo52SdmopqktPkaE08qlbQ0YNEisWsMAHTsCLz4olhTkQgANm0SXZ+ZmYVljRuLcZudOvF3hWwnP18kn6YuehcX4NlnC4+PGSN2YwLEMk716xe2jDZsKB4TUdmwq50qla8v8PbbwHffiZaD334T469efdVycWmqPVJTxQLjplZMpVIknX5+Ytxmt24clkH24eIixg83aGB9rKAA6NChMCnV68V7mWmTgsaNgQULCuv/+quYhR8RUThOmYjKjy2eVGYXLgDz54uB/kqlmAHv42PvqMgW8vLEWpu7donltsaPFzOLATH+7sIFoGVLjqWj6kGWxftY0aWdGjYERo4Ux3NzxVqypo0N6tYVyaypdbR+fctxpkS1GVs8qco0bAgsWQJ8+qmYhMSks2aTZbH7zK5dwL59QHZ24bErVwrvu7sDMTG2jo6o/CRJrF8cEiKGg9wpM1Ms73Xxopgsd/26uP36qzjetSswZYq4bzSKoUiRkWJMKREVj4knlYurKzB5skhKTK5cEWs0duxot7CokhkMwMSJ4sPWJCBAjNs0LQROVFP5+RWuxmBaa9Q0qz4uznJr4Rs3gLfeEvf9/S1n1EdFieFKnJBJxMSTKsj0RpqbC7z/vkhQjh0Ts+A5Hqr60euBM2cKF3F3dhbJZVKSWI6me3egRQt+gFLt4+srbm3bFpYV/eKt04kZ8jduiIlLKSnA778XHh81ChgyRNzX60ULap06/Fui2oeJJ1UKJyegfXvR4rljh1jW5N//LlyAnhyXLItkc9cuMWlMrxfjduvUEcefe06siejqat84iRxN0aSxeXMx/Cg7G7h82XLc6LVrljsy/fkn8M47YnjKnS2jISEcI001GxNPqhTOzsBTTwH33QcsXCi+9b/yCjB6tNh7md/qHc+tW8Du3SLhvHWrsDwoSCzObUo8AwPtEx9RdeTmJpLQ5s0Ly3JzLZPJ1FTxnpmVBZw8KW4majUwdSrQqlXhc52cRH2imoCz2qnSZWSIyUembqbWrUXrJxcPdxxHjwIzZxY+dnUVY3O7dxfLY/GLAlHVMhhES2jRltHLl0Wi+fHHhbvH/fCD2HwhIqKwVTQyUjxWKu14AUR34Kx2shtPT+A//wG2bgU++0ws5szxnvYjy6JFJT8faNNGlDVvLrr5GjYUyWb79mJ3ISKyDWdnkUBGRgI9e4oyo1H0FoWGFta7dk387V64IG4mCoXovv/Pf0QvBVC47BORI2OLJ1Wpa9dEguPnJx7n5Yk3RhcX+8ZVGyQkiG703bvFRIfwcGDp0sIPpuxstkITOTpZBm7etGwZjYsTk5kUCmDDhsKWz08/BU6cEK2ipvVGIyO5axjZBls8ySEUHVAPiBbQc+dE13vRb/VUObKyxAShXbuAs2cLy93dRRd6fn7hhxSTTiLHJ0liZYng4MKl6mRZjBO9ccOyu/3iRVF24wawd29heWCgSERfe41jRcn++CtINqPViqRIpwMmTRKzpXv0YNdQZfrkE+D//k/clyQxQaFHD6BdO44HI6opJEmsFervb1k+fbp1y+jNm2I5NFm2TDoXLRIL5BedUe/vz/djqnrsaiebSk0Vb3imWZwdOwIvvcRt58rj+nXRjd69e+Ge6EePAitXimSzSxex7iAR1V6ZmWLR++xs4IEHRJksAyNGiEaAojw9RQLaogUwbJjtY6XqrbT5GhNPsjmjEfjuO2DNGqCgQOyE89prQNOm9o7M8WVkiG0rd+0Czp8XZYMHi2WrgMIFrdlqQUQlMW2De+daowUF4njLlsC77xbWf/998SXW1DIaGiqWeCIqimM8yWEpFGIHj5YtgfnzRVfQu++KRcs5+92a0ShaMnftEktUGQyi3MlJzFJv2bKwLhNOIroXSRJf9It+2c/LA65eFa2jnp6F5ZmZ4stuUUolUL++SEJbtQLuv982cVPNwBZPsqvsbDET84EHgA4d7B2NYzIYgJEjRWsnIN7wu3cHHnoI0GjsGhoR1XB6PbB/f2HL6KVLosykd2/gxRfF/fx88X5uahmNiOAybbUJWzypWnBzA6ZMsSw7ckQsovzgg/aJyZ60WjE56ORJsT6fJIkJAf36iSS9WzduQ0pEtqNWiy+63buLx7IslmozJaLR0YV1r14Ftm8vfCxJYiF8UyLaqlXhwvhUe7HFkxzK7dvi23NGBvDww8DYsTW/+91gAP74Q3SlHzlSOM5q7lzLN3UiIkd265ZIPC9dEks7padbHh8zBhg0SNxPSQH27BHLPEVGAvx4r/7Y4knVkqen6LrZuBHYtg04c0ZMPKqJrXyJicDmzeLN19SNDojdhLp1A+rVs1toRERlVqcO8NRThY/T0iwnMBUdU/rXX8AXXxQ+DgiwXNqpaVMufF8hRqP4AL19G/DxEdvVKRT2jgoAWzzJQZ08CSxcKN64nJ3FrO0BA6r/5JmiW9qdPg1MnSru+/qKMZvdu1svuk9EVNOcPCm2VY6LE1/C7zRtmtjKFxAL4l+9KlpHAwKq/+dAlTtwQGxTd/asGLemUolM/qWXqnQyBZdTKg8H/oZQG+l0wIcfipncgJjB/Z//lHPnDTv+3+blAYcPi670kBDg2WdFuSwDy5YBbdsCMTFcnqRa4XsFUaXJygIuX7ZsHZ0xQ+y4BADr1wOrV4v7Hh6iB6zotqAhIUxGzQ4cAF59VbTaBAcDrq5ATo5YPsbHB1iwoMqSTyaeZWWnbwh2U00+OGVZfCtesUJ0P7/0UjlOYof/W1kW62zu3i0mC2VliXJPT+DLL7ltXbVW294riOzs55/F2NGrVwuXkyvqv/8VM+gBsbFGQYHYVKPWvc8ajWJngGPHRFZeNBuXZZHRt2olsvgq+Lxn4lkWdvyGYBfV8IPz2jXx7dc00SgjQ9x3cbnHE+3wf/vLL8CmTUB8fGGZv79InLt14x711Vpte6+gQqaPSlm++81oLLme6Vhxde48VrROcceKPvdux+6sc6/4q/pWgZ+PoUDCtRQ3xCV74eItD8Qle+HGbXesHrUTzk6izuJfW2LXubpwcSpAhI8OUf7piPLTIsr3Nur56KBUGGrszwdarehec3Ep7ELz9hYTBwCxKGt6umj9aNGiLL/9pcLJRaVlNIokLC3N8huCh4dow4+LAz76SCw06YAtgmVW0gfn8eOi3EE/OIuOe5RlsZNGRoaYeFRiImej/9u8PPHN2nSKW7dE0qlUiiWhunUTi7w75K9PcR+mRR+XdL+09Yp7jYqesyKvUZHXKygAZs0Sa8nUrSuaXkx7Dvr6ioFoM2cCb74pftfK+yFT1gTE9G/RD8/ijt35AVvcMUe7OdLPh+zOGUDkP7eeAKAC5DqAtK2wjuKyC9zSgOwCFS7cBC7AA4AHgFA4SwX4pvX7IvkEkJSrgZdzNtRO+ba+lKqRmSnep5RK6/dGQHze37olejrtiC2ep06J1bm9vUVCcvZsYYIiSWJFXL1e7EsYFCS+RUiSyCJM/97tflXVk6SyxwIAr78urrFevcLzmN5Yr10TLZ9z54pyo7HwZnpTv9v9om/SBQXW98t6nhLuJ9x2xas/PIgMvQvUzgY8d/+f6B55GRLueE5iotibs2jTaNFf97w80eLbp49okix67B6JimyUce52Hey83hi/JUTi37Hb0TrgOiDLSMz0xOnboXiwzgW4OeWVeI6yvF6lJlB3/hyodHQ64MQJ8S2juD48g0HcYmK4NgwVz/Sea3o/Nj0uejMdK65OWY4VrVPcsaLPvduxO+uU9ZgdbjIk3NS6Ii7JA3G3PBF3ywMXb3lA45aPT8afNNd7/YtmOBvvgVD/XEQFZSMqJAdRIdmIDNbDw10u/udpus7yHqusn3lxt3PngIkTxfuPaUkAhaLw/Yotng7i9m2RfLi6ig/jouvaAKIsJ0eMmfDzs0+MlaXoB+dff1kfNxjE3mgvveTQH5whAP7rvwuLLj2KkykRWLIpAsd8s/BixBa4O+cWVkxNFYMrjcbiR56b/m///lssKlcKSbka7E5pid2pLZGo9/2ntACHz2nQWn8AABAMIFg6CyRV6DKrv6I/8zvfTO91vyL1quKckiRaNJ2dxUDdO2eCmb6o6XSiNdQ04KwiicG9jpX0IVcNEoNib6VJxu52rLTJWGUkBmX9mZviIpuQ8M/7MICO/5TJ8j9j7T0izY+1PwNyHhAPID4J+L8kACdE/SZNxJbOJjk5Ik1waHXqiITy+HEx9Kfo75wsiyFBrVqJOR12xMTTx0eMcczJEd8QGjSwbCnKyRHfEkaNEq2EFW0BLK5+0dbB8rYylqae6Y3YzU3cL9rqJUmiVTArS+zDGBpa+S2uldji6ydJeFtywreHgDV7w7BPDsffBT3w6qOX0bRetojtyhXgvffE/6ubW+F1mpj+b194wXog9h1JR3auAu9+FoyTF91Er40noFbKeDAmC90fyEJ0g46AolP5kp17Ha+MpKm4a6vK166JTp0Sq2KbekfuZGpNeOONKmlNIKLykyTLP1tJEtt73r4tFrwvOqP+1i3L/eoB4LnnxMdP0bVGo6JEe5TDvOUpFKLh6NVXxYUEBVmPQ3/xRbuP+2JXu2kW2PHj4rfozm8IVTwLzKbuHFZwpypuhq8qf/8thqbevCm2Y1u69J//qgr838qyGMpnGj8qy+Lv+do1MV6ze3cxFLam76pERdSm9wqiWiwjQ2xRXKeOeKzVioXxi8uWvLzE58HTTxeWybKdk9HiJhA3ayaSTgdYx5MtntXkG0KlaN5cjOEs6YPTQZrhy6pxY2DJEmD5cqB//yL/VQoFkp98Gbpz84GzWrHysEol/hCTkwFNNLyGT0JAkf/bxESxBNLu3aLXdPVqkVxKEjBhgphDYlpbjmqZ2vReQVSLeXpatnhqNGIt0cuXRaeHqWX02jXxOWHa5hgQCevTTwP16xe2ijZoUNiJaBMdOohJsw66ZCJbPE3s9A3B5kyz2m/fLv6D00FntZfH118DixYBBm2mSDT1esAoAwpJZJMBAfAL98CKFWLNzV27xN+piZubmMTcpIn9roEcUG15ryCiu8rLE2uLurkV9o6dOSNG29xJpRLJaJ8+QNeuto3TVriOZ3lUk0XVK6wWfHBeuQKMGyd2PfL1BeqFy1DkZIoJVM7OgLsHcvQS0tLEPBClUjxPksSk5O7dxRdGlcqOF0GOq7a8VxBRmRgMYjk9U6voxYuipVSvF8efew7o10/cv3IFWLzYcsxo/fqFn0fVDbvay0OhqFZjG8vNwZvhK0PdukDPniLx1OmAK1clREV5QqEQOYObGwBJ/IHn54st2Lp3F99Eq/viBWQDteW9gojKxNlZNGZERIjPFEB85iQmikS0UaPCukW77U0UCjFXITJSJKhF65dGcnLh8sLF8fISo87siS2eVGPFxYk/3PT0wkn9sizmVjVqJCbwp6eL8aEdOzrQzEQiIqrx0tPFyoZFW0e12sLjM2cCrVuL+ydOiG1Di7aO3jnzPjkZGD5crCRYEj8/YN26qkk+2eJJBDGENTBQDGFNT7deWQgAQkKYdBIRkW15e4sOSNMIN1kWnZCmRNS00yUgFqXZt0/cTAIDRQIaGQn06iVaOlNTxRCx4tYczckRx3U6+7Z6lrlvNTMzEzNmzEDv3r3h6+sLSZKwatWqYuuePXsWvXv3hoeHB3x9ffHUU08hOTm5ojETlYmTk2jhbNZMjN8s+sdMRETkCCRJzElo2xZ44gnLfVzatxfLiXfsKHa7BoCkJODgQWDtWjFdwyQvTySwBgPg7l54c5QF8Mvc4pmSkoLZs2cjPDwc9913H/bs2VNsvfj4eHTu3BkajQZz5sxBZmYmFixYgFOnTuHw4cNQVtfRs1RtFbd0KRERkaNr0EDcTLKyxKSluDgxsz4oSCyED4jevcxM0aKq0dgj2rsrc+IZHByMxMREBAUF4ciRI2jbtm2x9ebMmYOsrCwcPXoU4eHhAIB27dqhZ8+eWLVqFcaNG1exyIlKKSenbOVERESOzN0diI4WtztpNKJ1093d9nGVRpm72lUqFYKCgu5Z79tvv0W/fv3MSScA9OjRA40aNcL69evL+rJEZeblJQZS5+aKb4B33nJzxXHOWSMioprC21ssy+Som51UyeSiGzduICkpCW3atLE61q5dO/z8888lPjc3Nxe5RQYr6O62LgDRXQQEiNl7jr60BBERUW1RJYlnYmIiANEtf6fg4GCkpaUhNzcXqmJW5547dy5mzZpVFWFRLRQQwMSSiIhqD0cfXlYlK4bn/HN1xSWWarXaos6dpk6dCq1Wa75dv369KkIkIiIiqjGqy/CyKmnxdP1nzn7RLnMT/T/7RrmWMK9fpVIVm7ASERERUfGqy/CyKkk8TV3spi73ohITE+Hr68vkkoiIiKgSVYfhZVXS1R4aGoqAgAAcOXLE6tjhw4cRExNTFS9LRERERA6sShJPABg8eDB++uknizGau3btwvnz5zF06NCqelkiIiIiclDl6mpfunQp0tPTkZCQAADYvHkz4uPjAQATJkyARqPBtGnTsGHDBnTt2hWTJk1CZmYm5s+fjxYtWmDMmDGVdwVEREREVC1IsizLZX1SREQErl69Wuyxy5cvIyIiAgBw5swZTJkyBb/99huUSiX69u2LhQsXok6dOqV+LZ1OB41GA61WCy97T8UiIiIiIiulzdfKlXjaEhNPIiIiIsdW2nytysZ4EhEREREVxcSTiIiIiGyCiScRERER2QQTTyIiIiKyiSrZuagymeY+6e62BxQRERER2Y0pT7vXnHWHTzwzMjIAAGFhYXaOhIiIiIjuJiMjAxqNpsTjDr+cktFoREJCAjw9PSFJUpW/nk6nQ1hYGK5fv17jl2+qTdcK1L7rparF3ycicmS2fo+SZRkZGRkICQmBQlHySE6Hb/FUKBSoW7euzV/Xy8ur1nyY1KZrBWrf9VLV4u8TETkyW75H3a2l04STi4iIiIjIJph4EhEREZFNMPG8g0qlwowZM6BSqewdSpWrTdcK1L7rparF3ycicmSO+h7l8JOLiIiIiKhmYIsnEREREdkEE08iIiIisgkmnkRERERkE0w8iYiIiMgmmHgSERERkU3UusQzMzMTM2bMQO/eveHr6wtJkrBq1api6549exa9e/eGh4cHfH198dRTTyE5Odm2AVfAmTNnMHToUERGRsLNzQ3+/v7o3LkzNm/ebFW3ul/rnj17IElSsbdDhw5Z1D1w4AA6duwINzc3BAUFYeLEicjMzLRT5OQIquJ9wWg04v3330f9+vWhVqvRsmVLfPXVV1V8JURU0/zxxx946aWX0Lx5c7i7uyM8PBzDhg3D+fPnrepWh/cnh98ys7KlpKRg9uzZCA8Px3333Yc9e/YUWy8+Ph6dO3eGRqPBnDlzkJmZiQULFuDUqVM4fPgwlEqlbQMvh6tXryIjIwOjRo1CSEgIsrOz8e2332LAgAFYtmwZxo0bB6BmXKvJxIkT0bZtW4uyBg0amO+fOHEC3bt3R9OmTbFo0SLEx8djwYIFuHDhArZu3WrrcMlBVMX7wn/+8x/MmzcPzz77LNq2bYtNmzZh+PDhkCQJTzzxhI2ujIiqu/feew/79+/H0KFD0bJlS9y8eRNLly5Fq1atcOjQIURHRwOoRu9Pci2j1+vlxMREWZZl+Y8//pAByJ9//rlVveeff152dXWVr169ai7bsWOHDEBetmyZrcKtdAaDQb7vvvvkxo0bm8tqwrX++uuvMgB5w4YNd633yCOPyMHBwbJWqzWX/e9//5MByNu2bavqMMlBVfb7Qnx8vOzi4iK/+OKL5jKj0Sh36tRJrlu3rmwwGKruYoioRtm/f7+cm5trUXb+/HlZpVLJTz75pLmsurw/1bqudpVKhaCgoHvW+/bbb9GvXz+Eh4eby3r06IFGjRph/fr1VRlilXJyckJYWBjS09PNZTXtWjMyMmAwGKzKdTodduzYgREjRsDLy8tcPnLkSHh4eFTLa6XKUdnvC5s2bUJ+fj5eeOEFc5kkSXj++ecRHx+PgwcPVu4FEFGN1aFDB6uex4YNG6J58+Y4e/asuay6vD/VusSzNG7cuIGkpCS0adPG6li7du1w/PhxO0RVfllZWUhJSUFcXBw++OADbN26Fd27dwdQ8651zJgx8PLyglqtRteuXXHkyBHzsVOnTsFgMFhdq1KpRExMTLW7VrKtsvytHD9+HO7u7mjatKlVPdNxIqLykmUZt27dgr+/P4Dq9f5U68Z4lkZiYiIAIDg42OpYcHAw0tLSkJub63D7n5bklVdewbJlywAACoUCgwYNwtKlSwHUnGtVKpUYPHgw+vTpA39/f/z1119YsGABOnXqhAMHDiA2Nvae17pv3z5bh03VSFn+VhITE1GnTh1IkmRVDwASEhKqPmAiqrHWrl2LGzduYPbs2QCq1/sTE89i5OTkAECxyZZarTbXcfRkzOTll1/GkCFDkJCQgPXr16OgoAB5eXkAas61dujQAR06dDA/HjBgAIYMGYKWLVti6tSp+OWXX+55rabjRMUpy99KSX8zResREZXHuXPn8OKLL6J9+/YYNWoUgOr1/sSu9mK4uroCAHJzc62O6fV6izrVQZMmTdCjRw+MHDkSP/30EzIzM9G/f3/IslzjrrWoBg0aYODAgfj1119RUFBwz2utrtdJtlGWvxVXV9ca+TdFRPZ18+ZN9O3bFxqNBhs3boSTkxOA6vX+xMSzGKbmZlPTdVGJiYnw9fV1+BbAuxkyZAj++OMPnD9/vsZfa1hYGPLy8pCVlXXPaw0JCbF1eFSNlOVvJTg4GDdv3oQsy1b1APB3jYjKTKvV4pFHHkF6ejp++eUXi/eR6vT+xMSzGKGhoQgICLCYmGJy+PBhxMTE2D6oSmRqRtdqtTX+Wi9dugS1Wg0PDw9ER0fD2dnZ6lrz8vJw4sSJan+tVLXK8rcSExOD7OxsixmnAPD777+bjxMRlZZer0f//v1x/vx5/PTTT2jWrJnF8er0/sTEswSDBw/GTz/9hOvXr5vLdu3ahfPnz2Po0KF2jKz0kpKSrMry8/Px5ZdfwtXV1fyLWxOutbidGf7880/8+OOP6NWrFxQKBTQaDXr06IE1a9YgIyPDXG/16tXIzMysNtdK9lPav5WBAwfCxcUFH3/8sblMlmV8+umnCA0NtRiPTER0NwUFBXj88cdx8OBBbNiwAe3bty+2XnV5f5LkO9taa4GlS5ciPT0dCQkJ+OSTTzBo0CDExsYCACZMmACNRoPr168jNjYW3t7emDRpEjIzMzF//nzUrVsXf/zxR7Xofn7ssceg0+nQuXNnhIaG4ubNm1i7di3OnTuHhQsXYsqUKQBQI661W7ducHV1RYcOHRAYGIi//voLy5cvh4uLCw4ePGheNuLYsWPo0KEDmjVrhnHjxiE+Ph4LFy5E586dsW3bNjtfBdlTZb8v/Pvf/8b8+fMxbtw4tG3bFj/88AO2bNmCtWvXYvjw4fa6TCKqZl5++WUsWbIE/fv3x7Bhw6yOjxgxAkDZPsvt+v5UpcvTO6h69erJAIq9Xb582Vzv9OnTcq9evWQ3NzfZ29tbfvLJJ+WbN2/aL/Ay+uqrr+QePXrIderUkZ2dnWUfHx+5R48e8qZNm6zqVvdrXbJkidyuXTvZ19dXdnZ2loODg+URI0bIFy5csKq7b98+uUOHDrJarZYDAgLkF198UdbpdHaImhxJZb8vFBQUyHPmzJHr1asnK5VKuXnz5vKaNWtseEVEVBN06dKlxPemO9O46vD+VCtbPImIiIjI9jjGk4iIiIhsgoknEREREdkEE08iIiIisgkmnkRERERkE0w8iYiIiMgmmHgSERERkU0w8SQiIiIim2DiSUREREQ2wcSTiIiIiGyCiScRERER2QQTTyIiIiKyCSaeRERERGQT/w/k6bnsFkKm2gAAAABJRU5ErkJggg==",
+      "text/plain": [
+       "<Figure size 800x500 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Top5 invalidity\n",
+    "CompoundT5 = [32.75, 18.76, 11.07, 20.99, 10.62]\n",
+    "ReactionT5 = [12.5, 12.4, 12.5, 12.6, 12.9]\n",
+    "T5Chem = [33.73, 38.94, 46.23, 69.05, 73.41][::-1]\n",
+    "\n",
+    "\n",
+    "# plot\n",
+    "import matplotlib.pyplot as plt\n",
+    "fig, ax = plt.subplots(1, figsize=(8, 5))\n",
+    "\n",
+    "\n",
+    "ax.plot([10,30,50,100,200], ReactionT5, \"o-\", label='ReactionT5', color='red', alpha=0.7)\n",
+    "ax.plot([10,30,50,100,200], CompoundT5, \"s--\", label='CompoundT5', color='blue', alpha=0.7)\n",
+    "ax.plot([10,30,50,100,200], T5Chem, \"v:\", label='T5Chem', color='green', alpha=0.7)\n",
+    "\n",
+    "\n",
+    "# plt.ylim(0, 35)\n",
+    "ax.set_xticks([10,30,50,100,200])\n",
+    "ax.set_xticklabels([10,30,50,100,200], fontsize=12)\n",
+    "# ax.set_yticks([10,20,30,40,50,60])\n",
+    "ax.set_yticklabels([int(i) for i in ax.get_yticks()], fontsize=12)\n",
+    "# plt.tight_layout()\n",
+    "ax.legend(loc=\"best\", fontsize=12)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "reactiont5",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.18"
+  },
+  "varInspector": {
+   "cols": {
+    "lenName": 16,
+    "lenType": 16,
+    "lenVar": 40
+   },
+   "kernels_config": {
+    "python": {
+     "delete_cmd_postfix": "",
+     "delete_cmd_prefix": "del ",
+     "library": "var_list.py",
+     "varRefreshCmd": "print(var_dic_list())"
+    },
+    "r": {
+     "delete_cmd_postfix": ") ",
+     "delete_cmd_prefix": "rm(",
+     "library": "var_list.r",
+     "varRefreshCmd": "cat(var_dic_list()) "
+    }
+   },
+   "types_to_exclude": [
+    "module",
+    "function",
+    "builtin_function_or_method",
+    "instance",
+    "_Feature"
+   ],
+   "window_display": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

task_forward/calculate_accuracy.py

@@ -0,0 +1,135 @@
+import argparse
+import os
+import sys
+import warnings
+
+import pandas as pd
+import rdkit
+from rdkit import Chem
+from transformers import AutoTokenizer
+
+rdkit.RDLogger.DisableLog("rdApp.*")
+
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from utils import canonicalize, seed_everything
+
+warnings.filterwarnings("ignore")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Script for reaction retrosynthesis prediction."
+    )
+    parser.add_argument(
+        "--input_data",
+        type=str,
+        required=True,
+        help="Path to the input data.",
+    )
+    parser.add_argument(
+        "--target_data",
+        type=str,
+        required=True,
+        help="Path to the target data.",
+    )
+    parser.add_argument(
+        "--target_col",
+        type=str,
+        required=True,
+        help="Name of target column.",
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        type=str,
+        default="sagawa/ReactionT5v2-retrosynthesis",
+        help="Name or path of the finetuned model for prediction. Can be a local model or one from Hugging Face.",
+    )
+    parser.add_argument(
+        "--num_beams", type=int, default=5, help="Number of beams used for beam search."
+    )
+    parser.add_argument(
+        "--seed", type=int, default=42, help="Seed for reproducibility."
+    )
+    return parser.parse_args()
+
+
+def remove_space(row):
+    for i in range(5):
+        row[f"{i}th"] = row[f"{i}th"].replace(" ", "")
+    return row
+
+
+if __name__ == "__main__":
+    CFG = parse_args()
+
+    seed_everything(seed=CFG.seed)
+
+    tokenizer = AutoTokenizer.from_pretrained(
+        os.path.abspath(CFG.model_name_or_path)
+        if os.path.exists(CFG.model_name_or_path)
+        else CFG.model_name_or_path,
+        return_tensors="pt",
+    )
+
+    df = pd.read_csv(CFG.input_data)
+    df[[f"{i}th" for i in range(CFG.num_beams)]] = df[
+        [f"{i}th" for i in range(CFG.num_beams)]
+    ].fillna(" ")
+    df["target"] = pd.read_csv(CFG.target_data)[CFG.target_col].values
+    df = df.apply(remove_space, axis=1)
+
+    top_k_invalidity = CFG.num_beams
+
+    top1, top2, top3, top5 = [], [], [], []
+    invalidity = []
+
+    for idx, row in df.iterrows():
+        target = canonicalize(row["target"])
+        if canonicalize(row["0th"]) == target:
+            top1.append(1)
+            top2.append(1)
+            top3.append(1)
+            top5.append(1)
+        elif canonicalize(row["1th"]) == target:
+            top1.append(0)
+            top2.append(1)
+            top3.append(1)
+            top5.append(1)
+        elif canonicalize(row["2th"]) == target:
+            top1.append(0)
+            top2.append(0)
+            top3.append(1)
+            top5.append(1)
+        elif canonicalize(row["3th"]) == target:
+            top1.append(0)
+            top2.append(0)
+            top3.append(0)
+            top5.append(1)
+        elif canonicalize(row["4th"]) == target:
+            top1.append(0)
+            top2.append(0)
+            top3.append(0)
+            top5.append(1)
+        else:
+            top1.append(0)
+            top2.append(0)
+            top3.append(0)
+            top5.append(0)
+
+        input_compound = row["input"]
+        output = [row[f"{i}th"] for i in range(top_k_invalidity)]
+        inval_score = 0
+        for ith, out in enumerate(output):
+            mol = Chem.MolFromSmiles(out.rstrip("."))
+            if not isinstance(mol, Chem.rdchem.Mol):
+                inval_score += 1
+        invalidity.append(inval_score)
+    print(CFG.input_data)
+    print(f"Top 1 accuracy: {sum(top1) / len(top1)}")
+    print(f"Top 2 accuracy: {sum(top2) / len(top2)}")
+    print(f"Top 3 accuracy: {sum(top3) / len(top3)}")
+    print(f"Top 5 accuracy: {sum(top5) / len(top5)}")
+    print(
+        f"Top {top_k_invalidity} Invalidity: {sum(invalidity) / (len(invalidity) * top_k_invalidity) * 100}"
+    )

task_forward/finetune.py

@@ -0,0 +1,251 @@
+import argparse
+import os
+import sys
+import warnings
+
+import datasets
+import pandas as pd
+import torch
+from datasets import Dataset, DatasetDict
+from transformers import (
+    AutoModelForSeq2SeqLM,
+    AutoTokenizer,
+    DataCollatorForSeq2Seq,
+    EarlyStoppingCallback,
+    Seq2SeqTrainer,
+    Seq2SeqTrainingArguments,
+)
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from train import preprocess_df
+from utils import filter_out, get_accuracy_score, preprocess_dataset, seed_everything
+
+# Suppress warnings and disable progress bars
+warnings.filterwarnings("ignore")
+datasets.utils.logging.disable_progress_bar()
+
+
+def parse_args():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Training script for reaction prediction model."
+    )
+    parser.add_argument(
+        "--train_data_path", type=str, required=True, help="Path to training data CSV."
+    )
+    parser.add_argument(
+        "--valid_data_path",
+        type=str,
+        required=True,
+        help="Path to validation data CSV.",
+    )
+    parser.add_argument(
+        "--similar_reaction_data_path",
+        type=str,
+        required=False,
+        help="Path to similar data CSV.",
+    )
+    parser.add_argument(
+        "--output_dir", type=str, default="t5", help="Path of the output directory."
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        type=str,
+        default="sagawa/ReactionT5v2-forward",
+        help="The name of a pretrained model or path to a model which you want to finetune on your dataset. You can use your local models or models uploaded to hugging face.",
+    )
+    parser.add_argument(
+        "--debug", action="store_true", default=False, help="Enable debug mode."
+    )
+    parser.add_argument(
+        "--epochs", type=int, default=3, help="Number of epochs for training."
+    )
+    parser.add_argument("--lr", type=float, default=2e-5, help="Learning rate.")
+    parser.add_argument("--batch_size", type=int, default=32, help="Batch size.")
+    parser.add_argument(
+        "--input_max_length", type=int, default=200, help="Max input token length."
+    )
+    parser.add_argument(
+        "--target_max_length", type=int, default=150, help="Max target token length."
+    )
+    parser.add_argument(
+        "--eval_beams",
+        type=int,
+        default=5,
+        help="Number of beams used for beam search during evaluation.",
+    )
+    parser.add_argument(
+        "--target_column",
+        type=str,
+        default="PRODUCT",
+        help="Target column name.",
+    )
+    parser.add_argument(
+        "--weight_decay",
+        type=float,
+        default=0.01,
+        help="Weight decay.",
+    )
+    parser.add_argument(
+        "--evaluation_strategy",
+        type=str,
+        default="epoch",
+        help="Evaluation strategy used during training. Select from 'no', 'steps', or 'epoch'. If you select 'steps', also give --eval_steps.",
+    )
+    parser.add_argument(
+        "--eval_steps",
+        type=int,
+        help="Evaluation steps.",
+    )
+    parser.add_argument(
+        "--save_strategy",
+        type=str,
+        default="epoch",
+        help="Save strategy used during training. Select from 'no', 'steps', or 'epoch'. If you select 'steps', also give --save_steps.",
+    )
+    parser.add_argument(
+        "--save_steps",
+        type=int,
+        default=500,
+        help="Save steps.",
+    )
+    parser.add_argument(
+        "--logging_strategy",
+        type=str,
+        default="epoch",
+        help="Logging strategy used during training. Select from 'no', 'steps', or 'epoch'. If you select 'steps', also give --logging_steps.",
+    )
+    parser.add_argument(
+        "--logging_steps",
+        type=int,
+        default=500,
+        help="Logging steps.",
+    )
+    parser.add_argument(
+        "--save_total_limit",
+        type=int,
+        default=2,
+        help="Limit of saved checkpoints.",
+    )
+    parser.add_argument(
+        "--fp16",
+        action="store_true",
+        default=False,
+        help="Enable fp16 training.",
+    )
+    parser.add_argument(
+        "--disable_tqdm",
+        action="store_true",
+        default=False,
+        help="Disable tqdm.",
+    )
+    parser.add_argument(
+        "--seed", type=int, default=42, help="Set seed for reproducibility."
+    )
+    parser.add_argument(
+        "--sampling_num",
+        type=int,
+        default=-1,
+        help="Number of samples used for training. If you want to use all samples, set -1.",
+    )
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    CFG = parse_args()
+    CFG.disable_tqdm = True
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    seed_everything(seed=CFG.seed)
+
+    train = preprocess_df(
+        filter_out(pd.read_csv(CFG.train_data_path), ["REACTANT", "PRODUCT"])
+    )
+    valid = preprocess_df(
+        filter_out(pd.read_csv(CFG.valid_data_path), ["REACTANT", "PRODUCT"])
+    )
+    if CFG.sampling_num > 0:
+        train = train.sample(n=CFG.sampling_num, random_state=CFG.seed).reset_index(
+            drop=True
+        )
+
+    if CFG.similar_reaction_data_path:
+        similar = preprocess_df(
+            filter_out(
+                pd.read_csv(CFG.similar_reaction_data_path), ["REACTANT", "PRODUCT"]
+            )
+        )
+        print(len(train))
+        train = pd.concat([train, similar], ignore_index=True)
+        print(len(train))
+
+    for col in ["REAGENT"]:
+        train[col] = train[col].fillna(" ")
+        valid[col] = valid[col].fillna(" ")
+    train["input"] = "REACTANT:" + train["REACTANT"] + "REAGENT:" + train["REAGENT"]
+    valid["input"] = "REACTANT:" + valid["REACTANT"] + "REAGENT:" + valid["REAGENT"]
+
+    if CFG.debug:
+        train = train[: int(len(train) / 40)].reset_index(drop=True)
+        valid = valid[: int(len(valid) / 40)].reset_index(drop=True)
+
+    dataset = DatasetDict(
+        {
+            "train": Dataset.from_pandas(train[["input", "PRODUCT"]]),
+            "validation": Dataset.from_pandas(valid[["input", "PRODUCT"]]),
+        }
+    )
+
+    # load tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(
+        os.path.abspath(CFG.model_name_or_path)
+        if os.path.exists(CFG.model_name_or_path)
+        else CFG.model_name_or_path,
+        return_tensors="pt",
+    )
+    CFG.tokenizer = tokenizer
+
+    model = AutoModelForSeq2SeqLM.from_pretrained(
+        os.path.abspath(CFG.model_name_or_path) if os.path.exists(CFG.model_name_or_path) else CFG.model_name_or_path
+    ).to(device)
+    tokenized_datasets = dataset.map(
+        lambda examples: preprocess_dataset(examples, CFG),
+        batched=True,
+        remove_columns=dataset["train"].column_names,
+    )
+
+    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
+
+    args = Seq2SeqTrainingArguments(
+        CFG.output_dir,
+        evaluation_strategy=CFG.evaluation_strategy,
+        save_strategy=CFG.save_strategy,
+        logging_strategy=CFG.logging_strategy,
+        learning_rate=CFG.lr,
+        per_device_train_batch_size=CFG.batch_size,
+        per_device_eval_batch_size=CFG.batch_size * 4,
+        weight_decay=CFG.weight_decay,
+        save_total_limit=CFG.save_total_limit,
+        num_train_epochs=CFG.epochs,
+        predict_with_generate=True,
+        fp16=CFG.fp16,
+        disable_tqdm=CFG.disable_tqdm,
+        push_to_hub=False,
+        load_best_model_at_end=True,
+    )
+
+    model.config.eval_beams = CFG.eval_beams
+    model.config.max_length = CFG.target_max_length
+    trainer = Seq2SeqTrainer(
+        model,
+        args,
+        train_dataset=tokenized_datasets["train"],
+        eval_dataset=tokenized_datasets["validation"],
+        data_collator=data_collator,
+        tokenizer=tokenizer,
+        compute_metrics=lambda eval_preds: get_accuracy_score(eval_preds, CFG),
+        callbacks=[EarlyStoppingCallback(early_stopping_patience=10)],
+    )
+
+    trainer.train()
+    trainer.save_model("./best_model")

task_forward/generate_embedding.py

@@ -0,0 +1,129 @@
+import argparse
+import os
+import sys
+import warnings
+
+import numpy as np
+import pandas as pd
+import torch
+from torch.utils.data import DataLoader
+from transformers import AutoTokenizer, T5EncoderModel
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from generation_utils import ReactionT5Dataset
+from train import preprocess_df, preprocess_USPTO
+from utils import filter_out, seed_everything
+
+warnings.filterwarnings("ignore")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--input_data",
+        type=str,
+        required=True,
+        help="Path to the input data.",
+    )
+    parser.add_argument(
+        "--test_data",
+        type=str,
+        required=False,
+        help="Path to the test data. If provided, the duplicates will be removed from the input data.",
+    )
+    parser.add_argument(
+        "--input_max_length",
+        type=int,
+        default=400,
+        help="Maximum token length of input.",
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        type=str,
+        default="sagawa/ReactionT5v2-forward",
+        help="Name or path of the finetuned model for prediction. Can be a local model or one from Hugging Face.",
+    )
+    parser.add_argument(
+        "--batch_size", type=int, default=5, help="Batch size for prediction."
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="./",
+        help="Directory where predictions are saved.",
+    )
+    parser.add_argument(
+        "--debug", action="store_true", default=False, help="Use debug mode."
+    )
+    parser.add_argument(
+        "--seed", type=int, default=42, help="Seed for reproducibility."
+    )
+    return parser.parse_args()
+
+
+def create_embedding(dataloader, model, device):
+    outputs_mean = []
+    model.eval()
+    model.to(device)
+    for inputs in dataloader:
+        inputs = {k: v.to(CFG.device) for k, v in inputs.items()}
+        with torch.no_grad():
+            output = model(**inputs)
+        last_hidden_states = output[0]
+        input_mask_expanded = (
+            inputs["attention_mask"]
+            .unsqueeze(-1)
+            .expand(last_hidden_states.size())
+            .float()
+        )
+        sum_embeddings = torch.sum(last_hidden_states * input_mask_expanded, 1)
+        sum_mask = input_mask_expanded.sum(1)
+        sum_mask = torch.clamp(sum_mask, min=1e-6)
+        mean_embeddings = sum_embeddings / sum_mask
+        outputs_mean.append(mean_embeddings.detach().cpu().numpy())
+
+    return outputs_mean
+
+
+if __name__ == "__main__":
+    CFG = parse_args()
+    CFG.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    if not os.path.exists(CFG.output_dir):
+        os.makedirs(CFG.output_dir)
+
+    seed_everything(seed=CFG.seed)
+
+    CFG.tokenizer = AutoTokenizer.from_pretrained(
+        os.path.abspath(CFG.model_name_or_path)
+        if os.path.exists(CFG.model_name_or_path)
+        else CFG.model_name_or_path,
+        return_tensors="pt",
+    )
+    model = T5EncoderModel.from_pretrained(CFG.model_name_or_path).to(CFG.device)
+    model.eval()
+
+    input_data = filter_out(pd.read_csv(CFG.input_data), ["REACTANT", "PRODUCT"])
+    input_data = preprocess_df(input_data, drop_duplicates=False)
+    if CFG.test_data:
+        input_data_copy = preprocess_USPTO(input_data.copy())
+        test_data = filter_out(pd.read_csv(CFG.test_data), ["REACTANT", "PRODUCT"])
+        USPTO_test = preprocess_USPTO(test_data)
+        input_data = input_data[
+            ~input_data_copy["pair"].isin(USPTO_test["pair"])
+        ].reset_index(drop=True)
+    input_data.to_csv(os.path.join(CFG.output_dir, "input_data.csv"), index=False)
+    dataset = ReactionT5Dataset(CFG, input_data)
+    dataloader = DataLoader(
+        dataset,
+        batch_size=CFG.batch_size,
+        shuffle=False,
+        num_workers=4,
+        pin_memory=True,
+        drop_last=False,
+    )
+
+    outputs = create_embedding(dataloader, model, CFG.device)
+    outputs = np.concatenate(outputs, axis=0)
+
+    np.save(os.path.join(CFG.output_dir, "embedding_mean.npy"), outputs)

task_forward/get_distance.py

@@ -0,0 +1,74 @@
+import argparse
+import os
+import sys
+import warnings
+
+import numpy as np
+import pandas as pd
+import torch
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from utils import seed_everything
+
+warnings.filterwarnings("ignore")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Search for similar reactions.")
+    parser.add_argument(
+        "--input_data",
+        type=str,
+        required=True,
+        help="Path to the input data.",
+    )
+    parser.add_argument(
+        "--target_embedding",
+        type=str,
+        required=True,
+        help="Path to the target embedding.",
+    )
+    parser.add_argument(
+        "--query_embedding",
+        type=str,
+        required=True,
+        help="Path to the target embedding.",
+    )
+    parser.add_argument("--batch_size", type=int, default=64, help="Batch size.")
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="./",
+        help="Directory where results are saved.",
+    )
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    config = parse_args()
+    seed_everything(42)
+
+    target_embedding = np.load(config.target_embedding)
+    query_embedding = np.load(config.query_embedding)
+
+    target_embedding = torch.tensor(target_embedding, dtype=torch.float32).cuda()
+    query_embedding = torch.tensor(query_embedding, dtype=torch.float32).cuda()
+
+    target_embedding = torch.nn.functional.normalize(target_embedding, p=2, dim=1)
+    query_embedding = torch.nn.functional.normalize(query_embedding, p=2, dim=1)
+
+    batch_size = config.batch_size
+    distances = []
+
+    for i in range(0, query_embedding.shape[0], batch_size):
+        print(f"Processing batch {i // batch_size}...")
+        batch = query_embedding[i : i + batch_size]
+        similarity = torch.matmul(batch, target_embedding.T)
+        distance, _ = torch.max(similarity, dim=1)
+        distances.append(distance.cpu().tolist())
+
+    distances = np.concatenate(distances)
+
+    df = pd.read_csv(config.input_data)
+    df["distance"] = distances
+    df.to_csv(os.path.join(config.output_dir, "distance.csv"), index=False)

task_forward/prediction.py

@@ -0,0 +1,143 @@
+import argparse
+import gc
+import os
+import sys
+import warnings
+
+import pandas as pd
+import torch
+from torch.utils.data import DataLoader
+from tqdm import tqdm
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from generation_utils import (
+    ReactionT5Dataset,
+    decode_output,
+    save_multiple_predictions,
+)
+from train import preprocess_df
+from utils import seed_everything
+
+warnings.filterwarnings("ignore")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Script for reaction product prediction."
+    )
+    parser.add_argument(
+        "--input_data",
+        type=str,
+        required=True,
+        help="Path to the input data.",
+    )
+    parser.add_argument(
+        "--input_max_length",
+        type=int,
+        default=400,
+        help="Maximum token length of input.",
+    )
+    parser.add_argument(
+        "--output_min_length",
+        type=int,
+        default=1,
+        help="Minimum token length of output.",
+    )
+    parser.add_argument(
+        "--output_max_length",
+        type=int,
+        default=300,
+        help="Maximum token length of output.",
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        type=str,
+        default="sagawa/ReactionT5v2-forward",
+        help="Name or path of the finetuned model for prediction. Can be a local model or one from Hugging Face.",
+    )
+    parser.add_argument(
+        "--num_beams", type=int, default=5, help="Number of beams used for beam search."
+    )
+    parser.add_argument(
+        "--num_return_sequences",
+        type=int,
+        default=5,
+        help="Number of predictions returned. Must be less than or equal to num_beams.",
+    )
+    parser.add_argument(
+        "--batch_size", type=int, default=5, help="Batch size for prediction."
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="./",
+        help="Directory where predictions are saved.",
+    )
+    parser.add_argument(
+        "--debug", action="store_true", default=False, help="Use debug mode."
+    )
+    parser.add_argument(
+        "--seed", type=int, default=42, help="Seed for reproducibility."
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    CFG = parse_args()
+    CFG.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    if not os.path.exists(CFG.output_dir):
+        os.makedirs(CFG.output_dir)
+
+    seed_everything(seed=CFG.seed)
+
+    CFG.tokenizer = AutoTokenizer.from_pretrained(
+        os.path.abspath(CFG.model_name_or_path)
+        if os.path.exists(CFG.model_name_or_path)
+        else CFG.model_name_or_path,
+        return_tensors="pt",
+    )
+    model = AutoModelForSeq2SeqLM.from_pretrained(
+        os.path.abspath(CFG.model_name_or_path)
+        if os.path.exists(CFG.model_name_or_path)
+        else CFG.model_name_or_path
+    ).to(CFG.device)
+    model.eval()
+
+    input_data = pd.read_csv(CFG.input_data)
+    input_data = preprocess_df(input_data, drop_duplicates=False)
+    dataset = ReactionT5Dataset(CFG, input_data)
+    dataloader = DataLoader(
+        dataset,
+        batch_size=CFG.batch_size,
+        shuffle=False,
+        num_workers=4,
+        pin_memory=True,
+        drop_last=False,
+    )
+
+    all_sequences, all_scores = [], []
+    for inputs in tqdm(dataloader, total=len(dataloader)):
+        inputs = {k: v.to(CFG.device) for k, v in inputs.items()}
+        with torch.no_grad():
+            output = model.generate(
+                **inputs,
+                min_length=CFG.output_min_length,
+                max_length=CFG.output_max_length,
+                num_beams=CFG.num_beams,
+                num_return_sequences=CFG.num_return_sequences,
+                return_dict_in_generate=True,
+                output_scores=True,
+            )
+        sequences, scores = decode_output(output, CFG)
+        all_sequences.extend(sequences)
+        if scores:
+            all_scores.extend(scores)
+        del output
+        torch.cuda.empty_cache()
+        gc.collect()
+
+    output_df = save_multiple_predictions(input_data, all_sequences, all_scores, CFG)
+
+    output_df.to_csv(os.path.join(CFG.output_dir, "output.csv"), index=False)

task_forward/train.py

@@ -0,0 +1,312 @@
+import argparse
+import os
+import sys
+import warnings
+from pathlib import Path
+
+import datasets
+import pandas as pd
+import torch
+from datasets import Dataset, DatasetDict
+from transformers import (
+    AutoModelForSeq2SeqLM,
+    AutoTokenizer,
+    DataCollatorForSeq2Seq,
+    EarlyStoppingCallback,
+    Seq2SeqTrainer,
+    Seq2SeqTrainingArguments,
+)
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from utils import (
+    add_new_tokens,
+    canonicalize,
+    filter_out,
+    get_accuracy_score,
+    preprocess_dataset,
+    seed_everything,
+    space_clean,
+)
+
+# Suppress warnings and disable progress bars
+warnings.filterwarnings("ignore")
+datasets.utils.logging.disable_progress_bar()
+
+
+def parse_args():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Training script for reaction prediction model."
+    )
+    parser.add_argument(
+        "--train_data_path", type=str, required=True, help="Path to training data CSV."
+    )
+    parser.add_argument(
+        "--valid_data_path",
+        type=str,
+        required=True,
+        help="Path to validation data CSV.",
+    )
+    parser.add_argument("--test_data_path", type=str, help="Path to test data CSV.")
+    parser.add_argument(
+        "--USPTO_test_data_path",
+        type=str,
+        help="The path to data used for USPTO testing. CSV file that contains ['REACTANT', 'REAGENT', 'PRODUCT'] columns is expected.",
+    )
+    parser.add_argument(
+        "--output_dir", type=str, default="t5", help="Path of the output directory."
+    )
+    parser.add_argument(
+        "--pretrained_model_name_or_path",
+        type=str,
+        required=True,
+        help="Pretrained model path or name.",
+    )
+    parser.add_argument(
+        "--debug", action="store_true", default=False, help="Enable debug mode."
+    )
+    parser.add_argument(
+        "--epochs",
+        type=int,
+        default=5,
+        help="Number of epochs.",
+    )
+    parser.add_argument("--lr", type=float, default=1e-3, help="Learning rate.")
+    parser.add_argument("--batch_size", type=int, default=16, help="Batch size.")
+    parser.add_argument(
+        "--input_max_length",
+        type=int,
+        default=400,
+        help="Max input token length.",
+    )
+    parser.add_argument(
+        "--target_max_length",
+        type=int,
+        default=150,
+        help="Max target token length.",
+    )
+    parser.add_argument(
+        "--eval_beams",
+        type=int,
+        default=5,
+        help="Number of beams used for beam search during evaluation.",
+    )
+    parser.add_argument(
+        "--target_column",
+        type=str,
+        default="PRODUCT",
+        help="Target column name.",
+    )
+    parser.add_argument(
+        "--weight_decay",
+        type=float,
+        default=0.01,
+        help="Weight decay.",
+    )
+    parser.add_argument(
+        "--evaluation_strategy",
+        type=str,
+        default="epoch",
+        help="Evaluation strategy used during training. Select from 'no', 'steps', or 'epoch'. If you select 'steps', also give --eval_steps.",
+    )
+    parser.add_argument(
+        "--eval_steps",
+        type=int,
+        help="Evaluation steps.",
+    )
+    parser.add_argument(
+        "--save_strategy",
+        type=str,
+        default="epoch",
+        help="Save strategy used during training. Select from 'no', 'steps', or 'epoch'. If you select 'steps', also give --save_steps.",
+    )
+    parser.add_argument(
+        "--save_steps",
+        type=int,
+        default=500,
+        help="Save steps.",
+    )
+    parser.add_argument(
+        "--logging_strategy",
+        type=str,
+        default="epoch",
+        help="Logging strategy used during training. Select from 'no', 'steps', or 'epoch'. If you select 'steps', also give --logging_steps.",
+    )
+    parser.add_argument(
+        "--logging_steps",
+        type=int,
+        default=500,
+        help="Logging steps.",
+    )
+    parser.add_argument(
+        "--save_total_limit",
+        type=int,
+        default=2,
+        help="Limit of saved checkpoints.",
+    )
+    parser.add_argument(
+        "--fp16",
+        action="store_true",
+        default=False,
+        help="Enable fp16 training.",
+    )
+    parser.add_argument(
+        "--disable_tqdm",
+        action="store_true",
+        default=False,
+        help="Disable tqdm.",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=42,
+        help="Random seed.",
+    )
+
+    return parser.parse_args()
+
+
+def preprocess_df(df, drop_duplicates=True):
+    """Preprocess the dataframe by filling NaNs, dropping duplicates, and formatting the input."""
+    for col in ["REACTANT", "PRODUCT", "CATALYST", "REAGENT", "SOLVENT"]:
+        if col not in df.columns:
+            df[col] = None
+        df[col] = df[col].fillna(" ")
+    if drop_duplicates:
+        df = (
+            df[["REACTANT", "PRODUCT", "CATALYST", "REAGENT", "SOLVENT"]]
+            .drop_duplicates()
+            .reset_index(drop=True)
+        )
+    df["REAGENT"] = df["CATALYST"] + "." + df["REAGENT"] + "." + df["SOLVENT"]
+    df["REAGENT"] = df["REAGENT"].apply(lambda x: space_clean(x))
+    df["REAGENT"] = df["REAGENT"].apply(lambda x: canonicalize(x) if x != " " else " ")
+    df["input"] = "REACTANT:" + df["REACTANT"] + "REAGENT:" + df["REAGENT"]
+    return df
+
+
+def preprocess_USPTO(df):
+    df["REACTANT"] = df["REACTANT"].apply(lambda x: str(sorted(x.split("."))))
+    df["REAGENT"] = df["REAGENT"].apply(lambda x: str(sorted(x.split("."))))
+    df["PRODUCT"] = df["PRODUCT"].apply(lambda x: str(sorted(x.split("."))))
+
+    df["input"] = "REACTANT:" + df["REACTANT"] + "REAGENT:" + df["REAGENT"]
+    df["pair"] = df["input"] + " - " + df["PRODUCT"].astype(str)
+
+    return df
+
+
+if __name__ == "__main__":
+    CFG = parse_args()
+    CFG.disable_tqdm = True
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    seed_everything(seed=CFG.seed)
+
+    # Load and preprocess data
+    train = preprocess_df(
+        filter_out(pd.read_csv(CFG.train_data_path), ["REACTANT", "PRODUCT"])
+    )
+    valid = preprocess_df(
+        filter_out(pd.read_csv(CFG.valid_data_path), ["REACTANT", "PRODUCT"])
+    )
+    if CFG.USPTO_test_data_path:
+        train_copy = preprocess_USPTO(train.copy())
+        USPTO_test = preprocess_USPTO(pd.read_csv(CFG.USPTO_test_data_path))
+        train = train[~train_copy["pair"].isin(USPTO_test["pair"])].reset_index(
+            drop=True
+        )
+    train["pair"] = train["input"] + " - " + train["PRODUCT"]
+    valid["pair"] = valid["input"] + " - " + valid["PRODUCT"]
+    valid = valid[~valid["pair"].isin(train["pair"])].reset_index(drop=True)
+    train.to_csv("train.csv", index=False)
+    valid.to_csv("valid.csv", index=False)
+
+    if CFG.test_data_path:
+        test = preprocess_df(
+            filter_out(pd.read_csv(CFG.test_data_path), ["REACTANT", "PRODUCT"])
+        )
+        test["pair"] = test["input"] + " - " + test["PRODUCT"]
+        test = test[~test["pair"].isin(train["pair"])].reset_index(drop=True)
+        test = test.drop_duplicates(subset=["pair"]).reset_index(drop=True)
+        test.to_csv("test.csv", index=False)
+
+    dataset = DatasetDict(
+        {
+            "train": Dataset.from_pandas(train[["input", "PRODUCT"]]),
+            "validation": Dataset.from_pandas(valid[["input", "PRODUCT"]]),
+        }
+    )
+
+    # load tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(
+        os.path.abspath(CFG.pretrained_model_name_or_path)
+        if os.path.exists(CFG.pretrained_model_name_or_path)
+        else CFG.pretrained_model_name_or_path,
+        return_tensors="pt",
+    )
+    tokenizer = add_new_tokens(
+        tokenizer,
+        Path(__file__).resolve().parent.parent / "data" / "additional_tokens.txt",
+    )
+    tokenizer.add_special_tokens(
+        {
+            "additional_special_tokens": tokenizer.additional_special_tokens
+            + ["REACTANT:", "REAGENT:"]
+        }
+    )
+    CFG.tokenizer = tokenizer
+
+    # load model
+    model = AutoModelForSeq2SeqLM.from_pretrained(
+        os.path.abspath(CFG.pretrained_model_name_or_path) if os.path.exists(CFG.pretrained_model_name_or_path) else CFG.pretrained_model_name_or_path
+    )
+    model.resize_token_embeddings(len(tokenizer))
+
+    tokenized_datasets = dataset.map(
+        lambda examples: preprocess_dataset(examples, CFG),
+        batched=True,
+        remove_columns=dataset["train"].column_names,
+        load_from_cache_file=False,
+    )
+
+    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
+
+    args = Seq2SeqTrainingArguments(
+        CFG.output_dir,
+        evaluation_strategy=CFG.evaluation_strategy,
+        eval_steps=CFG.eval_steps,
+        save_strategy=CFG.save_strategy,
+        save_steps=CFG.save_steps,
+        logging_strategy=CFG.logging_strategy,
+        logging_steps=CFG.logging_steps,
+        learning_rate=CFG.lr,
+        per_device_train_batch_size=CFG.batch_size,
+        per_device_eval_batch_size=CFG.batch_size,
+        weight_decay=CFG.weight_decay,
+        save_total_limit=CFG.save_total_limit,
+        num_train_epochs=CFG.epochs,
+        predict_with_generate=True,
+        fp16=CFG.fp16,
+        disable_tqdm=CFG.disable_tqdm,
+        push_to_hub=False,
+        load_best_model_at_end=True,
+    )
+
+    model.config.eval_beams = CFG.eval_beams
+    model.config.max_length = CFG.target_max_length
+    trainer = Seq2SeqTrainer(
+        model,
+        args,
+        train_dataset=tokenized_datasets["train"],
+        eval_dataset=tokenized_datasets["validation"],
+        data_collator=data_collator,
+        tokenizer=tokenizer,
+        compute_metrics=lambda eval_preds: get_accuracy_score(eval_preds, CFG),
+        callbacks=[EarlyStoppingCallback(early_stopping_patience=10)],
+    )
+
+    try:
+        trainer.train(resume_from_checkpoint=True)
+    except:
+        trainer.train(resume_from_checkpoint=None)
+    trainer.save_model("./best_model")

task_retrosynthesis/accuracy-and-invalidity-check.ipynb

@@ -0,0 +1,207 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "43813b12",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prediction: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 139/139 [35:54<00:00, 15.50s/it]\n",
+    "Top-1: 0.3% || Invalid 15.75%\n",
+    "Top-2: 0.5% || Invalid 22.04%\n",
+    "Top-3: 0.7% || Invalid 25.83%\n",
+    "Top-4: 0.9% || Invalid 28.69%\n",
+    "Top-5: 1.1% || Invalid 30.74%\n",
+    "prediction: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 139/139 [36:00<00:00, 15.55s/it]\n",
+    "Top-1: 0.3% || Invalid 23.68%\n",
+    "Top-2: 0.5% || Invalid 28.60%\n",
+    "Top-3: 0.7% || Invalid 32.01%\n",
+    "Top-4: 0.9% || Invalid 34.58%\n",
+    "Top-5: 1.0% || Invalid 36.95%\n",
+    "prediction: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 139/139 [35:03<00:00, 15.13s/it]\n",
+    "Top-1: 0.1% || Invalid 29.90%\n",
+    "Top-2: 0.1% || Invalid 34.33%\n",
+    "Top-3: 0.2% || Invalid 37.83%\n",
+    "Top-4: 0.3% || Invalid 40.49%\n",
+    "Top-5: 0.4% || Invalid 43.11%\n",
+    "prediction: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 139/139 [35:27<00:00, 15.31s/it]\n",
+    "Top-1: 0.0% || Invalid 55.78%\n",
+    "Top-2: 0.1% || Invalid 58.94%\n",
+    "Top-3: 0.1% || Invalid 61.21%\n",
+    "Top-4: 0.1% || Invalid 63.35%\n",
+    "Top-5: 0.1% || Invalid 65.17%\n",
+    "prediction: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 139/139 [35:27<00:00, 15.30s/it]\n",
+    "Top-1: 0.1% || Invalid 44.12%\n",
+    "Top-2: 0.1% || Invalid 48.06%\n",
+    "Top-3: 0.1% || Invalid 51.93%\n",
+    "Top-4: 0.1% || Invalid 54.31%\n",
+    "Top-5: 0.2% || Invalid 56.56%\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "cf10c9e8",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/tmp/ipykernel_2055775/4280584905.py:21: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
+      "  ax.set_yticklabels([int(i) for i in ax.get_yticks()], fontsize=12)\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "<matplotlib.legend.Legend at 0x7f7834dea750>"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAp4AAAGzCAYAAACGrBfmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAABluklEQVR4nO3dd3hUVeLG8e9MJpkkhCSEUBIgIEV6pIgCKwpYEAUVFHtZXd3dALqiKEWlqijiz7r2thZcUcFCU9TggoANUapIDR1SSJ+0ub8/DplkSIIEkpkkvJ/nmQfm3DMzZ5jh5s25p9gsy7IQEREREalmdn83QERERERODQqeIiIiIuITCp4iIiIi4hMKniIiIiLiEwqeIiIiIuITCp4iIiIi4hMKniIiIiLiEwqeIiIiIuITCp4iIiIi4hMnFDyXL1/OJZdcQoMGDQgJCaFdu3ZMnz7dq87q1au54IILCAsLIzIykuHDh7Nt27YqabSIiIiI1D6VDp6zZ8/mvPPOIyIigrfffpuFCxcybtw4Su+8uWnTJvr3709+fj5z5szhjTfeYPPmzfTr149Dhw5V6RsQERERkdrBVpm92vfs2UP79u25+eabeeGFFyqsd/XVV5OYmMjWrVsJDw8HYOfOnbRr144xY8bw+OOPn3zLRURERKRWqVSP52uvvUZ2djbjxo2rsE5hYSHz58/nyiuv9IROgJYtWzJgwADmzZt34q0VERERkVrLUZnK//vf/4iKimLTpk1cfvnlrFu3jqioKIYPH87MmTMJDw9n69at5ObmEh8fX+bx8fHxLFmyBJfLRXBwcLmvkZeXR15enue+2+0mNTWVhg0bYrPZKvn2RERERKS6WZZFZmYmsbGx2O0V92tWKnju2bOHnJwcRowYwYQJE3j66af58ccfmTx5MuvWrWPZsmWkpKQAEBUVVebxUVFRWJZFWloaMTEx5b7GjBkzmDp1amWaJSIiIiI1wK5du2jevHmFxysVPN1uNy6Xi8mTJzN+/HgA+vfvT1BQEHfffTdff/01oaGhAMfsnTzWsQkTJnDPPfd47qenpxMXF8euXbu8Lt2LiIiISM2QkZFBixYtqF+//jHrVSp4NmzYkD/++INBgwZ5lQ8ePJi7776b1atXc/nllwN4ej5LS01NxWazERkZWeFrOJ1OnE5nmfLw8HAFTxEREZEa7M+GRVZqclF54zYBz1JKdrudNm3aEBISwtq1a8vUW7t2LW3btq1wfKeIiIiI1F2VCp5XXnklAIsWLfIqX7hwIQC9e/fG4XAwdOhQ5s6dS2ZmpqdOUlISiYmJDB8+/GTbLCIiIiK1UKXW8QS47LLL+PLLL3nwwQfp3bs3P/30E1OnTuWCCy7g888/B8wC8r169aJHjx6MHz8el8vFpEmTSE1NZc2aNTRq1Oi4Xy8jI4OIiAjS09N1qV1ERESkBjrevFbp4Jmbm8vUqVOZPXs2+/btIzY2lhtuuIHJkyd7jc38+eefGTduHCtXrsThcDBw4EBmzZpFmzZtquWNiIiIiIh/VFvw9DUFTxEREZGa7XjzWqVmtdd0lmVRVFREYWGhv5si4iUwMJCAgAB/N0NERMSv6kTwtCyLw4cPc+jQIYqKivzdHJFyRUZG0rRpU+3AJSIip6w6ETz379/P4cOHPWt9OhwO/XCXGsOyLHJycjh48CBAhbt2iYiI1HW1PngWFRWRnp5Oo0aNiI6O9ndzRMoVEhICwMGDB2ncuLEuu4uIyCmpUut41kQFBQVYlkW9evX83RSRYyreTragoMDPLREREfGPWh88i+nSutR0+o6KiMiprs4ETxERERGp2RQ8RURERMQnFDyPxe2GtWvhf/8zf7rdPm/CW2+9hc1m89wcDgcxMTFce+21/PHHHz5vz9EeffRRPvnkkzLlS5cuxWazsXTp0mp53SlTpnj9u1R069+//zHrBwcHV0v7REREpKxaP6u92qxYAc8/Dxs3Ql4eOJ3QsSOMHg19+/q8OW+++SYdOnTA5XLx3Xff8cgjj5CYmMimTZto0KCBz9tT7NFHH+Wqq67iiiuu8Crv0aMHK1eupFOnTtXyurfffjsXX3yx5/6+ffsYPnw4d955J9dff72n/OjdExYvXkxERITnvt2u371ERER8RcGzPCtWwNixkJoKMTEQEgK5ufDLL6Z81iyfh88uXbpw5plnAtC/f3+KioqYPHkyn3zyCbfeeqtP23I8wsPD6d27d7U9f/PmzWnevLnn/o4dOwCIi4s75uv27NlTy26JiIj4Sd3t7rEscLkqf8vJgWeegZQUaN0aQkPBZjN/nnaaKX/2WVOvMs9rWVX69opD6IEDBzxlP/30E5dddhlRUVEEBwfTvXt35syZ4/W4Q4cOMXLkSDp16kRYWBiNGzdm4MCBLFu2rMxr5OXlMW3aNDp27EhwcDANGzZkwIABrFixAjCztLOzs/nPf/5T5tJ2RZfaP/vsM/r06UNoaCj169fnwgsvZOXKlV51ii+Lr1+/nuuuu46IiAiaNGnCbbfdRnp6+sn+04mIiIif1N0ez7w8GDGi8o/LyIA1a8DhgMOHyx4vLIRFi2DwYDjqMu4xffghVOF4wu3btwNw+umnA5CYmMjFF1/M2WefzUsvvURERAT//e9/ueaaa8jJyeGvf/0rAKmpqQBMnjyZpk2bkpWVxbx58+jfvz9ff/21JzgWFhYyePBgli1bxt13383AgQMpLCxk1apVJCUl0bdvX1auXMnAgQMZMGAADz30EFD20nZps2fP5oYbbuCiiy7i/fffJy8vj5kzZ3pe+5xzzvGqf+WVV3LNNdfwt7/9jbVr1zJhwgQA3njjjRP+d+vatSsHDx4kOjqaQYMG8fDDDxMXF3fCzyciIiLHr+4GzxNVUGAmEVW0s0xAAOTnm3o+VFRURGFhoWeM58MPP8y5557LZZddBsDIkSPp3Lkz33zzDQ6H+VgHDRpEcnIyEydO5Oabb8Zut9O+fXteeOEFr+cdNGgQO3bs4Nlnn/UEz/fff5/ExEReffVVbr/9dk/9oUOHev7eu3dv7HY7jRo1+tPL6m63m/vuu4+uXbuyaNEiz9jKSy65hDZt2jBu3Di+++47r8f87W9/47777gPgggsuYMuWLbzxxhu8/vrrlV4Ts02bNjzyyCN0796d4OBgfvjhB2bOnMmXX37Jzz//TLNmzSr1fCIiIlJ5dTd4Op2ml7Gy1q2D22+HiAgICyt7PCsL0tPN5fYuXSrXnpNwdLDr2LEjn376KQ6Hgy1btrBp0yZmzZoFmN7KYpdccgnz58/n999/p2PHjgC89NJLvPLKK2zYsIG8vDxP3Q4dOnj+vmjRIoKDg7nttttOqt3Ffv/9d/bu3cvdd9/tNaEnLCyMK6+8kpdffpmcnBzP7j6AJ1QXi4+Px+VycfDgQZo0aVKp17/pppu87g8YMIABAwbQp08fZs6cyTPPPHMC70pEREQqo+6O8bTZzKXtyt569IBOneDgQfMcdnvJzWYz5Z07m3qVed6T3LXm7bff5scff+Sbb77hH//4Bxs3buS6664DSsZ5jh07lsDAQK/byJEjAUhOTgbg//7v/0hISODss8/m448/ZtWqVfz4449cfPHF5Obmel7v0KFDxMbGVtms75SUFABiYmLKHIuNjcXtdpOWluZV3rBhQ6/7ziPhvXQ7T8ZZZ53F6aefzqpVq6rk+UREROTY6m6P54my282SSWPHwtat0LRpyaz2/fuhQQMYNcrU86GOHTt6JhQNGDCAoqIiXnvtNT766CO6du0KwIQJExg+fHi5j2/fvj0A7777Lv379+fFF1/0Op6Zmel1v1GjRixfvhy3210l4bM4RO7bt6/Msb1792K32/2yLJRlWVpSSURExEf0E7c8ffuaJZO6dzcTjHbsMH/26OGXpZTKM3PmTBo0aMCkSZNo164d7dq149dff+XMM88s91a/fn3AzER3HnXZ/7fffiszs3zw4MG4XC7eeuutY7bD6XQeVw9k+/btadasGbNnz8YqNcM/Ozubjz/+2DPT3ZdWrVrFH3/8Ua3LPomIiEgJ9XhWpG9f6N0b1q+HtDTT09m5s897OivSoEEDJkyYwP3338/s2bN5+eWXGTx4MIMGDeKvf/0rzZo1IzU1lY0bN7J69Wo+PDLedciQIUyfPp3Jkydz3nnn8fvvvzNt2jROO+00r7Gh1113HW+++Sb//Oc/+f333xkwYABut5vvv/+ejh07cu211wJmlvjSpUv5/PPPiYmJoX79+p7e1dLsdjszZ87khhtuYMiQIfzjH/8gLy+PJ554gsOHD/PYY49V67/XGWecwY033uhZGuqHH37giSeeoGnTptx///3V+toiIiJiKHgei90ORy5j10R33nknzz//PNOmTWPjxo388MMPPPLII9x9992kpaXRsGFDOnXqxNVXX+15zAMPPEBOTg6vv/46M2fOpFOnTrz00kvMmzfPa81Nh8PBwoULmTFjBu+//z5PP/009evX54wzzvDaMeiZZ55h1KhRXHvtteTk5HDeeedVuE3m9ddfT7169ZgxYwbXXHMNAQEB9O7dm8TERPpWcy9yp06deOWVV9i3bx/5+fnExsZy7bXXMmnSpHLHnYqIiNRabneN7TizWVYVr2xexTIyMoiIiCA9Pb3cNSJdLhfbt2/ntNNO077bUqPpuyoiItXOT1t+/1leK1Yz4q+IiIiInJziLb9Xr4bISGjVyvxZvOX3kZ0H/UnBU0RERKS2c7tNT2dqKjRrZja7SU83a5K3aWMuu//736aeH2mMp4iIiEhtY1lmmcekJNi5E1atgiVLTHnxutihoRAVZdYSb9oUNmwwYz/9OH9FwVNERESkprIs04u5c2fJLSnJ3ErtPkhKCrhcZu3xgADzZ716JcdDQuDAgZJQ6icKniIiIiI1QWamd8AsvmVnl18/MBBatICWLc39lBRo1MjMZD9abq6ZaOSHzVpKU/AUERER8aXc3JJey9IBs6LeSLvdjNts2dLc4uLMnzExJcskud3w009mIlFkpPdW3cWX5Xv0MEsr+ZGCp4iIiEh1yM+H3btLLo8XB8yDByt+TNOmJQGzOGQ2a2Z6N4+lhm75fTQFTxEREZGTUVQE+/aVvUS+d6/pbSxPVJR3wGzZ0lw2P5l1nou3/C5ex/PAAXN5vUcPEzprwJbfCp4iIiIix8Oy4NChsgFz1y4ote20l7Aws55m8eXx4l7M+vWrp401fMtvBU8RERGR0oqXJCp9ebz47y5X+Y8JDvYOl8UBs0ED7/GWvlCDt/xW8KxFfvvtN5566imWLl3Kvn37cDgcnH766Vx77bXcfvvtREVF+buJtUb//v0BWLp0Kf379+fbb7/908dMnjyZKVOmVFh/0KBBLF68uKqbKiIi1Skrq+xSRTt3mhnm5XE4SmaSl741auT7gFkLKXiW49AhyMio+Hh4uPl++dKrr77KyJEjad++Pffddx+dOnWioKCAn376iZdeeomVK1cyb9483zaqjnjhhRfIKPWBL1iwgIcffpg333yTDh06eMqbN2/u+Xvr1q157733vJ4nMjKy2tsqIiInyOUyl8SPvkyemlp+fZsNYmPLBsyYGLNOppwQBc+jHDoE119vlsKqSMOGMHu278LnypUrSUhI4MILL+STTz7B6XR6jl144YXce++96mk7CZ06dfK6v2nTJgC6dOnCmWeeWe5jQkJC6N27d7W3TUREKqmw0MwkP3qpov37K35M48Zllypq3hyCgnzX7lNEzRhpWoNkZJjQ6XSaZbCOvjmd5vixekSr2qOPPorNZuOVV17xCp3FgoKCuOyyywBwu93MnDmTDh064HQ6ady4MTfffDO7d+/2ekz//v3p0qULK1eupG/fvoSEhNCqVSvefPNNwPT69ejRg9DQULp27Vom2E6ZMgWbzcYvv/zC8OHDCQ8PJyIightvvJFDhw551T3eNrVq1Yq//vWvZd5f//79PZfGwVwet9lsvP/++zzwwAPExsYSHh7OBRdcwO+//+71WMuymDlzJi1btiQ4OJgePXqwaNGiY/+Di4hIzed2m1njK1fCf/8Ljz8OI0fCVVfBnXfCE0/AnDnw/fcloTMyEs44Ay67zNSZNQs++ABefx0mTYJbboEBA6B1a4XOalLnezwrGgMMZuxt6e+Vy2V2n3K7TcAMCSlbH0ydvLyKn/vo5z0ZRUVFfPPNN/Ts2ZMWLVr8af2EhAReeeUVRo8ezZAhQ9ixYwcPPfQQS5cuZfXq1URHR3vq7t+/n1tvvZX777+f5s2b89xzz3Hbbbexa9cuPvroIyZOnEhERATTpk3jiiuuYNu2bcTGxnq93rBhw7j66qv55z//yfr163nooYfYsGED33//PYFH1hyrTJsqY+LEifzlL3/htddeIyMjg3HjxjF06FA2btxIwJHLIFOnTmXq1Kn87W9/46qrrmLXrl3ccccdFBUV0b59+xN6XYCtW7cSFRVFRkYGLVu25Nprr+XBBx8k5OgvjYiInBzLguTkspfId+8262SWp169sj2YcXEQEeHbtksZdT54jhhR8bEzz4TJk0vu33ijGeqxfbsZO+wo9a9Tvz507Fhyf/z4ipfmatcO/u//Tq7dxZKTk8nJyeG0007707qbNm3ilVdeYeTIkTz33HOe8u7du3P22Wfz1FNP8cgjj3jKU1JS+OKLL+jZsycAZ555Jo0bN+axxx5jy5YtnpAZGxtLt27d+Pjjj7nzzju9XnP48OHMnDkTgIsuuogmTZpwww03MGfOHG644YZKt6kyOnXqxLvvvuu5HxAQwNVXX82PP/5I7969OXz4MI8//jjDhg3jtdde89Tr3Lkzf/nLX044eJ5zzjlcc801dOjQgdzcXBYtWsTMmTNZvnw5iYmJ2GvIkhUiIrVOenrZgJmUBDk55dcPCvIOl8W3qChN9Kmh6nzwPJUkJiYClLlcfdZZZ9GxY0e+/vprr5AXExPjCZ0AUVFRNG7cmFatWnn1bHY8krh37txZ5jVvuOEGr/tXX301t9xyC4mJidxwww2VblNlFA8vKBYfH+9pZ+/evVm5ciUul6tMG/v27UvL4n1tT8DDDz/sdf+SSy6hVatWjB07lk8//ZRhw4ad8HOLiJwSsrNLxmCWHouZnl5+/YAAM+by6KWKmjSpMetTyvGp88Hzww8rPnb0d/Xdd2HbNjO5KDLS9NRX5LHHzBCQ43nekxEdHU1oaCjbt2//07opR2ZExcTElDkWGxtbJjiWt/xSUFBQmfKgI+MGXOWMLWjatKnXfYfDQcOGDT1tqWybKqNhw4Ze94vHv+bm5nq99tFtrKjsZNx4442MHTuWVatWKXiKiBTLzy9/Jnlycvn1bbayW0a2bGlmlzvqfGQ5JdT5T7EyO08FB5uxnXZ7ya0iTufJ7Wp1vAICAjj//PNZtGgRu3fv9lrS52jFQWzfvn1l6u3du/eEx1Iey/79+2nWrJnnfmFhISkpKZ62VKZNwcHB5OXllXmN5OTkE2p78WvvL2cm4/79+2nVqlWln/PP6DK7iJySCgvNRJ+jA+b+/RWPS4uOLnuZvEUL8wNW6qw6HzxP1JFOs+Mur04TJkxg4cKF3HHHHXz66aeeHshiBQUFLF68mIEDBwLw7rvv0qtXL8/xH3/8kY0bN/LAAw9Uedvee+89r8v1c+bMobCw0DMLvTJtatWqFb/99pvX82/evJnff//9hIJn7969CQ4O5r333uPKK6/0lK9YsYKdO3dWafD8z3/+43lNEZE6y7JMmDx6qaI9eyreMjI83GwZefREn2NdVpQ6S8HzKOHhZp3OlBQzc708DRuaer7Sp08fXnzxRUaOHEnPnj1JSEigc+fOFBQU8Msvv/DKK6/QpUsX5s2bx9///neee+457HY7gwcP9swgb9GiBWPGjKnyts2dOxeHw8GFF17omdV+xhlncPXVVwPQvn37427TTTfdxI033sjIkSO58sor2blzJzNnzqTRCS6Y2qBBA8aOHcvDDz/M7bffzogRI9i1axdTpkw54Uvty5Yt45FHHmHYsGG0bt0al8vFokWLeOWVVxg4cCBDhw49oecVEalRLMvMtj16kk9SUsU/HENCyl4ib9lSM8nFi4LnURo1MovD17Sdi+644w7OOussnnrqKR5//HH2799PYGAgp59+Otdffz2jR48G4MUXX6RNmza8/vrr/Pvf/yYiIoKLL76YGTNmlBkTWRXmzp3LlClTePHFF7HZbAwdOpSnn37aq1f2eNt0/fXXs3fvXl566SXefPNNunTpwosvvsjUqVNPuH3Tpk2jXr16vPDCC7zzzjt06NCBl156iVmzZp3Q88XExBAQEMD06dNJTk7GZrPRrl07pk2bxr333qtL7SJS+2RklJ3ks3OnmQBUnsDA8reMjI7WTHL5UzbLqmjwRc2QkZFBREQE6enphJfTzehyudi+fTunnXYawb4YdCmAWUB+6tSpHDp0qFrGjtZF+q6KiF/l5pYNlzt3wuHD5de326FZs7LrYcbEaCa5lPFnea2YejxFRETqkvx8s7j60b2YBw9W/JijZ5LHxZnQeWQjEJGqouApIiJSGxUVwb59ZXsw9+6teCZ5VFTZS+QtWvhmmRYRKhk8ly5dyoABA8o9tnLlSq8ZvatXr+b+++9n1apVOBwOBg4cyKxZs2hd0eKXUqtMmTKFKVOm+LsZIiJ1n2XBoUNlA+auXRXPJA8LMzPJSy9VFBdntuET8aMT6vF89NFHywTQLl26eP6+adMm+vfvT7du3ZgzZw4ul4tJkybRr18/1qxZc8KzlEVEROosy4K0tLLjMJOSoJwNPADTU3n0dpFxcdCggSb6SI10QsGzXbt2x1yvcNKkSTidTubPn+8ZYNqzZ0/atWvHrFmzePzxx0+stSIiInVBZqZ3wCz+e2Zm+fUdjvJnkjdqpIAptUqVj/EsLCxk/vz53HzzzV6zmlq2bMmAAQOYN2+egqeIiJwaXK7yt4xMTS2/vs1mtoc8OmDGxJj9ykVquRMKnqNGjeLaa68lNDSUPn368NBDD3HOOecAsHXrVnJzc4mPjy/zuPj4eJYsWYLL5dJyMiIiUncUFJjde46+TF7Olr0ejRuXXaqoeXM4anc6kbqkUsEzIiKCf/3rX/Tv35+GDRuyZcsWnnjiCfr378+CBQsYNGgQKSkpAERFRZV5fFRUFJZlkZaWRkxMTLmvkZeX57Vfd8axVnIXERHxJbe7ZCZ56ZC5Z485Vp7IyPJnkoeG+rTpIjVBpYJn9+7d6d69u+d+v379GDZsGF27duX+++9n0KBBnmO2Y4w5OdaxGTNmnNRONSIiIifNsiA5uewl8t27zTqZ5alXr2wPZlyctowUKeWkx3hGRkYyZMgQXnrpJXJzcz1bIBb3fJaWmpqKzWYjMjKywuebMGEC99xzj+d+RkYGLVq0ONlmioiIlC89vWzA3LnT7PRTnqAg73BZfIuK0kQfkT9RJZOLinfdtNlstGnThpCQENauXVum3tq1a2nbtu0xx3c6nU6cTmdVNEtERKREdnbJ5fHSl8nT08uvHxBgxlwevVRRkybaMlLkBJ108ExLS2P+/Pl069bNEyiHDh3K3LlzmTlzJvWPLFablJREYmIiY8aMOdmXrHbJOcmkuyo4EQERwRFEh1b//uTHGpJQWmJiIv3796+w/owZMxg/fnyZ8mXLlvHcc8/x3XffcejQIUJCQujcuTM33XQTN998M/Xq1fO0Y9SoUTz//PMn/mZERHwlL897JnlxyExOLr++zVZ2y8iWLc3scoc2+BOpSpX6H3X99dcTFxfHmWeeSXR0NH/88QdPPvkkBw4c4K233vLUmzp1Kr169WLIkCGMHz/es4B8dHQ09957b1W/hypVUFTAP+f/k21p2yqs07pBa96/8n0CA6p3D9uVK1d63Z8+fTqJiYl88803XuWdOnXy/P2qq64q828cFxdX5rknT57MtGnT6Nu3L9OnT6dNmzbk5OSwYsUKpkyZwubNm3nqqaeq8N2IiFSxwkKzPeTRl8j37694y8jo6LKXyVu0AF1pE/GJSgXP+Ph4PvjgA1566SWysrKIiorinHPO4Z133qFXr16eeh06dGDp0qWMGzeOq666ymvLzJq+a5HD7qBZ/Wb8sv8XWka0LHN8Z/pOmtVvhsNe/b8FH71If6NGjbDb7cdcvL9JkybHPA7w4YcfMm3aNP72t7/x6quvevWUDh48mPvvv79M6BUR8RvLMmHy6KWK9uypeMvI8HCzZeTRE32OXMkREf+oVHoaP358uZdsy9OzZ0+++uqrE2pUVXIVmm3GnAFOT8AqdBdS6C4kwBbg1WtZXPfmM25mWdIyCt2FhDvDzRhWG2TmZRLhjOCWbreQV5RX4fPabXaCAmruOmzTpk2jQYMGPPvss+Venq9fvz4XXXRRmfJ33nmHRx99lJ07d9KuXTseeeQRhgwZ4lXnjz/+YPLkyXz11Vekp6fTunVrRo8ezahRozx1li5dyoABA3jvvff49ddfefvtt8nMzGTAgAG89tprhIaGcu+99zJ37lzADN147rnnCAsLq+J/CRGpUSzLLKx+9HaRSUnm8nl5QkLKXiJv2VIzyUVqqDo/eGXEhyMAeHfYu0QEmxPR3I1zeee3d7io9UXcefadnro3zr2RvKI8Xhv6Gv3i+vHF1i/IKchhV8YuooKjsLAY1GYQ3Zt258Z5N5KRl8G/L/k3cRHmUvbX277m+R+f5+xmZ/PguQ/6/s0Cs2fP5vXXX8ftdtOlSxdGjx7Nrbfe6jm+b98+1q1bxzXXXENoJdaQW7BgAT/++CPTpk0jLCyMmTNnMmzYMH7//Xdat24NwIYNG+jbty9xcXE8+eSTNG3alC+++IK77rqL5ORkJk+e7PWcEydOZMCAAbz11lvs2LGDsWPHct111+FwODjjjDN4//33+eWXX5g4cSL169fn2WefrZp/JBHxv4yMspfIk5LMBKDyBAaWv2VkdLRmkovUInU+eJ4Im83GLd1uYVnSMnIKcgBwFbloENyAW7rdctyTfnzt+uuv59JLL6VFixYcPHiQ119/ndtuu41t27Yxffp0wEzyAjjttNMq9dy5ubl89dVXnsliPXr0IDY2ljlz5nh6we+55x7q16/P8uXLPdulXnjhheTl5fHYY49x11130aBBA89zxsfH8+abb3rub9q0iaeffpq77rqLJ554wvP4lStX8t577yl4itRGubllL5Hv3AmHD5df326HZs3KrocZE6OZ5CJ1QJ0Pnh+O+BAwl8SLDe84nMvaX0aAzXvf23eHv+up2yi0Ef3i+rF4y2J6Nu3J1rSt9IvrR/emZgH91y97vczznt/6fM5rdR52m39Oju+9957X/SuvvJKhQ4d6Qt/JjK8dMGCAJ3SCGUvauHFjdu7cCYDL5eLrr78mISGB0NBQCkuNu7rkkkt4/vnnWbVqFYMHD/aUH32ZvmPHjgBceumlZco/+eQTsrKydLldpKbKzzeLqx+9VNHBgxU/5uiZ5HFxJnQGVu/ETRHxnzofPIMdZdcMddgd5U4OOrpuca/nnqw91Auq59XbWZnn9acbb7yR+fPn89NPPzF48GDPDPft27dX6nmKNwYozel0kntkgeWUlBQKCwt57rnneO6558p9juSjljI5elvVoCP7E1dU7nK5FDxF/K2oyMwkP7oXc+/eimeSR0WVv2XkMdZ0FpG6qWalpBqme9Pu9Ivrx7xN8xjWYZint7M2KV7c337kElVMTAxdu3blyy+/JCcnp1LjPI+lQYMGBAQEcNNNN3lNJCqtspf3RcSPLMv0Vpa3ZWRFM8nDwsxM8tJLFcXFQamrJSJyalPwPAabzcat3W8luyCbW7vfWmPHdh7LO++8Q2BgID179vSUPfTQQ1x99dXcddddZZZTAsjKymLFihXlzmyvSGhoKAMGDOCXX34hPj7e00spIjWcZUFaWtkezKQkcLnKf0xwcNntIuPioEEDTfQRkWNS8PwT3Zp247XLXvN3M/7UE088wYYNGzj//PNp3ry5Z3LRl19+yZQpU4iOLtlpacSIETz00ENMnz6dTZs28be//c2zgPz333/Pyy+/zDXXXFOp4AnwzDPPcM4559CvXz8SEhJo1aoVmZmZbNmyhc8//7zMwvci4mOZmd4Bs/jvmZnl13c4yp9J3qiRAqaInBAFzzqiQ4cOfPbZZyxYsIC0tDRCQkLo1q0b77//Ptdee22Z+tOmTeOCCy7gueee44EHHiA5OdmzZeY999zDP/7xj0q3oVOnTqxevZrp06fz4IMPcvDgQSIjI2nXrh2XXHJJVbxNETkeLpf3lpHFt9TU8uvbbGZ7yKMDZkyM2a9cRKSK2CyrotHgNUNGRgYRERGkp6d7lugpzeVysX37dk477TTPXvEiNZG+q1LlCgrM7j1HXyLfv7/ixzRuXHapoubNQcNjROQk/FleK6YeTxGRms7thn37yi5VtGePOVaeyMjyZ5JX0YRCEZEToeApIlJTWBYkJ5c/kzw/v/zH1KtXtgczLk5bRopIjaTgKSJSWW43rF9vZoM3aACdO1d+V5309LIBc+dOs9NPeYKCvMNl8S0qShN9RKTWUPAUEamMFSvg+edh40bIywOnEzp2hNGjoW/fsvWzs8vfMjIjo/znDwgwYy6PXqqoSRNtGSkitV6dCZ41fI6UiL6jdcGKFTB2rJkdHhMDISGmh/KXX+Cee8ytYUPvsZhH7djlYbOV3TKyZUszu9xRZ07NIiJeav3ZLeDIUh8FBQWEhIT4uTUiFSvev96hUFE7ud2mpzM1Fdq2hawsM+EnNxdycszyRWPGQPfuZS99R0eXvUzeooXpLRUROYXU+p+AgYGBOJ1O0tPTqV+/fq3cXUhODRkZGQQEBHh+WZJaZv162LDB9HJu2GAuoZfmdJoQGhMDvXp5T/SpV88/bRYRqWFqffAEiI6OZs+ePezevZuIiAgCAwMVQKXGsCyL7OxsMjIyiImJ0XezNsrJgc8+gx07zCQfm82Mt4yKMqEyJMSU79kDN90E557r7xaLiNRIdSJ4Fi9UmpyczJ49e/zcGpGybDYbkZGRRGiJm9olORk+/xwWLzaLsluWCZ3Nm5uF2EsPm8jKMr2eDRr4r70iIjVcnQieYMJneHg4BQUFFBUV+bs5Il4CAwN1ib022bYN5s2DZcug+HzSsaPp5dy3z1xOL91zbVkmmPboYZZWEhGRctWZ4FksMDCQwMBAfzdDRGoby4LVq2HuXPjtt5Lyrl1h2DA480xYudLMat+61cxIL57Vvn+/6ekcNUpLHomIHEOdC54iIpVSUABLl8Inn5glkMCEx3POMYGzbduSun37wqxZJet4HjhgLq/36GFCZ3nreIqIiIeCp4icmjIzYdEimD/f7EAEpgdz0CC47DJo1Kj8x/XtC717n/zORSIipyAFTxE5tezbB59+Cl99ZXYeArPO5mWXwUUXHd/SR3a7uQQvIiKVouApIqeGTZvMhKGVK814ToDWrc3l9HPO0W5BIiI+oDOtiNRdbjesWmUC56ZNJeU9e8Lw4abXUuuqioj4jIKniNQ9Lhd8/bW5pL5vnylzOGDAALjiCrObkIiI+JyCp4jUHWlpZrLQokVm8hBA/foweDAMGaLF3UVE/EzBU0Rqv6QksxxSYiIUFpqymBi4/HI4/3wIDvZr80RExFDwFJHaybLMQu/z5sHPP5eUd+xoLqf37q0ljkREahgFTxGpXQoLYflyEzi3bTNlNhv06WNmqHfo4N/2iYhIhRQ8RaR2yM6GL76Azz+H5GRT5nTChReaNThjYvzbPhER+VMKniJSsx08CJ99Bl9+afZFB4iMhKFDzaSh+vX92jwRETl+Cp4iUjP98YeZMLR8uVmPE6BFC7P+5nnnQWCgX5snIiKVp+ApIjWHZcFPP8HcubBuXUn5GWeY8Zs9emjBdxGRWkzBU0T8Lz/fLIX0ySewe7cpCwiAc881M9Rbt/Zn60REpIooeIqI/2RkwMKFZtH39HRTFhoKF19sxnBGR/u3fSIiUqUUPEXE9/bsMdtZfv216e0EaNTILPh+4YUmfIqISJ2j4CkivmFZsHGjGb/5ww/mPkDbtmb85l/+Yi6vi4hInaXgKSLVq6gIVq40C75v3lxS3quXmaHeubMmDImInCIUPEWkerhcsGSJmTB08KApCwyEgQPNhKHmzf3ZOhER8QMFTxGpWqmpZnehRYvMbkNgFnm/9FIYMgQiIvzbPhER8RsFTxGpGjt2mN7Nb781+6kDxMaa3s2BA832liIickpT8BSRE2dZsGaNGb/5yy8l5Z07mwlDZ52l8ZsiIuKh4CkilVdYaHo2P/nE9HSCCZh/+YsJnKef7s/WiYhIDaXgKSLHLysLFi82YzhTU01ZcDBcdBFcdhk0aeLf9omISI2m4Ckif+7AAbPg+5IlZrY6QFSU2V3o4oshLMy/7RMRkVrBfrJP8Nprr2Gz2Qgr5wfP6tWrueCCCwgLCyMyMpLhw4ezbdu2k31JEfGVzZvh8cfhjjtML6fLBa1awd13w+uvw1VXKXSKiMhxO6kezz179jB27FhiY2NJL95n+YhNmzbRv39/unXrxpw5c3C5XEyaNIl+/fqxZs0aGjVqdFINF5FqYllmZ6G5c2HDhpLy7t3N+M1u3TRhSERETojNsor3rau8oUOHYrPZiIqK4qOPPiIrK8tz7OqrryYxMZGtW7cSHh4OwM6dO2nXrh1jxozh8ccfP67XyMjIICIigvT0dM/ziEg1yMuDb74xE4b27jVlDgecd55ZEqlVKz82TkREarLjzWsn3OP57rvv8u2337JhwwYefPBBr2OFhYXMnz+fm2++2evFW7ZsyYABA5g3b95xB08RqWbp6TB/PixYAJmZpqxePRg82IzhjIryb/tERKTOOKHgefDgQe6++24ee+wxmpez7d3WrVvJzc0lPj6+zLH4+HiWLFmCy+UiODj4RF5eRKrC7t2md/Obb6CgwJQ1bmx6Ny+80MxWFxERqUInFDxHjhxJ+/btSUhIKPd4SkoKAFHl9JRERUVhWRZpaWnExMSUOZ6Xl0deXp7nfkZGxok0UUTKY1mwfr0Zv/njjyXlp59uxm/26QMBAf5rn4iI1GmVDp4ff/wxn3/+Ob/88gu2P5lgcKzjFR2bMWMGU6dOrWyzRORYiorgu+/MDkNbtpgym83sLDR8OHTsqAlDIiJS7SoVPLOyshg1ahR33nknsbGxHD58GID8/HwADh8+TGBgIA0bNgRKej5LS01NxWazERkZWe5rTJgwgXvuucdzPyMjgxYtWlSmmSJSLCfHrL356adw6JApCwqC88+Hyy+HZs382z4RETmlVCp4Jicnc+DAAZ588kmefPLJMscbNGjA5ZdfzkcffURISAhr164tU2ft2rW0bdu2wvGdTqcTp9NZmWaJyNGSk826m4sXm/AJEBEBQ4aYSUMREf5tn4iInJIqFTybNm1KYmJimfLHHnuMb7/9lkWLFhEdHY3D4WDo0KHMnTuXmTNnUr9+fQCSkpJITExkzJgxVdN6EfG2bZuZMPS//5nL6wDNm5sJQwMGmN5OERERPzmpdTyL/fWvfy2zjuemTZvo1asXPXr0YPz48Z4F5FNTUyu1gLzW8RT5E5YFq1eb8Zu//lpS3qWLGb955pkavykiItWq2tfx/DMdOnRg6dKljBs3jquuugqHw8HAgQOZNWuWdi0SqQoFBbB0qenhTEoyZXY7nHOO6eFs186PjRMRESmrSno8q5N6PEWOkpkJixaZRd/T0kxZcDBcfLFZ8L1xY/+2T0RETjl+7/EUkSq2b5+Znf7VV2Z7S4CGDeGyy2DQILPbkIiISA2m4ClS023aZMZvrlxpxnMCtG5tLqf362f2UxcREakF9BNLpCZyu2HVKhM4N20qKe/Z0+wwFB+vCUMiIlLrKHiK1CQuF3z9tbmkvm+fKXM4zFJIV1wBcXF+bZ6IiMjJUPAUqQnS0sxkoUWLzOQhgLAwuOQSs+h7gwb+bZ+IiEgVUPAU8aekJLMcUmIiFBaasqZNzXaWF1xgZquLiIjUEQqeIr5mWfDbb2b85s8/l5R36GDGb/bubdbjFBERqWMUPEV8pbAQli83gXPbNlNms0GfPmb8ZseOfm2eiIhIdVPwFKlu2dnwxRfw+eeQnGzKnE5zKf3yyyEmxr/tExER8REFT5HqcvAgfPYZfPkl5OaasshIs7vQ4MFQv75fmyciIuJrCp4iVW3LFnM5fflysx4nQIsWZvzmeedBUJB/2yciIuInCp4iVcGy4KefTOBcu7akPD4ehg+HHj204LuIiJzyFDxFTkZ+vlkK6ZNPYPduUxYQYLayHDbMbG0pIiIigIKnyInJyICFC82i7+nppiw0FAYNgssug+ho/7ZPRESkBlLwFKmMPXvMdpZff216O8GEzMsvh4suMuFTREREyqXgKfJnLAs2boS5c+GHH8x9gDZtzPjNvn3NfuoiIiJyTPppKVKRoiJYudJMGNq8uaS8Vy8zfrNLF00YEhERqQQFT5GjuVywZImZMHTwoCkLDIQBA8wOQy1a+LN1IiIitZaCp0ix1FSzu9CiRWa3ITCLvF96qblFRvq1eSIiIrWdgqecGtxuWL8e0tKgQQPo3BnsdnNsxw7Tu/ntt2Y/dYDYWNO7OXCg2d5SRERETpqCp9R9K1bA88+bCUJ5eSZIduxoZqFv2wa//FJSt1MnM37zrLNKgqmIiIhUCQVPqdtWrICxY81l9JgYCA6GffvM5fTFi00AbdgQ/vIX08PZvr2/WywiIlJnKXhK3eV2m57O1FRo1QqSk80+6vn5ZrKQy2Uurb/8sgmlIiIiUq0UPKXuWr8efvvNBNBffzV/AgQFQZMmEBICWVkmkCp4ioiIVDsFT6l7LMsEzWeegZ07TcC02cyuQk2bQlSUGb9ZVAQpKWbCkYiIiFQ7BU+pO/LzYelSs6VlUpLZT91uh3r1IC4OwsO96+fmmolGDRr4pbkiIiKnGgVPqf1SU2HhQjNhKCPDlAUHm7U3o6Jg0yazHmdplgX790OPHmZpJREREal2Cp5Se23dano3ly0rWX+zUSMYOtQslVSvHsTHm1ntW7eay+whIaanc/9+09M5apSWTRIREfERBU+pXdxu+P57+OwzWLeupLxjR7j8cujdGwICSsr79oVZs0rW8TxwwFxe79HDhM6+fX3/HkRERE5RCp5SO+TkmP3TP//chEcwAfOcc+Cyy+D00yt+bN++JpBWtHORiIiI+ISCp9RsBw6YsPnll+YSOZjxmhdfDJdcAtHRx/c8djt07Vp97RQREZE/peApNY9lwYYNZvzmqlXmPkDz5uZy+oAB2j9dRESkFlLwlJqjsNBMFPr0UzMZqFj37iZw9uhh1uMUERGRWknBU/wvI8MshbRgQcli7kFBMHCgmaEeF+ff9omIiEiVUPAU/0lKMr2bS5eaxd/BrLt56aVmDOfRC76LiIhIrabgKb5lWbB6NXzyCaxZU1Letq25nH7OOeDQ11JERKQu0k948Q2XCxITzfqbu3ebMpsN+vQxgbNjR43fFBERqeMUPKV6JSebsZuLF0NWlikLDTU7Cw0ZAk2a+Ld9IiIi4jMKnlI9Nm824ze/+w6KikxZ06ZmsffzzzfhU0RERE4pCp5SdYqKYOVKczl948aS8i5d4IoroFcv7RYkIiJyClPwlJOXnW12Fvr8czh0yJQ5HHDeeaaHs3Vr/7ZPREREagQFTzlxe/easPnVV2byEEBEBAwebLazbNDAv+0TERGRGkXBUyrHsmDtWjN+88cfS7azbNXK9G6ed55Z/F1ERETkKAqecnzy8+F//zOBc8eOkvJevcxySPHxWg5JREREjknBU47t8GFYuNDc0tNNmdNpZqZfdhk0a+bX5omIiEjtoeB5qnK7Yf16szd6gwbQubP3jPPt203v5rffQmGhKYuONnunX3QRhIX5p90iIiJSayl4nopWrIDnnzdLHuXlmR7Mjh1h1CgzG/2zz+C330rqt29vlkPq3VvbWYqIiMgJq9SiimvWrOHSSy8lLi6OkJAQoqKi6NOnD++++26ZuqtXr+aCCy4gLCyMyMhIhg8fzrZt26qs4XKCVqyAsWPNfumRkWZSUHg4LFsGw4fDPfeY0Gm3w7nnwqxZ5qY91EVEROQkVSpJHD58mBYtWnDdddfRrFkzsrOzee+997jpppvYsWMHDz74IACbNm2if//+dOvWjTlz5uByuZg0aRL9+vVjzZo1NGrUqFrejPwJt9v0dKamQtu2ZsH3vXvN2psFBZCTA3v2wO23m0vq0dH+brGIiIjUITbLKl4P58T17t2bvXv3kpSUBMDVV19NYmIiW7duJTw8HICdO3fSrl07xowZw+OPP37cz52RkUFERATp6eme55ITtHYt3Hyz6el0OmHTJsjNNceCg80anHY7vPMOdO3q16aKiIhI7XG8ea1K9i+Mjo7GceQybGFhIfPnz+fKK6/0euGWLVsyYMAA5s2bVxUvKSciLc2M6QwIMOM7c3PNmpunn26WQ2re3CyblJbm75aKiIhIHXRCg/bcbjdut5u0tDQ+/PBDvvjiC55//nkAtm7dSm5uLvHx8WUeFx8fz5IlS3C5XAQHB5f73Hl5eeTl5XnuZ2RknEgTpTwNGpgezfXrzWV3p9NMHCr+LHJzTZl2HBIREZFqcEI9niNHjiQwMJDGjRszZswYnn32Wf7xj38AkJKSAkBUVFSZx0VFRWFZFmnH6FGbMWMGERERnluLFi1OpIlSnvBws696ZmbJTPbi0GlZsH8/dOpkllYSERERqWInFDwnTpzIjz/+yIIFC7jtttsYPXo0s2bN8qpjO8YuNsc6NmHCBNLT0z23Xbt2nUgT5Wjbt8MDD0BsrFmDMzjYXFYvKoKsLNi61fR0jhrlvZ6niIiISBU5oUvtcXFxxMXFAXDJJZcAJjDecsstNGzYECjp+SwtNTUVm81GZGRkhc/tdDpxOp0n0iypyObNMGmS6e3s1QumToU33jDjPA8cML2fPXqY0Nm3r79bKyIiInVUlSzMeNZZZ/HSSy+xbds2evbsSUhICGvXri1Tb+3atbRt27bC8Z1SDdavN0EzNxc6dIApU6BePRg48Ng7F4mIiIhUsSpJGomJidjtdlq3bo3D4WDo0KHMnTuXzMxMT52kpCQSExMZPnx4VbykHI81a0xPZ26umbU+fboJnWBCZteuZpH4rl0VOkVERKTaVarH8+9//zvh4eGcddZZNGnShOTkZD788EM++OAD7rvvPs/C8FOnTqVXr14MGTKE8ePHexaQj46O5t57762WNyJH+eEHmDHD7LPesydMnGiWThIRERHxk0oFzz59+vDmm2/yn//8h8OHDxMWFsYZZ5zBO++8w4033uip16FDB5YuXcq4ceO46qqrcDgcDBw4kFmzZmnXIl9Yvtxsc1lUBH36wH33QWCgv1slIiIip7gq2bmoOmnnokr6+mt45hmzPNJ558GYMWbBeBEREZFqcrx5rUomF0kNsXAhvPii+ftFF2lpJBEREalRFDzrinnzzBJJAEOHwh13wDHWSxURERHxNQXP2s6y4IMP4L33zP0RI+CmmxQ6RUREpMZR8KzNLAvefhs++sjcv+kmuPpq/7ZJREREpAIKnrWVZcGrr8Lnn5v7t98Ol1/u3zaJiIiIHIOCZ23kdsPzz8OSJeb+yJEweLB/2yQiIiLyJxQ8a5vCQnj6afj2WzOO8+67zfaXIiIiIjWcgmdtUlAATzwBK1eatTnHjoVzzvF3q0RERESOi4JnbZGfD48+Cj//DA4HTJgAZ53l71aJiIiIHDcFz9ogNxemT4e1a8HphAcfhG7d/N0qERERkUpR8KzpsrNh8mT4/XcICYEpU6BTJ3+3SkRERKTSFDxrsowMeOgh2LYNwsJg2jRo187frRIRERE5IQqeNVVqqgmdSUkQEQEPPwytWvm7VSIiIiInTMGzJkpOhgcegL17ISoKHnkEmjf3d6tERERETord3w2Qo+zbB+PGmdDZuDE89phCp4iIiNQJ6vGsSXbvNj2dqakQG2t6OqOj/d0qERERkSqhHs+aYvt2GD/ehM64ONPTqdApIiIidYiCZ02weTNMnAjp6dCmDcyYAQ0a+LtVIiIiIlVKwdPf1q83C8JnZUGHDubyeni4v1slIiIiUuU0xtOf1qwxOxLl50N8vFk+KTjY360SERERqRYKnv7yww/mknphIfTsaS61BwX5u1UiIiIi1UbB0x+WL4dZs6CoCPr0gfvug8BAf7dKREREpFopePra11/DM8+AZcF558GYMRAQ4O9WiYiIiFQ7BU9fWrQIXnjB/P2ii2DUKLBrfpeIiIicGhQ8feWTT+D1183fhw6FO+4Am82vTRIRERHxJQXP6mZZMGcOvPuuuX/VVXDzzQqdIiIicspR8KxOlgVvvw0ffWTu33QTXH21f9skIiIi4icKntXFsuDVV+Hzz83922+Hyy/3b5tERERE/EjBszq43fD887Bkibk/ciQMHuzfNomIiIj4mYJnVSsshKefhm+/NeM4774bBg70d6tERERE/E7BsyoVFMATT8DKlWZtzrFj4Zxz/N0qERERkRpBwbOq5OfDo4/Czz+DwwETJsBZZ/m7VSIiIiI1hoJnVXC5YPp0+O03s9/6Qw9Bt27+bpWIiIhIjaLgebKys2HKFNi0CUJCYPJk6NzZ360SERERqXEUPE9GRgZMmgRbt0JYGEybBu3a+btVIiIiIjWSgueJSkuDBx+EpCSIiICHH4ZWrfzdKhEREZEaS8HzRCQnwwMPwN69EBUFjzwCzZv7u1UiIiIiNZrd3w2odfbtg3HjTOhs3Bgee0yhU0REROQ4qMezMnbvNj2dqakQG2t6OqOj/d0qERERkVpBPZ7Ha/t2GD/ehM64ONPTqdApIiIictwUPI/H5s0wcSKkp0ObNjBjBjRo4O9WiYiIiNQqutRemtsN69ebGesNGpj1ODduhKlTITcXOnQwa3bWq+fvloqIiIjUOgqexVasgOefN0EzLw+cTmja1ATO+vUhPt7sSBQc7O+WioiIiNRKCp5gQufYsWb8ZkyM2YHowAFYvhwCA2HYMLMjUVCQv1sqIiIiUmspeLrdpqczNRXatgWbzfx9zx4TQC0LcnLAoX8qERERkZOhyUXr15vL6zExJnQmJ5stMC3LzFrv3Nnsw75+vb9bKiIiIlKrVSp4fvPNN9x222106NCBevXq0axZMy6//HJ+/vnnMnVXr17NBRdcQFhYGJGRkQwfPpxt27ZVWcOrTFqaGdMZEgL5+bBjhwmdjRpB69YQGmqOp6X5u6UiIiIitVqlgueLL77Ijh07+Ne//sXChQt55plnOHjwIL179+abb77x1Nu0aRP9+/cnPz+fOXPm8MYbb7B582b69evHoUOHqvxNnJQGDcxEotxcM4azTRszqei000wPaG6uOa7lk0REREROis2yLOt4Kx88eJDGjRt7lWVlZdG2bVu6dOnCV199BcDVV19NYmIiW7duJTw8HICdO3fSrl07xowZw+OPP37cDczIyCAiIoL09HTPc1UptxtuvBF++cWETput5JhlmcvuPXrAO++AXSMTRERERI52vHmtUknq6NAJEBYWRqdOndi1axcAhYWFzJ8/nyuvvNLrhVu2bMmAAQOYN29eZV6y+tntMHq06dHcuhWysqCoyPy5daspHzVKoVNERETkJJ10mkpPT2f16tV07twZgK1bt5Kbm0t8fHyZuvHx8WzZsgWXy1Xh8+Xl5ZGRkeF1q3Z9+8KsWdC9Oxw+bMZ5Hj5sejpnzTLHRUREROSknPQaQaNGjSI7O5sHHngAgJSUFACioqLK1I2KisKyLNLS0oiJiSn3+WbMmMHUqVNPtlmV17cv9O5dduci9XSKiIiIVImTSlUPPfQQ7733Hk899RQ9e/b0OmYrPVbyKMc6NmHCBNLT0z234kv4PmG3Q9eucO655k+FThEREZEqc8I9nlOnTuXhhx/mkUceYfTo0Z7yhg0bAiU9n6WlpqZis9mIjIys8HmdTidOp/NEmyUiIiIiNdQJdelNnTqVKVOmMGXKFCZOnOh1rE2bNoSEhLB27doyj1u7di1t27YlWPudi4iIiJxyKh08p0+fzpQpU3jwwQeZPHlymeMOh4OhQ4cyd+5cMjMzPeVJSUkkJiYyfPjwk2uxiIiIiNRKlVrH88knn2Ts2LFcfPHF5YbO3r17A2YB+V69etGjRw/Gjx+Py+Vi0qRJpKamsmbNGho1anTcDaz2dTxFRERE5KQcb16rVPDs378/3377bYXHSz/Vzz//zLhx41i5ciUOh4OBAwcya9Ys2rRpc7wvByh4ioiIiNR01RI8/UHBU0RERKRmq5adi0RERERETpSCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ISCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ISCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ISCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ISCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ISCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ISCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ISCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ISCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ISCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ISCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ISCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ISCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ISCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ISCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ESlg2dmZib3338/F110EY0aNcJmszFlypRy665evZoLLriAsLAwIiMjGT58ONu2bTvZNouIiIhILVTp4JmSksIrr7xCXl4eV1xxRYX1Nm3aRP/+/cnPz2fOnDm88cYbbN68mX79+nHo0KGTabOIiIiI1EKOyj6gZcuWpKWlYbPZSE5O5rXXXiu33qRJk3A6ncyfP5/w8HAAevbsSbt27Zg1axaPP/74ybVcRERERGqVSvd42mw2bDbbMesUFhYyf/58rrzySk/oBBNaBwwYwLx58yrfUhERERGp1aplctHWrVvJzc0lPj6+zLH4+Hi2bNmCy+Uq97F5eXlkZGR43URERESk9quW4JmSkgJAVFRUmWNRUVFYlkVaWlq5j50xYwYRERGeW4sWLaqjiSIiIiLiY9W6nNKxLslXdGzChAmkp6d7brt27aqu5omIiIiID1V6ctHxaNiwIVDS81laamoqNpuNyMjIch/rdDpxOp3V0SwRERER8aNq6fFs06YNISEhrF27tsyxtWvX0rZtW4KDg6vjpUVERESkhqqW4OlwOBg6dChz584lMzPTU56UlERiYiLDhw+vjpcVERERkRrshC61L1q0iOzsbE+o3LBhAx999BEAl1xyCaGhoUydOpVevXoxZMgQxo8fj8vlYtKkSURHR3PvvfdW3TsQERERkVrBZlmWVdkHtWrVip07d5Z7bPv27bRq1QqAn3/+mXHjxrFy5UocDgcDBw5k1qxZtGnT5rhfKyMjg4iICNLT073WBBURERGRmuF489oJBU9fUvAUERERqdmON69V63JKIiIiIiLFFDxFRERExCcUPEVERETEJxQ8RURERMQnFDxFRERExCcUPEVERETEJxQ8RURERMQnFDxFRERExCcUPEVERETEJxQ8RURERMQnFDxFRERExCcUPEVERETEJxQ8RURERMQnFDxFRERExCcUPEVERETEJxQ8RURERMQnFDxFRERExCcUPEVERETEJxQ8RURERMQnFDxFRERExCcUPEVERETEJxQ8RURERMQnFDxFRERExCcUPEVERETEJxQ8RURERMQnFDxFRERExCcUPEVERETEJxQ8RURERMQnFDxFRERExCcUPEVERETEJxQ8RURERMQnFDxFRERExCcUPEVERETEJxQ8RURERMQnFDxFRERExCcUPEVERETEJxQ8RURERMQnFDxFRERExCcUPEVERETEJxQ8RURERMQnFDxFRERExCcUPEVERETEJxQ8RURERMQnFDxFRERExCcUPEVERETEJxz+boCISG2RnJNMuiu9wuMRwRFEh0b7sEUiIiVqwzmqWoNnVlYWDz74IHPmzCE1NZUOHTowfvx4rr322up82UqrDR+UnBh9tlJVCooK+Of8f7ItbVuFdVo3aM37V75PYECgD1smIlJ7zlHVGjyHDx/Ojz/+yGOPPcbpp5/O7Nmzue6663C73Vx//fXV+dLHraCogFs//vMPau71deOHyaakZPamVhzEYqMi6BBXN4LYqfbZSvVy2B00cDQjOesXYuu1xFbqmAXszd5Jz0bNcNh1IUlEfK+2nKOq7dUXLlzIkiVLPGETYMCAAezcuZP77ruPa665hoCAgOp6+eOWluLg12XNOBDxC0G5Lcsczw/ZSea6ZqQNctC4sR8aWIX27i/g3Jn/JCuw4iAWVtCaNQ++T2zT2h/ETqXPVo6fZVm4LTcAAfaSc1BOQQ5uy02II8RTnleYR1Z+FoEBgeRlhPPb27eQfNoyUtw52AvrY3cHAXaKHOkUBdbnp1UXs+nMZDrGNfI874GsAxS4C4gOjSbYEQyAq9BFck4yQQFBNK5X8uU7mH2Q/KJ8GoY0JCQwxKuuw+6gaVhTT91D2YfIK8qjQXAD6gXVAyC/KJ+D2QcJsAUQUz/GUzc5JxlXocurbkFRAQeyD2C32YmtH+upm5qbSk5BDhHOCOo76wNQ6C5kf9Z+bNhoFt6sTN1wZzjhznAAitxF7MvaB0Dz8Oaeuoddh8nKz6J+UH0igiM8n8WezD0ANKvfDJvN/KhMd6WTmZ9JWFAYkcGRZerGhMV4PqOMvAwy8jIIDQwlKiTK83p7MvZgYdE0rKnnB21mXibpeell6u7N3IvbctOkXhPPL6HZ+dmkudIIdgR7XRXZl7mPIquIxvUaExQQ5FXXGeCkUb2Sz35/1n4K3YU0Cm2E0+EEILcgl5TclAo/+/K+J4H2QJqENSlTNyokitDAUMB8Vw/lHDquz76i70lKTgq5hblEBkcSFhR2XJ99ed+Toz/7tNw0sguyvb4nbsvN3sy9FX72FX1PYuvHYrfZverWC6xHg5AGntfbnbEb4Lg++/K+J1n5WRx2HSbEEULD0IbH9dlX9D0p/dnnFOSQmpta5ntSVeeIw2kOzzkqtbAQe1EoNst8n4sc6bgdEfy27BaS+9loVPLyPldtk4vmzZtHWFgYI0aM8Cq/9dZb2bt3L99//311vXSlZGbaCP3jFhxFEQQEFuK01/PcAgILcRRFEPrHLWRm2v78yWq4nCwHVkYz3EHpOK3IMjd3UDpWRjNysupGj83xfLYhf9zs9dnmFeaRW5DrCSZgTtKHXYfJzMv0ev7knGT2Ze4jvyjfU5adn82Owzs8J9RiOw7vYOOhjWTlZ5W0Ly+TNfvX8Hvy7151NxzawMpdK0nJSfGUpbvSWbpjKd/v9v5/8/Pen1m8ZTF7MvZ41f1006cs3rLYq+53Sd/x33X/5Y+UP7zq/mfNf3jvt/e86iZuT+Sln15izf41Xu19ZtUzPPv9s151v9jyBY8tf4zlScs9ZbkFuUxdOpUpS6d4/Vsu/GMhE76awKI/FnnKCt2FjFk8hrsX342r0OUp//z3z0mYn8B/1/3X6/Vu/+x2bv30Vq8hFJ/9/hnXfXwdL/30klfdG+bewLAPhrEvc5+nbP7m+VzxwRXMWjHLq+4/5/+T6z6+jl0Zuzxl/9v5P/766V95etXTZGRAflJ3wg/3I6/+JlyRv2IPzsERaOEOPUhoTju2Rv+bZ394yut5H/7fwyQsSGBT8iZP2doDa0lYkMBjyx/zqvvEd0+QsCCB3w785inbnLKZhAUJTPt2mlfdZ79/loQFCfy09ydP2Y7DO0hYkMCD3zzoVfeln14iYUGC12e0L2sfCQsSuH/J/V51X1/9OgkLEkjckegpS8lJIWFBAv9a/C+vuu/99h4JCxK8vmtZ+VkkLEggYUGCV9056+eQsCCBz37/zFOWV5TnqVv6/9Gnv39KwoIEPtrwkddzFNct/f9o8ZbFJCxIYPba2V5171p8FwkLEkjNTfWUfbP9GxIWJPDGL2941b1vyX0kLEjwhCaAZUnLSFiQwMs/vexV94FvHiBhQQI7D+/0lP2490cSFiSU+b8x7dtpJCxI4I/Ukv9za/avIWFBQpnv32PLHyNhQQLrDq7zlG08tJGEBQk8uuxRr7pPrXyKhAUJrN632lO2LW0bCQsSmLJ0ilfd5394noQFCazavcpTtjtjNwkLEhj/9Xivuq+ufpWEBQn8b+f/PGUHsg6QsCCBe7+816vuf9b8h4QFCSzZtsRTlu5KJ2FBAncuutOr7n/X/ZeEBQks2LzAU5ZTkOP5PIusIk/5xxs/JmFBAnM3zvWUFVlFnrq5Bbme8vmb55OwoOw5YvTC0SQsSCAjL8NT9uXWL0lYkMDbv77tVfeeL+8hYUECB7MPesq+3fEtCQsSeG31a151x389noQFCV7n25W7V5KwIIF///Bvr7qTl04mYUGC1xW31ftWk7AggadWVc85YsaKaZ5zlDv0IO7Q/QQG4jlHhR/uR35SdzIy8KtqSxjr1q2jY8eOOBzeLxEfH+853rdv3+p6+UoJSe9OVGY/0hp+QVBeOFkh6ym05eJwhxJ1aCjOtO78tO9HHlz9DB2iOzB5QMlJfco3D7M7YxcJPe+kc6MuAGxK3shra14mLjKO+8+5x1P336teISljJ9d0vJ5OjToDkJSRxOx1bxMdGs3o3v/01P3gt7nsOLyDQa0H0yG6IwCHcg4x7/ePCHeG89ceN3jqfrE5kd0Zuzgrtjftok4HTC/Aku1fEOIIZniXoQDYbDZCU3uTFbMAy5GFs9D8Bm3ZisgNSsKGjYa7bvH85vl90mq2pW2nU3Rn2jfsAJjfrOb9/hGWZXHbmTd52vDtthWsO7iO+Mbd6BV7FmAC28urX8DCYkzf0Z6ehCV/LGXFru84M+Yszj/tQtMGy2Laskm4cfPQeRMJc5rfzBdv/oqFfyzkrNjeXNXxas/r3fPVXeQX5vHYhY8SXc/8Rjp/02LeXzebs5v14e/dE8jLA2dadyyHiyznH0Rmn0WAFYqFRV7IdqwiB+mNFpGX1xPXkbzzj4UJJOcc4qmL/4/To9sBsGzHCv5v5ZPEN+7G5HOne9rwwFeT2J2xixkXzCC+qfnsV+/9jceWP0r7hh15dMBMT93/++5Ztqb9weT+kzmr+Znme3JoC1OWTqJVZGuevOAZT903V7/DhkPruP8v4zjvtHMA2Jm2hyeWP0lMWCxnRJ/tqTtvw+f8sv9n/tX7bk9vxMGsFF756TUahETRv/nFnrrfbFvGqt3fUc8RTruG5r2l52YxZ91HhAbW48rTS75TP+3+laU7v6ZhcGO6Ne1mPvuCfL7c8hUBdgd/P+MuT93fD21j2Y7vaBYWxzlxpqywyM0Pu00gysl147Cb33F3pe3nt/3raB3ZzvN4y4LNyVsAyM4pAtORQEp2JkmHd5OcneZV90BWMm53Edm5RTiPlGe7CsjIzSI7z+VVN7+giPyCQnJdFq4jnfiFhXbcRVBUkoexLHC7bbiLINfl9nwfCgvsYAVguU37bdhokXILqc3eB5uF3W7DsmcQaIXSKG0IB+p/QZAtlNKc9nqEBtSnsCDA87xFhQ5CA+rjtHvXDbKFlqlbmB9AaEB9ggPqedUNtIUQGlCfokJHmbohAWHedTF1raJAT938PHv5dY88L0VBnrp5eTbT3gCnV12HLZjQgPrY3E5PXVce5vFHCbCcptxd8ryuwvLr2q0gQgPqYy/1vJZVuq6tTN0AK9hTF0xdB/nYSvWv2KzAI+UhXnVD7GEUBlhede0V1DWfp8urLpb5PINsoV51iz9PmxVw3HXtpX4sW27zeQbZvesWf0Z2Sq5Mud32I9+pel51HZStax2pG3xU3QDLfJ52q2zdkADvNgRQTl3LfE8C7AFede1HPvsAW1Cp57V5Pk+XCxxH/jltblPXYXOWet6Szz4vz0aA27tuoD3Yq25IQBhuy/w/dhV/VdzmexJoO6quPQwCAsjLs3nOEVZRYLl1g+31CA0oID/fXvK9LDxS1x5CacH2sLL/7wvK/39fVecIp7ue5xy1udEyihy5BBRBQYA5R7VIuYUi/N+JVm3BMyUlhdatW5cpj4qK8hwvT15eHnl5eZ77GT6I5jZsND1wCxkNllEYkEFevhu3Ix93bhSZy28hc5+Nx2flsee0dHZHZjN5QMlj3513kEzHXjbPLiDiyC9ih0Oz2Ry7lbgwG/efU1L33+9vIcW+ke/eGEKD7CPvLySdTc2+p2lIc0b3Lqk7461f2Get4ZMD3YnONMEzy3mYDS0WEuVs5BU8J726gh2Fq2h1sDGNM0zwzA1MZ23LtwkLrM/wmUM9dZPzd5NXUER+wF5y0htjw4bbXkBhzA7s2U0JSe/uqfvQKyv4NfsLmqfcSGyaCZ4FAXn8ctoH2O1wa88bPSH1yXfW8/3hz4lNC6Z5igmeRTaLn9t8DcBdfRIIPHLie3XOLr5NXkWTw41omVzyKfzQdg0AY84qIOzIOef9T1JZvO8PvsloxQclv5Dyc+t9FNldjDurkOgjP48XLM7nq51p/JKZxZIDkJ0NO7bbKDjYiqLYPeTbMwkpCqUwIAN3gRN3eiyHD8Po0VDvyHP82tJGXiDsOcPN6UeumixdaufndTb+yIF1z5W0YX3zYFxBIexqD/FHroCu+i6IX3+IZIsrjD9eKKn7R9Nocp1Z7GoVxFlHrkCt/j6UDStasa0glhGlOlV2NIojx1nE7pj6cNqR1/qlPn8s68aOwoaMeL2k7t4GHcl2BrG3YSNoa8o2/RbG9qX92VUUxoi3SuoeDO9GjrM++8NbgPk42boxjN1fX4HdHcSId0vqptY7ixxnI5KDO0CXI+3aEsr+L2/BZgUw4v2Suhkh55AbFEd6QDvoZsp273SSuvBuAK79wO451eUEDcAVdDo5Rc3hzCPvYXcAGZ9PBmzc/IHT8+M8z3EBeY548i9qCEey9qFDkD13FmDjjg/CPXUL7RdREHA21vmh8Jcj7cqAjP8+D5aNuz6I9LTBbbsQt+08Avs74Nwjr5UHaf95DSwb930QUOrUfD5wPs6/AEdCdVhWdxwbriM/7gsy00NxR2whaPsgDn57M+7CW6AecGWpz/O1x8jLg6nvUEp3YLb5t72wVN23ppKTAY951e1s6rYDSn6PYOe7E8k5CE+9AyX9J22A2bhbAENK1f3vveTsgn+/AyX9Ms2B2YQ1Bi4vqZv00Why/hjNq8CrntLGwGwc4d7vLWneHeSsu4O3gZJ+pAhgNk4nUPK7IrsX3ELOT7cwB5jjKQ027w1wlq67+FpyvruWecA8T6mtpO5lpep+dQU5X1/BQmAhpb0FQNCgkpI9Sy8hZ+ElfA187VXX/AcM6l/qeZedT86881kOLPeqa04CQb2BI1dh9648h5z3z+EHwPsan+nVDOoOHLlSvu+nXuS8M5tfgRHPlK77MACBHYEjIx/2rYkn553ZbAJGPF+67iRT9zQ838v969qT885stgMjvDr+x5m6scCRH8n7NrYi553Z5AAjXi1d925TtyFw+pG6f8SU1PXqKE4AEgj8J9DpSN1tUeS8Yz6jEf8pXfc24DYct+I5n+zfVc9T9zqv7/sNwA0EXIfnfLJvj8NT9xavulcBV2EfBvQ0JYcOQdbb5mT2D6+6Q4Gh2C/Bcz7JyICUN80J9S6vuhcBF2E/H8/5JC8P9r5qTupjveqeB5yH/S94zicA21/6PwAmetXtbW5nAqVyRFWdI4qv3odldScqqx+7nV8QikW+4yCNMgYRltWdimd4+E61XlMtDiWVOTZjxgymTp1aXU2qUPEHdSj8C+yHOkD4VoJ2DyTgUHeKgPpZ3emS9BzdI71/42+XPIbcwlxC8uM8ZfXy2nH63il0aOP9G1CrjBuIdKVTL6+ktyc4vxmtDo6mVax3b0bz3ItxZPWgnqutpyyoMIrY1GtpEuX9G1DjgrPITW9MSH7JOMYAdyiNMi4kol6wV13H4fYE7biUwrglWEEZ2PIjIDAbe344QTsvxVbqR25DqwPRGYWE5LfylNndTpocHkqgw/vza0p3YlNDqJ/bpaSu5aBF8l8Bu9dg5ha2s2l1sJFXewHa7B8L2DxjlgBaO/px+t7TCCr0HpBy+l7zHYkMLhnX0y7wPLokdSXA7d2DU++rN8nr+yCFp/0Pq6gx+Y6D1N97MYVLHsadHwRnlNTtkvQCNmy0Di/5Lb5jyLn02nIuR+u825xYTo8s9R5CetJ9+ztl6rbbP9HUjSgpax7cni67nitTt9Uhc4ny9PCSssbOFrTfO71M3di0awBoW6rTqEFQY9ocuLdM3cYZJrW0LvVVCwuMIC75b2XqRmX3JSq7L3GlvsLBASHEpl1Vpm54blfCc7sSW+q/hsPuIDrz/DJ1Q/NPIzT/NBqVGkJss9mIzDmzTF1nYWOchY2JPGooeL28tmXqOtz1cbjrU++owUNBhWUnytmtIOxWEIG2o8uPb1yzDRvBv99CQewy3GG7sBWGEvz7LV7/d0RE/KW413NP42W4QncR4Da9nTXlHGWzLMuqjifu06cPRUVF/PDDD17l69evp0uXLrz88sv8/e9/L/O48no8W7RoQXp6OuHh4WXqn6ytW2HECIiMhMJGq/m15e1YFGEjgK7bX8NxqAeHD8Ps2dC6NdjtEFRytcDrcsLRTqZuXp7p2i+PzYbpTahk3a1b4aqrIDzCYmfHu0mO+IJQV1tygrcQnT6IlhufJjPDxocfQps2kJ8Pbnf5zwsQXCrTVmVdp9O0G6CgAIqKTqzutm1w/fUln+3a027HshVhswKI3/EaAQe9P9uKnrew0NwqEhRkPr+qrhsYCMXz7ypTt6jI/FtUxOEwt8rWdbvNZ1fVdS3LfIerom5AgPm3qOq6djvs2lVyrggOsdgQdzcHGsyjSdowOiU9TU62jcOH4f33oWPHksfWpnME+O//vT/OEceq+2f/53SOKFv3VD9H+CIbHKvu9u1www3mHBVaz2Jdc3OOanp4GJ13lZyjin/GV7WMjAwiIiL+NK9VW49n165def/99yksLPQa57l27VoAunTpUu7jnE4nTqez3GPVLTzH9HrujzQfVKSrOzl288E7nd4nxmLllVWkMnUr809Qmbo2GwTYbcSl3kJa/WW4nOa3objUWwiwe/82VPrL/meqq25gYMl//MrWdTrNZ2e3Q6TL+7ONyP3zz7ZY6ZPan6kJdQMCSn7AVGVdu/34v8OVqWuz1a66cOT/UMqtuAOyiUu5lQC7zfNdO/r7XdvOETXh/72vzhHHUhP+L+scYdSE//eVPUf4o27p/zc2Ss5RLZJvrTG9nVCNs9qHDRtGVlYWH3/8sVf5f/7zH2JjYzn77LMreKR/5OZCTraN6F230iD1IqJ33UpOto3c3D9/bG2TmwsBh7oTntaPfHsa4Wn9CDjUvU6+Vzi1Plupfrm5ZvywI7kbbTe8hiO5G9nZ6PskIjVCTT9HVVuP5+DBg7nwwgtJSEggIyODtm3b8v7777N48WLefffdGrGGJ0B4ODRsCCkpR7rRD3ej6Z7XKAIOH6nTsKGpV9t5v1cb9f64lZzCbOptv5X0DPPbUF15r3BqfbZS/cp8n8qh75OI+EttOUdV2xhPMFtmPvDAA15bZk6YMKFSW2Ye75iBk3HoEMdc1yo8HL8utlqVTqX3Cqfe+5Xqpe+TiNRk/jxHHW9eq9bgWRV8ETxFRERE5MQdb16rtjGeIiIiIiKlKXiKiIiIiE8oeIqIiIiITyh4ioiIiIhPKHiKiIiIiE8oeIqIiIiITyh4ioiIiIhPKHiKiIiIiE8oeIqIiIiITyh4ioiIiIhPKHiKiIiIiE8oeIqIiIiITyh4ioiIiIhPKHiKiIiIiE8oeIqIiIiITyh4ioiIiIhPKHiKiIiIiE8oeIqIiIiITyh4ioiIiIhPKHiKiIiIiE8oeIqIiIiITzj83YA/Y1kWABkZGX5uiYiIiIiUpzinFee2itT44JmZmQlAixYt/NwSERERETmWzMxMIiIiKjxus/4smvqZ2+1m79691K9fH5vNVu2vl5GRQYsWLdi1axfh4eHV/nr+dCq9Vzj13q9UL32fRKQm8/U5yrIsMjMziY2NxW6veCRnje/xtNvtNG/e3OevGx4efsr8MDmV3iuceu9Xqpe+TyJSk/nyHHWsns5imlwkIiIiIj6h4CkiIiIiPqHgeRSn08nkyZNxOp3+bkq1O5XeK5x671eql75PIlKT1dRzVI2fXCQiIiIidYN6PEVERETEJxQ8RURERMQnFDxFRERExCcUPEVERETEJ0654JmZmcn999/PRRddRKNGjbDZbEyZMqXcuqtXr+aCCy4gLCyMyMhIhg8fzrZt23zb4JOwZs0aLr30UuLi4ggJCSEqKoo+ffrw7rvvlqlb29/r0qVLsdls5d5WrVrlVbe2v1epetV1Xnjuuefo0KEDTqeT0047jalTp1JQUFCN70RE6ppvvvmG2267jQ4dOlCvXj2aNWvG5Zdfzs8//1ymbm04P51ywTMlJYVXXnmFvLw8rrjiigrrbdq0if79+5Ofn8+cOXN444032Lx5M/369ePQoUO+a/BJOHz4MC1atODRRx9l4cKFvP3227Rq1YqbbrqJhx9+2FOvLrzXYo8++igrV670unXp0sVzvC69V6k61XFeeOSRR/jXv/7F8OHD+eKLLxg5ciSPPvooo0aNquZ3IyJ1yYsvvsiOHTv417/+xcKFC3nmmWc4ePAgvXv35ptvvvHUqzXnJ+sU43a7LbfbbVmWZR06dMgCrMmTJ5epN2LECCs6OtpKT0/3lO3YscMKDAy07r//fl81t1qcffbZVosWLTz368J7TUxMtADrww8/PGa9uvBepepV9XkhOTnZCg4Otv7+9797Pf6RRx6xbDabtX79+up5IyJS5xw4cKBMWWZmptWkSRPr/PPP95TVlvPTKdfjWXz59VgKCwuZP38+V155pdf+pi1btmTAgAHMmzevuptZraKjo3E4HEDdf6+lnUrvVSqnqs8LixcvxuVyceutt3o9x6233oplWXzyySdV2n4RqbsaN25cpiwsLIxOnTqxa9cuoHadn0654Hk8tm7dSm5uLvHx8WWOxcfHs2XLFlwulx9admLcbjeFhYUcOnSIF154gS+++IJx48YBde+9jho1CofDQXh4OIMGDWL58uWeY3XtvYpvVeb7s27dOgC6du3qVS8mJobo6GjPcRGRE5Gens7q1avp3LkzULvOTwqe5UhJSQEgKiqqzLGoqCgsyyItLc3XzTphI0eOJDAwkMaNGzNmzBieffZZ/vGPfwB1571GRETwr3/9i5dffpnExESeeeYZdu3aRf/+/fniiy+AuvNexT8q8/1JSUnB6XRSr169cusWP5eIyIkYNWoU2dnZPPDAA0DtOj85qvXZa7ljXXr7s8tyNcnEiRO5/fbbOXjwIJ9//jmjR48mOzubsWPHeurU9vfavXt3unfv7rnfr18/hg0bRteuXbn//vsZNGiQ51htf6/iX8f7/dH3TESqw0MPPcR7773Hc889R8+ePb2O1Ybzk4JnORo2bAhQbupPTU3FZrMRGRnp41aduLi4OOLi4gC45JJLAJgwYQK33HJLnXuvpUVGRjJkyBBeeuklcnNz6/R7lepXme9Pw4YNcblc5OTkEBoaWqbu0T8sRESOx9SpU3n44Yd55JFHGD16tKe8Np2fdKm9HG3atCEkJIS1a9eWObZ27Vratm1LcHCwH1pWNc466ywKCwvZtm1bnX+vlmUB5je4uv5epXpV5vtTPHbq6Lr79+8nOTnZa4kvEZHjMXXqVKZMmcKUKVOYOHGi17HadH5S8CyHw+Fg6NChzJ07l8zMTE95UlISiYmJDB8+3I+tO3mJiYnY7XZat25dp99rWloa8+fPp1u3bgQHB9fp9yrVrzLfn4svvpjg4GDeeustr+d46623sNlsx1wrVETkaNOnT2fKlCk8+OCDTJ48uczxWnV+qtbFmmqohQsXWh9++KH1xhtvWIA1YsQI68MPP7Q+/PBDKzs727Isy9q4caMVFhZmnXvuudbChQutuXPnWl26dLFiY2OtgwcP+vkdHJ877rjDuvfee60PPvjAWrp0qfXRRx9Z11xzjQVY9913n6deXXiv1113nTVu3Djrww8/tBITE61XXnnFat++veVwOKwlS5Z46tWF9yrVo6rPCw8//LBls9msiRMnWkuXLrWeeOIJy+l0WnfccYc/3p6I1FKzZs2yAOviiy+2Vq5cWeZWrLacn07J4NmyZUsLKPe2fft2T72ffvrJOv/8863Q0FArPDzcuuKKK6wtW7b4r+GV9MYbb1j9+vWzoqOjLYfDYUVGRlrnnXee9c4775SpW9vf64wZM6xu3bpZERERVkBAgNWoUSNr2LBh1g8//FCmbm1/r1I9quO88Mwzz1inn366FRQUZMXFxVmTJ0+28vPzffSORKQuOO+88yo8Nx3df1gbzk82yzoyCE5EREREpBppjKeIiIiI+ISCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ISCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ISCp4iIiIj4hIKniIiIiPiEgqeIiIiI+ISCp4iIiIj4xP8DEzWQQ2SkM5YAAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 800x500 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# top1 accuracy\n",
+    "CompoundT5 = [0, 0, 0, 0, 0]\n",
+    "ReactionT5 = [20.8, 30.4, 34.8, 46.1, 54.7]\n",
+    "T5Chem = [0.1, 0.0, 0.1, 0.3, 0.3]\n",
+    "\n",
+    "\n",
+    "# plot\n",
+    "import matplotlib.pyplot as plt\n",
+    "fig, ax = plt.subplots(1, figsize=(8, 5))\n",
+    "\n",
+    "\n",
+    "ax.plot([10,30,50,100,200], ReactionT5, \"o-\", label='ReactionT5', color='red', alpha=0.7)\n",
+    "ax.plot([10,30,50,100,200], CompoundT5, \"s--\", label='CompoundT5', color='blue', alpha=0.7)\n",
+    "ax.plot([10,30,50,100,200], T5Chem, \"v:\", label='T5Chem', color='green', alpha=0.7)\n",
+    "\n",
+    "\n",
+    "plt.ylim(-5, 60)\n",
+    "ax.set_xticks([10,30,50,100,200])\n",
+    "ax.set_xticklabels([10,30,50,100,200], fontsize=12)\n",
+    "# ax.set_yticks([10,20,30,40,50,60])\n",
+    "ax.set_yticklabels([int(i) for i in ax.get_yticks()], fontsize=12)\n",
+    "# plt.tight_layout()\n",
+    "ax.legend(loc=\"best\", fontsize=12)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "d0f29837",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<matplotlib.legend.Legend at 0x7f7834b445d0>"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqkAAAGzCAYAAAAMg46nAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAABrAklEQVR4nO3dd3gU1eLG8e8mm142HUIJXXokdNEgQaUJXgV7Q+yAXUSx0BHFcu0oYvnZ8IKKV7mgchUEpSiClyKgdKSlkoQ0Uub3x5hNlk2QQDa7Sd7P88yT7Jmzs2fCsnlz5sw5FsMwDEREREREPIiXuxsgIiIiInIihVQRERER8TgKqSIiIiLicRRSRURERMTjKKSKiIiIiMdRSBURERERj6OQKiIiIiIeRyFVRERERDyOQqqIiIiIeJwqh9Ts7GzGjx/PgAEDiI6OxmKxMHny5Arrrl+/ngsvvJDg4GDCwsIYPnw4u3btqrDuyy+/TLt27fDz86NFixZMmTKFwsLCqjZPREREROqAKofUtLQ05syZQ0FBAZdeemml9bZt20a/fv04fvw48+fP5+233+b3338nMTGRlJQUh7ozZszg3nvvZfjw4Xz99deMGTOGJ598krFjx1b5hERERESk9rMYhmFU5Qml1S0WC6mpqURHRzNp0iSn3tQrr7ySZcuWsXPnTkJDQwHYu3cvbdq04f777+fpp58GzNDbpEkTbrzxRt544w3785988kkef/xxNm/eTIcOHc7kHEVERESklqlyT6rFYsFisZy0TlFREYsWLWLEiBH2gArQrFkzkpKSWLhwob3sq6++Ij8/n1GjRjkcY9SoURiGweeff17VJoqIiIhILWd1xUF37txJXl4e8fHxTvvi4+NZunQp+fn5+Pv7s3nzZgA6d+7sUC82NpaoqCj7/ooUFBRQUFBgf1xSUkJ6ejqRkZF/G6RFREREpOYZhkF2djaNGjXCy6vy/lKXhNS0tDQAIiIinPZFRERgGAYZGRnExsaSlpaGn58fQUFBFdYtPVZFZs6cyZQpU6qv4SIiIiJSI/bv30+TJk0q3e+SkFrqZL2Z5fedar0TTZgwgQceeMD+ODMzk7i4OPbv3+8wzEBEREREPENWVhZNmzYlJCTkpPVcElIjIyMBKuwFTU9Px2KxEBYWZq+bn59Pbm4ugYGBTnW7detW6ev4+fnh5+fnVB4aGqqQKiIiIuLB/m5opksm82/VqhUBAQFs2rTJad+mTZto3bo1/v7+QNlY1BPrHj58mNTUVDp16uSKJoqIiIiIB3NJSLVarQwbNozPPvuM7Oxse/m+fftYtmwZw4cPt5cNGjQIf39/3n33XYdjvPvuu1gslpPOxSoiIiIiddNpXe5fsmQJOTk59gD622+/8cknnwAwZMgQAgMDmTJlCj169GDo0KE88sgj5OfnM3HiRKKionjwwQftx4qIiODxxx/niSeeICIiggEDBvDzzz8zefJkbr31Vs2RKiIiIlIPVXkyf4DmzZuzd+/eCvft3r2b5s2bA/DLL7/w8MMPs3r1aqxWK/379+fZZ5+lVatWTs976aWXePXVV9mzZw8NGzZk1KhRPPbYY/j4+Jxyu7KysrDZbGRmZmpMqoiIiIgHOtW8dloh1VMppIqIiIh4tlPNay6dgkpERESqV3FxMYWFhe5uhogTHx8fvL29q+14CqkiIiK1gGEYHD58mKNHj7q7KSKVCgsLo2HDhtWy8qdCqoiISC1QGlBjYmIIDAzU8t/iUQzDIDc3l+TkZMBc3v5MKaSKiIh4uOLiYntALV0wR8TTBAQEAJCcnExMTMwZX/p3yTypIiIiUn1Kx6CeuDKjiKcpfY9Wx7hphVQREZFaQpf4xdNV53tUIVVEREREPI5CqoiIiIh4HIVUERGR+qqkBDZtghUrzK8lJW5pxrvvvovFYrFvVquV2NhYrr76av744w+3tKm8J598ks8//9ypfPny5VgsFpYvX+6S1508ebLDz6WyrV+/fiet7+/v75L2uZru7hcREamPVq2CV16BrVuhoAD8/KB9e7jrLujTxy1Neuedd2jXrh35+fn8+OOPzJgxg2XLlrFt2zbCw8Pd0iYwQ+rll1/OpZde6lDetWtXVq9eTYcOHVzyurfeeiuDBg2yPz506BDDhw/n7rvv5tprr7WXn7hq01dffYXNZrM/9vKqnX2SCqkiIiL1zapVMG4cpKdDbCwEBEBeHmzYYJY/+6xbgmqnTp3o3r07AP369aO4uJhJkybx+eefM2rUqBpvz98JDQ2ld+/eLjt+kyZNaNKkif3xnj17AIiLizvp63br1o2oqCiXtaum1M5oLSIiImAYkJ9ftS03F158EdLSoGVLCAwEi8X82qKFWf7SS2a9qh7bMKr19EoD65EjR+xl69at45JLLiEiIgJ/f38SEhKYP3++w/NSUlIYM2YMHTp0IDg4mJiYGPr378/KlSudXqOgoICpU6fSvn17/P39iYyMJCkpiVWrVgHm3eo5OTn83//9n9Pl9cou93/xxRecc845BAYGEhISwkUXXcTq1asd6pRemt+yZQvXXHMNNpuNBg0acPPNN5OZmXmmP7o6QT2pIiIitVVBAVxxRdWek5UFv/4KVitUtMRqUREsWQKDB8MJl5H/1oIFUI3jH3fv3g3AWWedBcCyZcsYNGgQvXr14vXXX8dms/Hxxx9z1VVXkZuby0033QRAeno6AJMmTaJhw4YcO3aMhQsX0q9fP7799lt7yCwqKmLw4MGsXLmS++67j/79+1NUVMSaNWvYt28fffr0YfXq1fTv35+kpCSeeOIJwPnyenkfffQR1113HQMGDGDevHkUFBQwa9Ys+2ufd955DvVHjBjBVVddxS233MKmTZuYMGECAG+//fZp/9w6d+5McnIyUVFRDBw4kOnTpxMXF3fax3MXhVQREZH6pLDQvEGqstWAvL3h+HGzXg0rLi6mqKjIPiZ1+vTp9O3bl0suuQSAMWPG0LFjR7777jusVjPCDBw4kNTUVB599FFuvPFGvLy8aNu2La+99prDcQcOHMiePXt46aWX7CF13rx5LFu2jDfffJNbb73VXn/YsGH273v37o2XlxfR0dF/e2m/pKSEhx56iM6dO7NkyRL7WNAhQ4bQqlUrHn74YX788UeH59xyyy089NBDAFx44YXs2LGDt99+m7feeqvKc462atWKGTNmkJCQgL+/Pz/99BOzZs3im2++4ZdffqFx48ZVOp67KaSKiIjUVn5+Zu9lVWzeDLfeCjYbBAc77z92DDIzzUv+nTpVvT1n4MQQ2L59e/79739jtVrZsWMH27Zt49lnnwXMXtBSQ4YMYdGiRWzfvp327dsD8PrrrzNnzhx+++03CgoK7HXbtWtn/37JkiX4+/tz8803n1G7S23fvp2DBw9y3333OdysFBwczIgRI3jjjTfIzc11WDmsNICXio+PJz8/n+TkZBo0aFCl17/hhhscHiclJZGUlMQ555zDrFmzePHFF0/jrNxHY1JFRERqK4vFvLxela1rV+jQAZKTzed7eZVtFotZ3rGjWa+qxz7D1Ybee+89fv75Z7777jvuuOMOtm7dyjXXXAOUjUsdN24cPj4+DtuYMWMASE1NBeD5559n9OjR9OrVi08//ZQ1a9bw888/M2jQIPLy8uyvl5KSQqNGjart7ve0tDQAYmNjnfY1atSIkpISMjIyHMojIyMdHvv9FfTLt/NM9OzZk7POOos1a9ZUy/FqknpSRURE6hMvL3OaqXHjYOdOaNiw7O7+w4chPBzGjjXr1bD27dvbb5ZKSkqiuLiYuXPn8sknn9C5c2cAJkyYwPDhwyt8ftu2bQH44IMP6NevH7Nnz3bYn52d7fA4OjqaH374gZKSkmoJqqWB89ChQ077Dh48iJeXl1um0jIMo1ZOQ1X7WiwiIiJnpk8fc5qphATz5qk9e8yvXbu6bfqpisyaNYvw8HAmTpxImzZtaNOmDf/73//o3r17hVtISAhg3pHvd8LQg40bNzrdYT948GDy8/N59913T9oOPz+/U+rZbNu2LY0bN+ajjz7CKDfTQU5ODp9++qn9jv+atGbNGv744w+XTpXlKupJFRERqY/69IHevWHLFsjIMHtQO3Z0Sw9qZcLDw5kwYQLjx4/no48+4o033mDw4MEMHDiQm266icaNG5Oens7WrVtZv349C/4anzt06FCmTZvGpEmTOP/889m+fTtTp06lRYsWDmNZr7nmGt555x3uvPNOtm/fTlJSEiUlJaxdu5b27dtz9dVXA+bd8suXL+fLL78kNjaWkJAQe69teV5eXsyaNYvrrruOoUOHcscdd1BQUMAzzzzD0aNHeeqpp1z68zr77LO5/vrr7dNp/fTTTzzzzDM0bNiQ8ePHu/S1XUEhVUREpL7y8oK/LqN7qrvvvptXXnmFqVOnsnXrVn766SdmzJjBfffdR0ZGBpGRkXTo0IErr7zS/pzHHnuM3Nxc3nrrLWbNmkWHDh14/fXXWbhwocOcplarlcWLFzNz5kzmzZvHCy+8QEhICGeffbbDSk8vvvgiY8eO5eqrryY3N5fzzz+/0qVQr732WoKCgpg5cyZXXXUV3t7e9O7dm2XLltHHxT3UHTp0YM6cORw6dIjjx4/TqFEjrr76aiZOnFjhOFlPZzGMap55142ysrKw2WxkZmaedA4zERGR2iQ/P5/du3fTokWLWrsOu9QPp/JePdW85jl9+iIiIiIif1FIFRERERGPo5AqIiIiIh5HIVVEREREPI5CqoiIiIh4HIVUEREREfE4CqkiIiIi4nEUUkVERETE4yikioiIiIjHUUgVEREREY+jkCoiIiIeYePGjYwaNcq+pGZwcDBdu3Zl1qxZpKenu7t5tUq/fv3o16+f/XuLxfK32+TJk09af9CgQTV6DtYafTURERFxu5QUyMqqfH9oKERH11x7AN58803GjBlD27Zteeihh+jQoQOFhYWsW7eO119/ndWrV7Nw4cKabVQd8dprr5FV7h/8P//5D9OnT+edd96hXbt29vImTZrYv2/ZsiUffvihw3HCwsJc3tbyFFJFRETqkZQUuPZaSEurvE5kJHz0Uc0F1dWrVzN69GguuugiPv/8c/z8/Oz7LrroIh588EG++uqrmmlMHdShQweHx9u2bQOgU6dOdO/evcLnBAQE0Lt3b5e37WR0uV9ERKQeycoyA6qfH4SFOW9+fub+k/W0Vrcnn3wSi8XCnDlzHAJqKV9fXy655BIASkpKmDVrFu3atcPPz4+YmBhuvPFG/vzzT4fn9OvXj06dOrF69Wr69OlDQEAAzZs355133gHM3sSuXbsSGBhI586dnULw5MmTsVgsbNiwgeHDhxMaGorNZuP6668nJSXFoe6ptql58+bcdNNNTudX/tI8wPLly7FYLMybN4/HHnuMRo0aERoayoUXXsj27dsdnmsYBrNmzaJZs2b4+/vTtWtXlixZcvIfeC2hkCoiIlLL5edXvh0/7li3oABKSswwGhBQ8XY6xz1dxcXFfPfdd3Tr1o2mTZv+bf3Ro0fz8MMPc9FFF/HFF18wbdo0vvrqK/r06UNqaqpD3cOHDzNq1ChuvfVW/v3vf9O5c2duvvlmpk6dyoQJExg/fjyffvopwcHBXHrppRw8eNDp9S677DJat27NJ598wuTJk/n8888ZOHAghYWFp9Wmqnj00UfZu3cvc+fOZc6cOfzxxx8MGzaM4uJie50pU6bYX/vzzz9n9OjR3HbbbU5htqp27txJREQEVquVVq1a8dhjj5GXl3dGx6wqXe4XERGp5a64ovJ93bvDpElljx94AHbvBqvV3MoLCYG4uLLHt9xSeY9qmzbw/POn3+ZSqamp5Obm0qJFi7+tu23bNubMmcOYMWN4+eWX7eUJCQn06tWLf/7zn8yYMcNenpaWxtdff023bt0A6N69OzExMTz11FPs2LGDRo0aAdCoUSO6dOnCp59+yt133+3wmsOHD2fWrFkADBgwgAYNGnDdddcxf/58rrvuuiq3qSo6dOjABx98YH/s7e3NlVdeyc8//0zv3r05evQoTz/9NJdddhlz58611+vYsSPnnnsubdu2Pa3XPe+887jqqqto164deXl5LFmyhFmzZvHDDz+wbNkyvLxqpo9TPakiIiJSKyxbtgzA6ZJ5z549ad++Pd9++61DeWxsrD2gAkRERBATE0OXLl3sARWgffv2AOzdu9fpNa+77jqHx1deeSVWq9Xelqq2qSpKhziUio+Pd2jn6tWryc/Pd2pjnz59aNas2Wm/7vTp0xk9ejRJSUkMGTKEl19+maeeeooVK1bw73//+7SPW1XqSRUREanlFiyofN+JnV7PP2/eOBUWBkFBzvXLX9F9661TP+7pioqKIjAwkN27d/9t3bS/7vaKjY112teoUSOnkBkREeFUz9fX16nc19cXgPz8fKf6DRs2dHhstVqJjIy0t6WqbaqKyMhIh8el43VLL7uXvvaJbays7Excf/31jBs3jjVr1nDZZZdV67Ero55UERGRWs7fv/Ltr/xl5+dnBszKttM97uny9vbmggsu4JdffnG60ehEpaHt0KFDTvsOHjxIVFRU9TSqnMOHDzs8LioqIi0tzd6WqrTJ39+fgoICp3qnO2619LVPbGNlZdWhpi71g0KqiIhIvZSXBzk5zlsN3xsDwIQJEzAMg9tuu43jFdyRVVhYyJdffkn//v0BHMZpAvz8889s3bqVCy64oNrbduJcofPnz6eoqMh+N35V2tS8eXM2btzoUO/3338/7Zucevfujb+/v1MbV61adUY9uBX5v//7P/tr1hRd7hcREalHQkPNeVDT0sw7/SsSGWnWqynnnHMOs2fPZsyYMXTr1o3Ro0fTsWNHCgsL2bBhA3PmzKFTp04sXLiQ22+/nZdffhkvLy8GDx7Mnj17eOKJJ2jatCn3339/tbfts88+w2q1ctFFF7FlyxaeeOIJzj77bK688koA2rZte8ptuuGGG7j++usZM2YMI0aMYO/evcyaNYvo05yQNjw8nHHjxjF9+nRuvfVWrrjiCvbv38/kyZNP+3L/ypUrmTFjBpdddhktW7YkPz+fJUuWMGfOHPr378+wYcNO67inQyFVRESkHomONifq97QVp2677TZ69uzJP//5T55++mkOHz6Mj48PZ511Ftdeey133XUXALNnz6ZVq1a89dZbvPrqq9hsNgYNGsTMmTOdxnBWh88++4zJkycze/ZsLBYLw4YN44UXXrCPY61Km6699loOHjzI66+/zjvvvEOnTp2YPXs2U6ZMOe32TZ06laCgIF577TXef/992rVrx+uvv86zzz57WseLjY3F29ubadOmkZqaisVioU2bNkydOpUHH3ywRi/3WwzDMGrs1VwsKysLm81GZmYmoTX5J6CIiIgL5efns3v3bvua9uJ6kydPZsqUKaSkpLhkrGtddSrv1VPNaxqTKiIiIiIeRyFVRERERDyOQqqIiIjICSZPnoxhGLrU70YKqSIiIiLicRRSRURERMTjKKSKiIiIiMdRSBURERERj6OQKiIiIiIeRyFVRERERDyOQqqIiIiIeByFVBERERHxOAqpIiIi9Uxqbio703dWuqXmptZIOywWyylty5cvP2n9p556qsLjr1y5kiuvvJLGjRvj6+uLzWajT58+zJ49m5ycHId23HXXXTVxylIFVnc3QERERGpOYXEhdy66k10Zuyqt0zK8JfNGzMPH28elbVm9erXD42nTprFs2TK+++47h/IOHTrYv7/88st58MEHHfbHxcU5HXvSpElMnTqVPn36MG3aNFq1akVubi6rVq1i8uTJ/P777/zzn/+sxrOR6qaQKiIiUo9Yvaw0DmnMhsMbaGZr5rR/b+ZeGoc0xurl+ojQu3dvh8fR0dF4eXk5lZfXoEGDk+4HWLBgAVOnTuWWW27hzTffxGKx2PcNHjyY8ePHOwVk8Ty63C8iIlLL5Rflk1+Uj2EY9rKikiLyi/IpLC50qFtQXMDVna7G5mejqKSIIN8gAn0CCbAGcLz4ODY/GyO7jMRisZz0uMeLj9fY+VXV1KlTCQ8P56WXXnIIqKVCQkIYMGCAU/n7779P+/btCQwM5Oyzz2bRokVOdf744w+uvfZaYmJi8PPzo3379rz66qsOdZYvX47FYuGjjz7i4YcfJjY2luDgYIYNG8aRI0fIzs7m9ttvJyoqiqioKEaNGsWxY8eq7wdQRyikioiI1HJXLLiCKxZcQVZBlr3ss62fccWCK3h93esOda//7HpmrJxBt9huJOckYxgGR3KO8PPBn/k97XcS4xJJaJgAwC1f3MIVC65gf9Z++/O/3fUtVyy4glk/zqqZkzvBRx99REBAAH5+fnTr1o133nnHYf+hQ4fYvHkzAwYMIDAw8JSP+5///IdXXnmFqVOn8umnnxIREcFll13Grl1lwyJ+++03evTowebNm3nuuedYtGgRF198Mffccw9TpkxxOuajjz5KcnIy7777Ls899xzLly/nmmuuYcSIEdhsNubNm8f48eN5//33efTRR0//h1JH6XK/iIhIPWOxWLiq01X8cugXe7AtKikiwCfA3ovqia699louvvhimjZtSnJyMm+99RY333wzu3btYtq0aQDs27cPgBYtWlTp2Hl5efz3v/8lJCQEgK5du9KoUSPmz5/PI488AsADDzxASEgIP/zwA6GhoQBcdNFFFBQU8NRTT3HPPfcQHh5uP2Z8fLxDiN62bRsvvPAC99xzD88884z9+atXr+bDDz/kpZdeOs2fTN3k0pC6YcMGpkyZwk8//cTRo0eJi4vj2muvZdy4cQ5/3axfv57x48ezZs0arFYr/fv359lnn6Vly5aubJ6IiEidsOCKBQD4efvZy4a3H84lbS/B2+LtUPeD4R8A4OvlS2JcIl/v/JpW4a3IzM9kQKsB9l5UgLcuecvpuBe0vIDzm5+Pl6XmL8Z++OGHDo9HjBjBsGHD7AExOjr6tI+dlJRkD6hgjn2NiYlh7969AOTn5/Ptt98yevRoAgMDKSoqstcdMmQIr7zyCmvWrGHw4MH28qFDhzq8Rvv27QG4+OKLnco///xzjh07RnBw8GmfQ13jsnfYb7/9Rp8+fdizZw8vvPACixYt4uqrr2bq1Klcc8019nrbtm2jX79+HD9+nPnz5/P222/z+++/k5iYSEpKiquaJyIiUmf4W/3xt/o79IBavaz4W/2d7tAvrevl5cXILiMJ9Ankz+w/CfYN5uaEmx2OcbLj+nr7uv7ETsH1119PUVER69atA8ru9N+9e3eVjhMZGelU5ufnR15eHgBpaWkUFRXx8ssv4+Pj47ANGTIEgNRUx6m7IiIiHB77+vqetDw/P79Kba7rXNaT+tFHH5Gfn8+nn35Kq1atAOjfvz+HDh1izpw5ZGRkEB4ezsSJE/Hz82PRokX2rvNu3brRpk0bnn32WZ5++mlXNVFERKReS2iYQGJcIgu3LeSydpc59KLWFqU3dXl5mf1usbGxdO7cmW+++Ybc3NwqjUs9mfDwcLy9vbnhhhsYO3ZshXWqOsRATs5lIdXHx/zLzWazOZSHhYXh5eWFr68vRUVFLFq0iBtvvNEeUAGaNWtGUlISCxcu9LiQmpICWVmV7w8NhTO42iAiIlJjLBYLoxJGkVOYw6iEUR47FvVk3n//fXx8fOjWrZu97IknnuDKK6/knnvucZqCCuDYsWOsWrWqwjv8KxMYGEhSUhIbNmwgPj7e3vspruOykDpy5EheeOEFRo8ezdNPP010dDTff/89b7zxBmPHjiUoKIjt27eTl5dHfHy80/Pj4+NZunQp+fn5+Pv7V/gaBQUFFBQU2B9nnSw9VoOUFLj2WkhLq7xOZCR89JGCqoiI1A5dGnZh7iVz3d2Mv/XMM8/w22+/ccEFF9CkSRP7jVPffPMNkydPJioqyl73iiuu4IknnmDatGls27aNW265xT6Z/9q1a3njjTe46qqrqhRSAV588UXOO+88EhMTGT16NM2bNyc7O5sdO3bw5ZdfOi1CIGfGZSG1efPmrF69mssuu8x+uR/gnnvu4YUXXgDM8R3gPDajtMwwDDIyMoiNja3wNWbOnFnhlA+ukpVlBlQ/PwgIcN6fl2fuz8pSSBUREalO7dq144svvuA///kPGRkZBAQE0KVLF+bNm8fVV1/tVH/q1KlceOGFvPzyyzz22GOkpqYSEBBAx44deeCBB7jjjjuq3IYOHTqwfv16pk2bxuOPP05ycjJhYWG0adPGPi5Vqo/FKD9DbzXas2cPF110EQ0aNOD+++8nOjqatWvXMn36dC6//HLeeustVq1axbnnnsvHH3/MVVdd5fD8mTNn8uijj3Lo0CEaNmxY4WtU1JPatGlTMjMzHYYPVJedO+GKKyAsDIKCnPfn5MDRo7BgAZTL5SIiImckPz+f3bt306JFi0qvLop4glN5r2ZlZWGz2f42r7msJ/WRRx4hKyuLX3/9laC/El3fvn2Jiori5ptv5sYbb7SHz7QKrp+np6djsVgICwur9DX8/Pzw8/OrdL+rlZTArl3QoAGUm7VCRERERM6Qy0Lqr7/+SocOHewBtVSPHj0A2Lx5M+eeey4BAQFs2rTJ6fmbNm2idevWHv0XY3IypKebW2goVDBqQUREREROg8vmSW3UqBFbtmxxWot29erVADRp0gSr1cqwYcP47LPPyM7OttfZt28fy5YtY/jw4a5qXrWIiICYGLBYzHGoO3bAwYPmVxERERE5fS4Lqffddx+pqalcdNFFzJ8/n++++44nn3ySBx54gA4dOthXZJgyZQq5ubkMHTqUJUuWsHDhQi6++GKioqJ48MEHXdW8auHrC82bw9lnl4XVvDx4+mmYNAnKLUYhIiIiIlXgspB6ySWX8O233xIaGsq9997L0KFD+b//+z/uuOMOVqxYYZ9frF27dixfvhwfHx8uv/xybrrpJlq3bs2KFSvOaHkzV8rLM2+SKt0KC827+Zs3Ny/7e3uDvz9YXbrorIiI1DcuutdZpNpU53vUpTEqKSmJpKSkv63XrVs3/vvf/7qyKdUiNNScBzUtDcpNKuCgXTt44QWzXqnkZJgzB666Ctq0qZGmiohIHWL9q9ejSJfoxMOVvket1dBTp76+KoiONifqr+qKU/Pnw9q15tazp7kggKaoEhGRU+Xt7Y23tzdZWVmEaDoZ8WBZWVn29+uZUkitoujoqk/UP3w4HD8Oy5fDTz+ZW69eZlht2dIlzRQRkTrEYrEQExPDoUOH8PPzIygoqFYuYSp1l2EY5OTkkJWVRWxsbLW8P102mb87nOrksO5y4AB8/DF8/z2U/tQvuADuu8+tzRIRkVrAMAwOHz5MZmamxqaKR7JYLNhsNho2bHjSkOr2yfzFWePG8OCDcOWVZlhdudJx7KqIiEhlLBYLsbGxxMTEUFhY6O7miDjx8fGplsv8pdST6kb79kF4eNlqVZs2weLF5jCApk3d2zYRERERV1BPai0QF+f4+KOPYPNm+PFHSEyEa66BJk3c0zYRERERd3LZPKlSdbffDuecY45XXbECxoyB5583x7KKiIiI1Ce63O+Bdu0ye1XXrjUfWyzmONbrr3dvu0RERETO1KnmNfWkeqCWLeHxx+Gf/4QePcye1caN3d0qERERkZqjMakerHVrmDgRdu40l1wt9fXXsH27uYJVgwZua56IiIiIyyik1gLlV6cqKjKnr0pNhe++M+dZveoqiIlxX/tEREREqpsu99cyVis8/DAkJEBxMXzzDdxxB7z6KqSkuLt1IiIiItVDN07VYlu3wocfwv/+Zz62Ws0ZAgYPdm+7RERERCqjG6fqgfbtYfp0eOopiI83hwKUHxogIiIiUltpTGod0LEjzJgBu3dDixZl5f/3f1BQAJdfDhER7mufiIiISFUppNYh5QNqZib8+99QWGjOBjB4sBlWw8Lc1jwRERGRU6bL/XVUaKg5fVX79nD8uBlYb7kF3n7bDLAiIiIinkw3TtVxhgEbNpgrWG3fbpb5+cH48dCzp3vbJiIiIvXPqeY1Xe6v4ywW6NrVnLJq/XpzNoC9e6FNm7I6hmHWExEREfEUCqn1hMUC3bqZgfXAAQgPL9v39NPmsquXXgohIW5rooiIiIidQmo9Y7FAkyZlj//4A3780fz+yy/hkkvMsBocbC4OkJVV+bFCQyE62qXNFRERkXpKY1LrOcOAtWvNMau7d5tlgYGQlATz58PRo5U/NzLSfJ6CqoiIiJwqjUmVU2KxQO/e0KsXrF4N8+bBnj1mQN2wAZo3dxwaUCovD9LSzJ5WhVQRERGpbgqpAphhtU8fOOccWLUKZs82l1sNC4OgoIqfU1BQo00UERGRekQhVRxYLHDuudCggRlSS+/6Nwz4/Xdo3Rq8vd3bRhEREan7NJm/VMhiAR+fsseHD5uLAKSnu69NIiIiUn8opMopKb29TiFVREREaoJCqpySiAjza1YWFBa6ty0iIiJS92lMqpxUXl7Z976+kJsLhw6Z01SJiIiIuIp6UqVCoaHmPKgFBeZcqUePmmNUi4rgyBGzPDLSrCciIiJS3dSTKhWKjjYn6i+/4lRaGjzyiHlT1axZEBenOVJFRETENRRSpVLR0Y4htFUr6N4dtm6FnBwFVBEREXEdhVSpkjvvNC/xR0W5uyUiIiJSlymkSpW0bOnuFoiIiEh9oBun5LQVF7u7BSIiIlJXqSdVquzAAZgzB44dg+eec3drREREpC5SSJUqCwmBX3+FkhIzsDZu7O4WiYiISF2jy/1SZaGhkJBgfr9ypXvbIiIiInWTQqqclsRE86tCqoiIiLiCQqqclt69wWqFfftg7153t0ZERETqGoVUOS1BQdCtm/m9elNFRESkuimkymnr29f8umIFGIZ72yIiIiJ1i+7ul0ql5qaSmZ9Z6f6WnWx06RJFnz7mnf7e3jXYOBEREanTFFKlQoXFhdy56E52ZeyqtE7L8JbMmzwPH2+fGmyZiIiI1Ae63C8VsnpZaRzSmMyCTML8w5y2zIJMGoc0xuqlv3NERESk+imkSoUsFgsju4zE5mejqKSIIN8g+1ZUUoTNz8bILiOxWCxkZsLixbBnj7tbLSIiInWFQqpUKqFhAolxiRw+dpgDmQfYn7mfkpISknOSSYxLJKGhOaP/W2/B7NnwzTdubrCIiIjUGQqpUqnS3lR/qz97M/dy+NhhDh07RKBPoL0XFcom9v/hB/MGKhEREZEzpZAqTkqMEv5I+wMwe1OTmidhsViIs8WRW5jr0IsK5hKpwcGQkQFbtrir1SIiIlKXKKSKg5zjOdyz5B7G/3c8R44dsfemxtniKCgucOpFBXPlqXPOMb9fscJNDRcREZE6RSFVHAT5BhHuH46ftx/7MvcBZWNTM/IySIxLpHV4az7b+hlGuRn8Syf2X7UKiorc0XIRERGpSzR/UD33Z9affL7tc27vdju+3r4A3N3rbgJ9Agn2DQbMsamjEkaRU5jDDWffwLil4zh07BB+3n5cfNbFAHTuDDYbZGbCxo3QtavbTklERETqAPWk1mMlRgmTlk3i651fs/iPxfbymKAYe0At1aVhF+ZeMpfujboz9KyhNApuRNuotvb93t5w7rnm1/37a+wUREREpI6yGEbdWXU9KysLm81GZmYmoaGh7m6OR8ovysff6m9/vHTnUtYeWMvIs0fS1Nb0lI5hGAYFxQUOxwFITwcfHwgJqdYmi4iISB1yqnlNIbUe+WL7F8zbPI8J500gvkE8YAbO8jdBnY6c4zkE+QZVRxNFRESkjjvVvKbL/fXIoexDHDt+jKU7l9rLzjSgrvlzDbd9eRur9q9yKD927IwOKyIiIvWcbpyqowzD4KcDP9Emsg0RAREAXN3palpFtKJ/i/7V9jrbU7eTfTybJX8s4Zwm55CdbWHaNHOJ1A8+AD+/anspERERqUcUUuuoN9e/yZe/f8mgVoMY23MsADZ/Gxe2vLBaX+e6+OsIDwhnSJshWCwWQkLMsan5+bBunXkzlYiIiEhV6XJ/HXVe3Hn4evti87fhymHHVi8rl7S9BKuX+feOxVI2Z6om9hcREZHTpZ7UOiA1N5WPNn1E28i2DGw9EIAO0R149x/vEuJXc7faG4bB0l1LadC5MXzSkXXrIC8PAgJqrAkiIiJSR6gntQ5Y8+calu5ayoebPqSwuNBeXpMBFWDJjiW8/NPL/Gvfs8Q0yeH4cVi7tkabICIiInWEy0PqDz/8wJAhQwgPDycgIIA2bdowbdo0hzrr16/nwgsvJDg4mLCwMIYPH86uXbtc3bRa63jxcVJyUuyPB7UeRL9m/Xgs8TF8vH3c1q6k5kk0DW3KsLbDSDo3ENAlfxERETk9Lg2pH330Eeeffz42m4333nuPxYsX8/DDDzuMkdy2bRv9+vXj+PHjzJ8/n7fffpvff/+dxMREUlJSTnL0+mlz8mZu//J2nl/9vP3naPWy8mCfBx1WgHKHAJ8AXhr8EsPbD+f8882prTZs0HRUIiIiUnUuG5N64MABbr/9du644w5ee+01e3lSUpJDvYkTJ+Ln58eiRYvsE7p269aNNm3a8Oyzz/L000+7qom1UsPghmQVZOFl8eJo/lHCA8Ld3SQHpTdQNW0KQy4uoWnrLHx9w9zbKBEREal1XNaTOnfuXHJycnj44YcrrVNUVMSiRYsYMWKEw4oDzZo1IykpiYULF7qqebXGluQtLP5jsf1xVGAUU5Om8vrQ1z0uoJaXnpfOn22e4OvjT4D3cXc3R0RERGoZl4XUFStWEBERwbZt2+jSpQtWq5WYmBjuvPNOsrKyANi5cyd5eXnEx8c7PT8+Pp4dO3aQn59f6WsUFBSQlZXlsNUlO9N38si3j/Dm+jdJzkm2l3eK6YSvt68bW/b3LFjYm7mXwzmH2ZWh8cUiIiJSNS4LqQcOHCA3N5crrriCq666iv/+97889NBDvPfeewwZMgTDMEhLSwMgIiLC6fkREREYhkFGRkalrzFz5kxsNpt9a9q0qatOp8YUlRTZv28V0Ypusd24sMWFHh9KTxQeEM6E8ybweLcX2Ly8HatW/f1zREREREq5bExqSUkJ+fn5TJo0iUceeQSAfv364evry3333ce3335LYKB5B/jJ1o8/2b4JEybwwAMP2B9nZWXV2qCaV5jHvM3z+OnAT7w0+CV7KJ14/kS8LLVzprCOMR1Z+CP83/9Bp07Qp4+7WyQiIiK1hcvST2RkJAADBw50KB88eDBgTjtVWqe0R7W89PR0LBYLYWFhlb6Gn58foaGhDltt5e3lzYq9KziQfYA1f66xl9fWgFrqvPPMr+t+P8Bz379OiVHi3gaJiIhIreCyntT4+HjWrFnjVF46bZKXlxetWrUiICCATZs2OdXbtGkTrVu3xt/f31VNdKvikmLWH1pPj8Y9APD19mV099FYvax0je3q5tZVn+hoOKv9cf6VN4HM9Rm0iIlhePvh7m6WiIiIeDiXddONGDECgCVLljiUL15s3qneu3dvrFYrw4YN47PPPiM7O9teZ9++fSxbtozhw+tmmCkqKeLer+5l6oqpbDpSFtB7NelFt0bdTjrEoTbql+hL07Sb8EqJp1/zfu5ujoiIiNQCLutJHTBgAMOGDWPq1KmUlJTQu3dv1q1bx5QpUxg6dCjn/XUdeMqUKfTo0YOhQ4fyyCOPkJ+fz8SJE4mKiuLBBx90VfPcyuplpWN0RzLyM8gqqFszElTk3HMh6s0kjE1JFGVbIMDdLRIRERFP59IBj//617+47777mDNnDoMHD2b27Nncf//9fPLJJ/Y67dq1Y/ny5fj4+HD55Zdz00030bp1a1asWEF0dLQrm1dj9hzdw5MrnyQ9L91edsPZN/DmsDc5N+5cN7asZkREQHxnCxYsrFxplh3MPuiw8piIiIhIeRajDiWFrKwsbDYbmZmZHnUT1cNLH+a31N8Y3HowY3qMcXdz3OKrr2DuXLj0Ugju/jnv/vouY3uM5aJWF7m7aSIiIlKDTjWvuexyf12VmptKZn5mpftt/jZ8vX0J9Am0LxF649k38p8//sNl7S6rqWZ6nKQk6NcP/P3hk9+KKDaK2ZKyRSFVREREKqSe1CooLC7kmk+vOekKSgE+ATQKacQtCbcwqPWgam9DXWAYBj8d+ImejXvWuZvERERE5ORONa/V7kk4a5jVy0rjkMZkFmQS5h/mtGUWZBLhH0FeYR5r/1zr7uZ6rIMHLfRq0ksBVURERCqly/1VYLFYGNllJCv3raSopAibn430/HT8vP3sj5/o+wSFJYX0aarllU5UUgIPPgg7dsArr0CzZuZ0XO//7306xXSyzxkrIiIiop7UKkpomEBiXCLJOcn8mfUnO9J3sO/oPpJzkkmMS6RH4x6cG3euegkr4OUFfy0yZr/L/8vtX/LZts94Ye0L5Bbmuq9xIiIi4lEUUquotDc10CcQf6s/Vi8rVm8rAdYARnYZqXD6N/r2Nb+uWAGGAUPPGkp8TDx397ybQJ9A9zZOREREPIZC6mko7U3NyM/g7AZnA9C3WV8SGia4uWWer2dP8PWFQ4dg507w8fZhev/p9G7S291NExEREQ+ikHoayvemHsg+QKBPoHpRT5G/vxlUoeySf/mfW35RPgeyDrihZSIiIuJJFFJPk703NS+DxLhE9aJWQWKi+XXlSvOSf6kDWQe4d8m9TF4+WeNTRURE6jmF1NNksVgYlTCKAa0GMCphlHpRq6B7d7NHNSUFtm0rKw/zD6OopIgio4jknGT3NVBERETcTpP5i1ssWQJRUZCQANZyE6HtObqHyIBIQvxC3Nc4ERERcRktiyoebfDgisubhzWv0XaIiIiIZ9LlfvFY/zv8P6Z+P5WikiJ3N0VERERqmEKquM2RI/Dee/Dxx8778ovymbVqFj8f/JnPt31e420TERER99LlfnGbQ4dgwQIIDYXLL3ccm+pv9eeenvfw88GfGXbWMPc1UkRERNxCIVXcpnNnsNkgMxM2boSuXR3392rSi15NermncSIiIuJWutwvbuPtDeeea36/YsXf1994ZCN1aDIKEREROQmFVHGr0on916yBwsLK6835ZQ6PffeYxqeKiIjUEwqp4lYdO0JEBOTkwPr1ldeLs8VhwUJ+UX7NNU5ERETcRmNSxa0sFrM39d//Ni/596pkCOrAVgNpF9VO86iKiIjUE+pJFbdLTISQELNHtTIWi8UhoGpsqoiISN2mnlRxu7POMudLtZ7iuzG7IJsX177IeXHn0a95P5e2TURERNxDIVXczmI59YAKsHTXUtYeWMvW1K30btIbf6u/6xonIiIibqGQKh7DMGD7dmjeHPxPkjv/0fYf7M/cz7C2wxRQRURE6iiNSRWPMXEiPPSQOR3VyXh7eXNv73tpGd6yZhomIiIiNU4hVTxGu3bm11OZ2L+8lJwUtqdur/4GiYiIiNsopIrH6NvX/LphAxw7dmrP+SPtD+756h5mrJxBZn6m6xonIiIiNUohVTxG06bmeNSiIli9+tSeE2eLI9w/nOjAaAqKC1zaPhEREak5unFKPEpiIuzZY17yv+iiv6/vZ/VjatJUwvzDsHrp7SwiIlJXqCdVPEpiovl140bIPMWr91GBUQ4BtcQocUHLREREpCYppIpHiY2FNm2gpATWrq3acw3D4KsdX3H/V/eTX5TvmgaKiIhIjdD1UfE4I0eCtzd07Fi15+UW5jJv8zzS89L5esfX/KPdP1zTQBEREXE5hVTxOGeffXrPC/IN4sFzHmRn+k4uaXtJ9TZKREREapRCqtQp8Q3iiW8Q7+5miIiIyBnSmFTxSKmp8PrrMH366R+jxChh6c6lFJcUV1/DREREpEaoJ1U8krc3LF4MhgHJyRATU/VjzFw5kzUH1nAk5wjXx19f/Y0UERERl1FPqnik8HDo3Nn8fuXK0ztG32Z98bf60yikUfU1TERERGqEelLFYyUmmvOlrlgBI0acxvObJRLfIB6bv636GyciIiIupZ5U8Vh9+oCXF+zaBQcOnN4xygfUwuJCDMOoptaJiIiIKymkiscKDYWEBPP7073kX+rPrD8Z9804luxYcuYNExEREZdTSBWPVrpM6ooVZ3acDYc2sOvoLhb8toDjxcfPvGEiIiLiUhqTKh6td29o1Ai6d4eiIrCe5jt26FlDyT6ezaDWg/D19q3eRoqIiEi1sxh1aJBeVlYWNpuNzMxMQkND3d0cqSaGARaLu1shIiIi1eFU85ou94vHc0VA3Z66nV8O/lL9BxYREZFqoZAqtUJREaxfD4cOnfmxNh3ZxMP/fZhnVj1DSk7KmR9QREREqp1CqtQKr74KkybBV1+d+bHaR7enVXgrusZ2JdAn8MwPKCIiItVOIVVqhR49zK8rV5pjVM+E1cvKtP7TeKjPQwT5Bp1540RERKTaKaRKrdC9OwQEQEoKbNt25scL9AnEUm6wa15h3pkfVERERKqNQqrUCr6+5nRUcOYT+5dXWFzIW+vfYuzisWQXZFffgUVEROSMKKRKrVE6sf8PP0BJSfUcs9goZu2BtaTkprDmzzXVc1ARERE5Y5rMX2qNhAQIDoaMDNiyBTp3PvNj+lv9GX/ueHam76RleEt2pu+ssJ7N30ZUYNSZv6CIiIicEoVUqTWsVjjnHFi6FH79tXpCKkAzWzMe+e8j7MrYVWmdluEtmTdiHj7ePtXzoiIiInJSCqlSq4wYAUOHQosW1XdMq5eVxiGN2XB4A01CmpCSk0JMcAxeFnM0zN7MvTQOaYzVS/9dREREaorGpEqt0rgxtGxZvatQWSwWRnYZic3Pxq6MXaTlp5Gel06QbxBFJUXY/GyM7DLSYTYAERERcS2FVKm1znS+1PISGiaQGJeIt5c3vl6+RAREYBgGyTnJtApvRVZ+Fhl5GdX3giIiInJSCqlS6+TkwPPPw803Q2Fh9RyztDc1IiCCZmHNCPELIasgi0CfQLwsXjy35jl2pO+w1997dC+zf57Nd7u/q54GiIiIiAOFVKl1AgNh40ZITYX166vvuKW9qam5qfZe1MS4RPq36E/H6I7E2eLsdX9P+53FOxazbPcyh2M8u+pZnv7haf7M+tNeZlRnl6+IiEg9oZAqtY7FAuedZ36/YkV1HtfsTQ30CWR/1n4CfQIZ2WUkd3S/g6cufIoGwQ3sdZuHNefy9pfTt1lfe5lhGKw9sJYf9v/gcNxle5Zx0+c38faGtx3KM/IyFGBFREQqoduVpVZKTIR//xt++gkKCsDPr3qOW9qbunDbQi5rdxkJDRMqrNcmsg1tIts4lBkYjO8znj+z/qRhcEN7+Z9Zf5KWl0ZBUUFZXcNg9H9GU1hSyCuDXyE2JBaA5JxksguyaRzaGH+rf/WclIiISC2kkCq10llnQUwMJCfDzz+X9ayeKYvFwqiEUeQU5jAqYVSV7uj3snjRo3EPejTu4VA+ov0IejXuRaBPoL3s2PFjFBQXUFxSTGRgpL186c6lfLzlYwa0HMDdve62ly/5YwkNgxvSKaaT5moVEZF6QSFVaiWLBfr2hU8+gZUrqy+kAnRp2IW5l8yttuMF+QbRNqqtQ1mIXwifXPEJqbmp+Hr7Ouyz+dloEtrE/ji7IJvX1r0GwIIrFuCDGVJX71/NroxddG/U3en4IiIitV2NjkmdO3cuFouF4OBgp33r16/nwgsvJDg4mLCwMIYPH86uXZWvACSSmGh+XbcOcnPd25bT4e3l7TDOFeC6+Ov4YPgHXNruUntZQXEBvRv3Jj4m3mEIwKr9q/h4y8dsSt5kL8stzOXx7x5n9s+zKTFKXH4OIiIirlJjPakHDhxg3LhxNGrUiMzMTId927Zto1+/fnTp0oX58+eTn5/PxIkTSUxM5NdffyU6Orqmmim1SIsW0KsXtG4NJXUsj5UfZhAVGMVjfR9zqtO9UXd8vX1pH9XeXnYg6wD/O/I/9mXuY3SP0fby2T/PZmvqVq7udDV9mvYBoLikmMKSQo19FRERj1RjIfXOO++kb9++RERE8MknnzjsmzhxIn5+fixatIjQ0FAAunXrRps2bXj22Wd5+umna6qZUotYLPD44+5uhfuc3/x8zm9+vkNZTFAM9/W6j8ISxwlkd2bsZPfR3Q69q3sz93LvV/fSIqwFLw1+yV6+P3M//lZ/ogKjtMqWiIi4TY1c7v/ggw/4/vvvee2115z2FRUVsWjRIkaMGGEPqADNmjUjKSmJhQsX1kQTReoEm7+NC1pewKDWgxzKx/UZx8S+E+kU08ledij7EIBTT+qrP7/KzV/czIq9ZfN7peels2LvCvYc3eO6xouIiJTj8p7U5ORk7rvvPp566imaNGnitH/nzp3k5eURHx/vtC8+Pp6lS5eSn5+Pv78uSUrFCgrMO/xDQuDss93dGs/UMLihw7RYAOfGnctHDT4ipzDHqb7Vy0rj0Mb2x1tTtvLMqmdoG9mWZwc8ay//cvuX9mNFBES4qPUiIlIfuTykjhkzhrZt2zJ69OgK96elpQEQEeH8Cy4iwlw/PSMjg9jYWKf9BQUFFBSUzT2ZlZVVTa2W2uSLL+C996BLF4XUqgrxCyHEL8Sh7KkLn6K4pNjhUr+vty8dojrQOqK1Q91Ptn5Cel46Z0WeZQ+pm5M3s/iPxZzd4GwGth7o+pMQEZE6yaUh9dNPP+XLL79kw4YNfzu27WT7K9s3c+ZMpkyZckZtlNrvvPPMkLpxI2Rmgs3m7hbVft5e3g6PK5r/1TAM+jfvz59ZfzpMmbUtdRsr963Ey+LlEFIf+uYh/K3+3NXzLvusBkUlRXhbvDX2VUREnLgspB47doyxY8dy991306hRI44ePQrA8ePHATh69Cg+Pj5ERpoTmZf2qJaXnp6OxWIhLCyswteYMGECDzzwgP1xVlYWTZs2rd4TEY8XGwtt2sAff8CPP8KQIe5uUf1QuozsibrGdsXb4u0wXKCgqIBtadsACPAJsJcv+n0RH2z8gIvbXMyohFH28n2Z+4gJitHMAyIi9ZjLQmpqaipHjhzhueee47nnnnPaHx4ezj/+8Q8++eQTAgIC2LRpk1OdTZs20bp160rHo/r5+eFXXethSq2WmGiG1JUrFVLdrWV4S1qGt3Qo8/by5pmLnuFQ9iFC/cpukDyQdYCC4gKHVbQKiwu5a/FdGBi8f9n7hPmHAbDn6B6O5h+leVhze5mIiNRdLgupDRs2ZNmyZU7lTz31FN9//z1LliwhKioKq9XKsGHD+Oyzz5g1axYhIeb4uH379rFs2TLuv/9+VzVR6pDERHj7bdiyBdLSIDLy758jNcfqZaVdVDvaRbVzKL+j+x1c2u5Shx7TjPwMQvxCKCopwuZXNnZjyR9LWLxjMZe3v9zeg1tUUsT8LfNpEtqEc5ue6zRMQUREai+XhVR/f3/69evnVP7uu+/i7e3tsG/KlCn06NGDoUOH8sgjj9gn84+KiuLBBx90VROlDomKgvbtYetW+OEH+Mc/3N0iORUnziIA5lyvHw7/kNzCXIexqjZ/G01CmhBni7OXHTl2hHmb5+Hn7UdiXKK9fMkfS9ibuZe+zfrSIbqD609ERESqXY0ui1qZdu3asXz5cnx8fLj88su56aabaN26NStWrNBqU3LKSpdJ3bvXve2Q6hHoE+jw+NrO1zJ76GySWiTZy7wsXlzU8iIS4xIdAu2aP9fwnz/+w/7M/fay5Jxk7l1yLy+secHhuFo+VkTEM1kMwzDc3YjqkpWVhc1mIzMz02FhAKkfjh2D7GzzRiqp31btX8Xvab+T1DyJZmHNAFh3cB1Tvp9CM1szXhnyir3utO+nsevoLu7sdie9mvQCzBu9so9nExkQedozD6TmppKZn1npfpu/jajAqNM6tohIbXaqea3GlkUVcbXgYHMT6dO0D32a9nEoOyvyLB5PdF5Hd3/WflJzUx1mHdiaupUnlj1By7CWvDj4RXv5luQtBPgE0CS0Cb7evpW+fmFxIXcuupNdGbsqrdMyvCXzRsxzuGlMRETKKKRKnZSXBwEBf19P6o9Qv1B7T2l5zw14jj+z/qR5WHN7WUZeBt4Wb6KDHIcbvfrzq+zP2s/UflNJiE0AzBkKNhzeQKvwVrSPbg/8NdY2pDEbDm+gma2Z02vuzdxL45DGWL30ESwiUhl9QkqdYhjw9NOwdi288go0bvz3z5H6LcQvxB4uSyW1SCKxWSJ5hXn2MsMwCPcP52j+UYebvTYe2cgbv7xB99juTOo3CTDnkA33D6e4pJj8onwiA8umm8jMz8TmZ2Nkl5FaxEBE5CQUUqVOsVggPx+Kisw5U6++2t0tktrK6mV1WDLWYrEw44IZgBlYS0UGRtKrcS+HWQQMw2Bj8kYsFgvJOclEBERgsVg4nH2Y7Wnb6dGoBwkNE+z1P978MVYvKwNbDbS/Zl5hHkUlRQT7BivMiki9pJAqdU5iIvzyC6xYoZAqrlE+NPZs3JOejXs67C8sKeT6+OtpFd6K/+7+L1kFWdj8bWQfz8ZisdCzcU/7MQzD4F9b/kVRSRHnNzvfHlKX7lrKm+vfJDEukfHnjrcf+4U1L+Bl8eL6+OuJCIgAzJu0UnNTiQqM0s1YIlJneMQUVCLVqXdvsFph/35NRyXu4evty/D2w5l10Sz6xvUlOScZwzAoMUoY0X4E9/W+z163xChh2FnDuKDFBQ6rceUW5gI4lBmGwfd7v2fprqUUlxTby3/c9yMPLX2Idza849COCf+dwCP/fYQjx47Yy/Zn7ufbXd+yPXW7Q906NNGLiNQR6kmVOicoCLp1M8elrlgBN9zg7hZJfWWxWBjZZSQr961kf9Z+gn2DuavnXTQIbmCv4+3lzc0JNzs99+pOV3N5h8spKimylxkY3NHtDnvPbCkfbx8aBDVw6EU1DINtadsoKilyWIlr/aH1zN0wl75xfXko6iF7+c1f3ExxSTFPXvAkTUKbALA9dTs/HfiJ1hGtOafpOfa6ablpBPoE4m/111AEEXEZhVSpk/r2NUPqypVw/fXmWFURd0homEBiXCILty3ksnaXOYxF/TtWL6vDDABeFi8GtR7kVG9ImyEMaTPEqXxKvylkFWQR5h9mL4sMjCShYQKtIlrZywzDICMvg2Kj2GGJ2m2p25j/23z6xvV1CKn3fX0fR/OP8tKgl2gR3gKADYc28M3Ob+gY05GhZw21192ashVfb1+a2pqedNouEZETKaRKndSzJ/j6wqFDsHMntG7t7hZJfWWxWBiVMIqcwhxGJYyqsZ5Hi8VCfIN4p/Lz4s7jvLjznMrnXjKXrIIswv3D7WXNw5oztM1QWkeU/QcyDIOCogLAcSjC3sy9/LD/B7y9vB1C6pM/POkUaFftX8W/Nv+LrrFdGdllpL3u8j3L8bJ4kdAwwT421zAM9daK1FMKqVIn+fvD8OEQEgIxMe5ujdR3XRp2Ye4lc93djEpZLJYKb7o6u+HZnN3wbKe6/7r8XxQUF+Dn7Wcvj28Qz+1dbyc2pGzJN8MwiAqIwtvi7TA84cixI+w6uos4W5zDseeun0tmQSYvD37ZHlK/2/0ds9fNpk/TPjxwzgP2uvO3zKfEKOGilhfZp/jKK8wjryiPEN8QLZIgUgcopEqddd117m6BSN1ksVgchgWAuYJWy/CWTvX+OeifTs9PbJZInC3OoScWzKB7NP+ow/CErIIsCooLnG7s+mL7F2QWZNK7SW97SP1x/4+8uPZFusV2Y3K/yfa6L699mcKSQq7pdI09RKfnpXMg6wBRgVEOwVpEPIdCqoiI1KjKpsoqP9VWqSFthnBO03Pwtng7lA9uPZiM/AwiA8oWSsgvysfL4uUUftccWENWQRYj2o+wl60/tL7CQDtx2URyjudwd6+77auQHcg6wKbkTTQKaeQwhEJDEURcSyFV6rRjx2D1anNy/8GD3d0aEakqP6sfDYMbOpVfF+98qWToWUO5uM3FFJYUOpTfknALmfmZDsHYx8uHJiFNnI69I32HOZ8tZeHzt5TfePXnV+kW280hpI75zxiOFhxl8vmTaRvVFoBdGbtYvmc5LcJakNQiyV43OScZf6s/Ib4hCrYip0ghVeq0rVvhpZcgPBwGDgQvzQwsUqdZLBanWQT6t+jvVO/85udzfvPzncqf6PsEWQVZDtOERQRE0LNRT86KPMuhbmZBJseOHyPAJ8BetjN9Jwu3LaR7bHeHkDpx2UQOZB9g5gUz6RTTCYAtyVtYuG0hZ0WexZUdr7TX/S3lN7wsXjSzNXM4tkh9o5AqdVpCAgQHQ0YGbN4M8c43O4uI2LWPbu9U1q1RN7o16uZUPvvi2WQVZDn0xsbZ4ri07aX2uWZLlc53W34owsHsg6w9sJYSo8Sh7otrXuTgsYMOgfaXg7/w1oa36BTTiTE9xtjrfr/ne4pKiuga25XwAHNmhhKjBAsW9dhKraeQKnWa1QrnnANLl5pzpiqkikh1sfnbHGYtAGgb1dZ+6b+8uZfMNRdWKDe2tkN0B8b2GGtf3rZUTFAMxUaxww1kaXlp7M/a7zQ84cNNH3Lo2CGeuuApe0hd++danv7xabrGdmXi+RPtdT/57RPyCvO4sOWF9pvF8grz7ItDnHgznIi7KaRKnde3rxlSf/wR7rjDDK4iIjWt/MIMAI1DG9M4tLFTvWn9pzmV9Wzckxn9ZxBgdbz8f3aDs4kNjrXPcADmjAjFRrHDuFqAb3Z+w6Fjh+jWqJs9pP56+Fee/OFJ2kW245kBz9jrvr7udTLzM7my45X2+W2P5h9lV8YuogKjnKYPE3EF/bqWOq9zZ7DZIDMTNm6Erl3d3SIRkaoJ8w9z6FktNbbnWKeyC1peQPdG3TFwnLZrUOtBpOSkEBNUNnl0QXEBvt6+TjMirD+0nkPHDnFJ20vsZb+l/MbMH2bSPqo9sy6aZS+fvmI6yTnJjO4+2j5c4sixI/x04CdiQ2Lp3qi7vW5RSZFTWBepjN4pUud5e8O558LixbBihUKqiNRtVi+rQ89qqeHthzuV9Wvej37N+9nHzJa6qctNpOWm0Sikkb3Mx8uH5rbmTuNt92Xu49CxQw5lOzN2Mmf9HNpHtXcIqeOXjmdf5j4eS3yMhFhzieD9mftZsmMJTUKbOCzve+TYEaxeVmz+NgXbekr/6lIvJCbCkiWQl+fuloiIeJ4TQ2Cfpn2c6vRo3IMejXs4lT9y3iMczT/qMATA5mfj3KbnOgXa7IJsCooLHMa/7svcx5e/f0n7qPYOIfXpH5/mj/Q/eKLvE/Rs3BMwpwj7YOMHtAhr4bCk7m8pv1FcUkzzsOb21cqk9lNIlXqhY0d47z0IC3N3S0RE6pYTVxoD6BjTkY4xHZ3KXxz8IlkFWQ43izUObcyVHa6ssPf3xMUZjhw7wi+HfiGv0LHHYe76ufyR/gcT+060B+nfUn7jn6v/SZvINg4LRazcu5Kcwhy6xna1D30oLikGwNvLcdEIcS+FVKkXLBYFVBERdwv0CSTQJ9ChrHlYc/vqXuU9P/B5p+Vw20S24b5e9xHkG+RQ3iCoAXmFeQ7jdtPz0jmcc9gp/C7cttAeaEtD6paULTz23WO0jWzLswOeLau7dSEZ+Rn0b9Hf3sb8onxSc1Ox+dnUa+tiCqlS76Snm3On+vr+fV0REXGfE+d6jQmK4YKWFzjVe/i8h53KujTswqwLZzn1jp7d4GwiAiIcbiDLKsgCzHG35a3Yu4IdGTvoHNPZHlK3p27n8WWP0zS0Ka9d/Jq97tz1czl87DAj2o+w30CWXZDNbym/EREQQZvINlU4c9dKzU0lMz+z0v02f1uFSxfXNIVUqVdeeAG++w7Gj4fzznN3a0RExFWCfYMrXJyh/FjWUn2a9uGDyz5wuoFsQKsBdD7W2WFs7fHi4wT5BGHzc5wjd3PyZnZm7GRgq4H2sp0ZO5m+cjpxoXG8evGr9vJZP85id8Zubu16q32hiPS8dL7f8z3RQdGcF1f2C+p48XF8vHyqbXGGwuJC7lx0J7sydlVap2V4S+aNmIePt0+ldWqCQqrUK+HhYBjmxP4KqSIiAubY1xMXZgAY3GawU1mPxj34+PKPnYYi3BB/Aym5KfZ5ZcG8Ia1NRBtig2Md6h7MPsif2X86TBO2L3Mfb//6Ns1szRxC6rTvp7EpeRPj+oyzlx8+dphPf/uUBsENuLzD5fa6h48dBsylfE9cHrh8mxqHNGbD4Q00szVz2r83cy+NQxp7xIwK7m+BSA1KTIRPPoF16yA3FwID//45IiIiJzqxZ7OipXM7xXTi+YHPO5U/1Och0vPSHcbihviG0K9ZP6cVyEoXZyi/kMPhY4f5audXNLM1cwipr/70Kr8e+ZUHej9AUoskwJzi6/V1r9MopBFje47FYrEwsstIlu5aytH8o8QExdh7TDPzM7H52RjZZaRHLKurkCr1SosW0LgxHDgAa9dCUpK7WyQiIvVNRauNtYpoxYN9HnSq+8yAZ8guyCbYN9heFhMUw3WdryPIx/EGMm8vb6fFGdLy0tiYvNE+7hYgoWECQb5BbE3Ziq+3L9FB0RiGQXJOMgNbDSShYUJ1neoZUUiVesViMXtTP/7YvOSvkCoiIp7M19vXaYaCRiGNuLrT1U51J/ebDOAwFCHOFsdDfR5yuHxvsVi4oPkF7Dm6h+PFxwGzxzbQJ9BjelEBvNzdAJGa1rev+XXDBsjOdm9bREREqlv5kBkREEHfZn2dFmiYeeFMru10LbmFufZe1MS4RI/pRQWFVKmHmjaF5s2hqAjWrHF3a0RERGpe6djUQJ9A9mft97heVNDlfqmnrrwSCgqgd293t0RERMQ9EhomkBiXyMJtC7ms3WUe1YsKCqlSTyUmursFIiIi7mWxWBiVMIqcwhxGJYzyqF5UAItx4kRftVhWVhY2m43MzExCQ0P//gkiIiIiUqNONa9pTKrUW8eOwb//DbNnu7slIiIiciKFVKm38vNh7lxYsgTS0tzdGhERESlPIVXqragoaN/eXCb1hx/c3RoREREpTyFV6rXSOVNXrnRvO0RERMSRQqrUa+eea65CtX07HDni7taIiIhIKYVUqdfCw6FzZ/N79aaKiIh4DoVUqfdK50xVSBUREfEcCqlS7/XpA/7+0LAhHD/u7taIiIgIaMUpEUJD4cMPwdfX3S0RERGRUupJFUEBVURExNMopIqUc/AgpKe7uxUiIiKikCryl7fegjvugP/8x90tEREREY1JFQFSUiAwEHJyzJB6zjnm/KmlQkMhOtp97RMREalvFFKl3ktJgWuvNb/u2WMuk7ppE/j5ldWJjISPPlJQFRERqSm63C/1XlYWpKVBQIAZRq1WKCmBsDBz8/Mz92dlubulIiIi9Yd6UkX+EhAAsbFw7Ji5BQWV7SsocF+7RERE6iP1pIqUY7OBt7c5qX9ysuM+w3BPm0REROoj9aSKlOPlBeHhkJoKPj5l5Tk5cPfd0KaN2dvasGHZ10aNICrKfK6IiIhUD4VUkRPExZmX/gMDy8oKC81L/rt3m9uJHnkEzj3X/H73bti40QyxsbHQoIEWCxAREakqhVSRE1itZrgsz2aDqVPNm6gOH4ZDh8q2I0cc6//6K7z9tuPzIyPLQutll0HTpma5YThOdSUiIiImhVSRv+TlVV5usZgBs1Ur5/0lJY5BMzbW7FU9dMgMtLm55uwAaWmweTMMGVJW94sv4F//ch4+UPo4PFwhVkRE6ieFVKn3QkPNns60tMrv4o+MNOtV5MSxqL17mxuYPaXZ2WW9rocPQ+PGZXUPHTL3Z2fDH384H/v5581xsABbtsDevWUBNjra7PUVERGpi/QrTuq96Ghzov6TzYN6uitOWSzmc0NDoW1b5/033QSDBpUF2PLDCFJSzEBa6scf4csvyx57eUFMTFloveYas+cVNIxARERqP4VUEcwA6o7VpPz9oXlzcztRUZFjT2mLFtCrV1mgPX7c/Hr4sDkO9vrry+q+9RasWFEWYE+ckSA0VCFWREQ8m0KqiIc68VL+RReZG5g9pRkZjr2uISFldQ8eNPdnZMDWrc7H/uAD82YwgJ9/NuuVhtioKAVYERFxP4VUkVrIYoGICHPr2NF5/4MPmkG1dAhB+aEEeXmO42u//hrWri17bLWa02aV9sDedFPZFFoaRiAiIjVFIVWkDgoKMm+4Kr3pqryiIseg2a6dWXbokLnKVlERHDhgbv7+cNttZXWfegp+/70swJ44jKD8UrIiIiJnQiFVpJ45cRjB5ZebG5jTaaWmlvXClk6/VergQXN/aips2uR4HF9f+OSTsvorVpiLIJSG2bAw9cKKiMipc1lI/e677/jggw9YtWoV+/fvJywsjO7duzNx4kS6devmUHf9+vWMHz+eNWvWYLVa6d+/P88++ywtW7Z0VfNEpAKlMwbExFS8f/r0yocRnBhCP/nEcXUuf3+zt7VhQ3NVrxtuKNunYQQiIjUnJcU1M9pUN5eF1NmzZ5OWlsa9995Lhw4dSElJ4bnnnqN37958/fXX9O/fH4Bt27bRr18/unTpwvz588nPz2fixIkkJiby66+/Eu0JPyURAcybrWw2aN/eeV9hoePj+Hjzg670xq78fNizx9wOHHAMqePGmXPFlh8+UH44gZaVFRGpHikpcO215tzglYmMNKdmdHcEsxiGYbjiwMnJycSc0B1z7NgxWrduTadOnfjvf/8LwJVXXsmyZcvYuXMnoX/dzbF3717atGnD/fffz9NPP33Kr5mVlYXNZiMzM9N+LBFxv6Iic/nY0p5XX18YMMDcZxhw9dXmylwVadIEZs8ue/zNN2W9srGxjrMaiIjIye3cCVdcYS7zHRDgvD8vz1zYZsGCildZrA6nmtdc1pN6YkAFCA4OpkOHDuzfvx+AoqIiFi1axI033ujQyGbNmpGUlMTChQurFFJFxDNZreZKW+VX2yrv1VedFzMofVx+QQOAt9+GnJyyx0FBZT2u7drBP/5Rtk/DCEREKhYQUHaza0mJ4+qJla2+WNNq9MapzMxM1q9fb7/Uv3PnTvLy8oiPj3eqGx8fz9KlS8nPz8ff378mmykiNchiMedmjYqCTp0c9xmG4zCCoiJzydnSAJuebgbWHTvMLS/PMaSOGgWBgY4zEDRqZH5t0EDLyorUd4ZhBrTiYuctKMjsbQTzcyYlpeJ6xcXmOPuoKLNuejr873+VHzc+Hlq3NuseOQKLF5vlFdU/7zzo2dOse/AgvPGG4/7yzxk4EC6+2Kx76BA88ojz8UpKzLGoaWnmfQRgLgxz9Gjl9yK4U41+RI8dO5acnBwee+wxANL+GhARERHhVDciIgLDMMjIyCA2NrbC4xUUFFBQLu5nnWwUsIjUOhaL43hUqxXuu6/scUFB2apbhw6V/ZIA85dKWpq5/XXxxkHXrjBlStnjL74wn18aaCu6DFYVteXGBBEww5phmEHGai27ApGba27lw1BRUdnjZs3K/o8ePGj+PzyxTunWu3fZ8Jxt22Dz5sqD3CWXmH9IAvzyCyxbVnmQGzWqLPStWgXz5pW14cTn3H8/dO9u1l2+HJ57rvKfyUMPQd++5vf/+x/MnFl53bvuMkMiwN698Pzzlde97bay9qanw2efVV63SZOykJqfD+vXV143Pb3se8NwfFxe6b91KYvF8bEnqbGQ+sQTT/Dhhx/y8ssvO93dbznJ9biT7Zs5cyZTyv+WEZF6xc/P/CXZrJnzvsBAePPNUxtGkJtr1i3PZiu7eSshAZKSyvb93TCC2nRjglSsopBVuhmGY6/Tn3+afxRVVB+gR4+yuuvXO/bIlX+OYcCVV5bVXboUdu2qvOds3LiyqwHz58OGDRW3t6TEDGOll3bnzoVvv624rQDvvmu+P8F8j/7735X/nF5/vWwYz7ffmu2oTMuWZSF140Z4//3K6557bllIPXgQvv++8rrl/xjMyTFvzqzM8eNl35e/vH0iLy/Hn4m/P4SHg7e3uc/b23ErPzbeZjM/M8rvL/+cJk3K6kZGwqWXmv+OpXXKf9+hQ1nd6Gjzj/QT65Ru5T/ToqPhxRcdj1najn374Oaby+r6+JT9rD1NjYTUKVOmMH36dGbMmMFdd91lL4/8639BWgWf5Onp6VgsFsJK+6MrMGHCBB544AH746ysLJo2bVp9DReRWstiKZvyqksXx30nDiM4fhzOP98MsAcPmjMNZGaa27Zt5i+o0pCalwcjRzrOQlD++6iosstpJ7sxIS3NrOfJIbW0x6U0sFksZZc/DcNc/KGiAFVUZAaiuLiyY/38s/kzr6h+eDj06lVWd+FC59670i06umxeXzBvqjt6tKzXrPxxY2Kg3K8IpkwxZ5YorVO+py0qCl55pazuuHHmDSYVCQtzDFgvvwy//VZxXX9/8waUUl98YfYMVsRicQyp69aZPYOVuf/+spB64IDZM1mZoqKy748fh2PHKq9bPpxVFohKt/J/rEVFmT2ElQW58v8XWrSACy4oC08nPqc0JIO5qt6tt5r7K6rfvHlZ3a5dzanyytcp/5zyx+3d21wiuqIweeIfoV27wnvvVf4zK69lS5g69dTqxsTALbecWt2QEPNndip8fMx2VCQz8+QB3ZO4PKROmTKFyZMnM3nyZB599FGHfa1atSIgIIBNJ84KDmzatInWrVufdDyqn58ffqWfmCIip+jEYQRhYWYoKZWT4zgXbPk7XEsXOdi923Ee2FKDBpkbmIEuO9v86u1dFvpKQ8KBA2XHzs6GlSsr7gUrLjZvCiu9CJWdbYakysbHde0KQ4aYdY8dM39hVlb3nHPKfknm55tTg5UPh+Wddx48/HDZ41tvrfxn3L07TJpU9vippxx7scrr1MkxpH76qfmLtCJt2jiG1HXrzLBckROHWyQnm/+mFQkMdHzs7e1cpzTA+Pg4lkdGmj1RFYWdE39FtW3rGJpOrF++l75PH2jatOJwWLqVGjLE7LGtrF751eCuvtq8nH7ia5eG0fI/i1GjzO1UDB5sbqeiRw/HHuaTadmy8sB1oshIxyB6Mr6+9Xt6u7y8qpW7g0tD6rRp05g8eTKPP/44k8p/WpW+uNXKsGHD+Oyzz5g1axYhf/WX79u3j2XLlnH//fe7snkiIhUKCjLDY0XTrzRtavbelR86UNoDe+SI4yW3ggLz0tqJiorMbd26sjFvR486TrV1oksvLQupBQWwZEnldcPDy743DNi6tfK6R4+Wfe/tbQbVypQPrRaL2TPm5VVxr1n5NoAZsgsLK+4JO3G4Rv/+5jlWFORODCDXX2+2uaJgdmLwvP9+sw0V1T0xeE6fbp5jaRu9vCof4jF+fOU/sxNdc82p1z3//FOv27atuZ2KiAhzk/opNNT8f5SWVvld/JGRZj13c9k8qc899xzjxo1j0KBBFQbU3r17A+Zk/j169KBr16488sgj9sn809PTqzyZv+ZJFRF3Kr2EvH+/OQ+hvz9kZJi/CEp7yCwWs05Bgdm7eNNN5nPT0807dyvrCevc2exZA/NS+BdfVH4JtmnTsrFspWG4skuwNltZsDYMM2hXNpautLdNRGo3d9/Yeap5zWUhtV+/fnx/kpHO5V/2l19+4eGHH2b16tUOy6K2quIssgqpIuIJSifLDgtzvMxaKifH7MF05WTZIiKeyu2T+S9fvvyU63br1s2+ApWIiIiIiC7ciIi4SG24MUFExFMppIqIVLPadGOCiIinUkgVEalm0dHmJOhacUpE5PQppIqIuEB0tEKoiMiZqCVrDoiIiIhIfaKQKiIiIiIeRyFVRERERDyOQqqIiIiIeByFVBERERHxOAqpIiIiIuJxFFJFRERExOMopIqIiIiIx1FIFRERERGPo5AqIiIiIh5HIVVEREREPI5CqoiIiIh4HIVUEREREfE4CqkiIiIi4nEUUkVERETE4yikioiIiIjHUUgVEREREY+jkCoiIiIiHkchVUREREQ8jkKqiIiIiHgchVQRERER8TgKqSIiIiLicRRSRURERMTjKKSKiIiIiMdRSBURERERj6OQKiIiIiIeRyFVRERERDyOQqqIiIiIeByFVBERERHxOAqpIiIiIuJxFFJFRERExOMopIqIiIiIx1FIFRERERGPo5AqIiIiIh5HIVVEREREPI5CqoiIiIh4HIVUEREREfE4CqkiIiIi4nEUUkVERETE4yikioiIiIjHUUgVEREREY+jkCoiIiIiHkchVUREREQ8jkKqiIiIiHgchVQRERER8TgKqSIiIiLicRRSRURERMTjKKSKiIiIiMdRSBURERERj6OQKiIiIiIeRyFVRERERDyOQqqIiIiIeByFVBERERHxOAqpIiIiIuJxFFJFRERExOMopIqIiIiIx1FIFRERERGPY3V3A0RERETEDUpKYMsWyMiA8HDo2BG8PKf/0iNC6rFjx3j88ceZP38+6enptGvXjkceeYSrr77a3U2rnIf/w1Yrnau7WyW1md5TIuKJVq2CV16BrVuhoAD8/KB9e7jrLujTx92tAzwkpA4fPpyff/6Zp556irPOOouPPvqIa665hpKSEq699lp3N89ZLfiHrTY617p5rlIz9J4SEU+0ahWMGwfp6RAbCwEBkJcHGzaY5c8+6xGfURbDMAx3NmDx4sVcfPHF9mBaasCAAWzZsoV9+/bh7e19SsfKysrCZrORmZlJaGioaxpc2T/s4cNmL4mH/MNWC51r3TxXqRl6T4mIJyopgeuvh/XroXVrsFjK9hkG7NwJXbvC+++77KrPqeY1t4fU2267jY8//piMjAys1rKO3Xnz5nHttdfy448/0ucUP8hdHlLL/8M2aQJpaWX7DAOSk6F5c7jzTqgsWJd/M1T2/ak+p6p1qvL6JSXwwguwezc0bOj8Jj58GFq1gvvuq/2XLktK4J//hF276v65Ss04lfdUy5Zl76kz/Rj+u+efbL87X/tMj+/utp/J67vz53amx3fnz83Vx/fktlfXax84AO+9Z/7h7OdXtj88HGw2OHYMjh4163TufGavWYlTzWtuv9y/efNm2rdv7xBQAeLj4+37KwupBQUFFBQU2B9nZWW5rqFgjivbutXsFSkshIMHHfcXFZl13noLXNWTW1OysmDjRrBaYf9+5/1FRfDrr2aQrQvn+r//1Y9zlZpxKu+p//0PXnxR7ykRqVlpaWYILShw/APaz88MqQEBcOSIOY7ezdweUtPS0mjZsqVTeUREhH1/ZWbOnMmUKVNc1jYnGRnmP2pAgBlSGzRw3F9SYv7j9+gBbdqYZeX/8qnsr6BTqXOyeqfznL+rs3MnbNsGERFlvYfl65WUmD+Pjh3NXsbarPRcw8Mr7imtS+cqNaOi91T5XwYlJeYwgE6dTv09dbIrJKey313Hro7j19Vzc/fP9UyO7+6219X3hKuPbbHAvn1mJ1tQEPj7l+0LCTG/5uWZgTU8/PTbWU3cHlIBLCf5oZ5s34QJE3jggQfsj7OysmjatGm1ts1BeLj5D5eXB8HB0KyZ4/5jx8DHB0aOdFkXeY3ZtMkcUxcWZp7riY4dM8P66NF141xXr64f5yo141TeU/7+5tAgvadEpCaVlJhXBzdsMK8MVzQcqWtXs2PGzdw+wC4yMrLC3tL09HSgrEe1In5+foSGhjpsLtWxo3ln7uHDzr2Spf+wHTp4xD/sGdO5murauUrN0HtKRDyVl5c5w0h4uHnV59gxKC42v+7caZaPHesR92C4vQWdO3dm69atFBUVOZRv2rQJgE6dOrmjWRWrRf+wZ0znWjfPVWqG3lMi4sn69DFnGElIMMen7tljfu3a1aNmHnH73f1LlixhyJAhfPzxx1x11VX28sGDB7Nx40bPm4IKKp77sEMH85eOh/zDVhuda908V6kZek+JiCdz02IjtWYKKjDnRF23bh1PP/00rVu3Zt68ebz55pt88MEHXHfddad8nBoLqVC/VpHRubq7VVKb6T0lIuKgVoXUY8eO8dhjjzksizphwoQqL4taoyFVRERERKqsVoXU6qKQKiIiIuLZTjWv6ZqTiIiIiHgchVQRERER8TgKqSIiIiLicRRSRURERMTjKKSKiIiIiMexursB1al0ooKsrCw3t0REREREKlKa0/5ugqk6FVKzs7MBaNq0qZtbIiIiIiInk52djc1mq3R/nZontaSkhIMHDxISEoLFYnH562VlZdG0aVP2799f5+dl1bmKnD69p0TEU7nj88kwDLKzs2nUqBFeJ1mBr071pHp5edGkSZMaf93Q0NB684tH5ypy+vSeEhFPVdOfTyfrQS2lG6dERERExOMopIqIiIiIx1FIPQN+fn5MmjQJPz8/dzfF5XSuIqdP7ykR8VSe/PlUp26cEhEREZG6QT2pIiIiIuJxFFJFRERExOMopIqIiIiIx1FIFRERERGPo5B6EtnZ2YwfP54BAwYQHR2NxWJh8uTJFdZdv349F154IcHBwYSFhTF8+HB27dpVsw0+A7/++isXX3wxcXFxBAQEEBERwTnnnMMHH3zgVLe2n+vy5cuxWCwVbmvWrHGoW9vPVaqfqz4XXn75Zdq1a4efnx8tWrRgypQpFBYWuvBMRKSu+e6777j55ptp164dQUFBNG7cmH/84x/88ssvTnVrw+eTQupJpKWlMWfOHAoKCrj00ksrrbdt2zb69evH8ePHmT9/Pm+//Ta///47iYmJpKSk1FyDz8DRo0dp2rQpTz75JIsXL+a9996jefPm3HDDDUyfPt1ery6ca6knn3yS1atXO2ydOnWy769L5yrVxxWfCzNmzODee+9l+PDhfP3114wZM4Ynn3ySsWPHuvhsRKQumT17Nnv27OHee+9l8eLFvPjiiyQnJ9O7d2++++47e71a8/lkSKVKSkqMkpISwzAMIyUlxQCMSZMmOdW74oorjKioKCMzM9NetmfPHsPHx8cYP358TTXXJXr16mU0bdrU/rgunOuyZcsMwFiwYMFJ69WFc5XqV92fC6mpqYa/v79x++23Ozx/xowZhsViMbZs2eKaExGROufIkSNOZdnZ2UaDBg2MCy64wF5WWz6f1JN6EqWXgE+mqKiIRYsWMWLECIc1b5s1a0ZSUhILFy50dTNdKioqCqvVCtT9cy2vPp2rVE11fy589dVX5OfnM2rUKIdjjBo1CsMw+Pzzz6u1/SJSd8XExDiVBQcH06FDB/bv3w/Urs8nhdQztHPnTvLy8oiPj3faFx8fz44dO8jPz3dDy05PSUkJRUVFpKSk8Nprr/H111/z8MMPA3XvXMeOHYvVaiU0NJSBAwfyww8/2PfVtXOVmlWV98/mzZsB6Ny5s0O92NhYoqKi7PtFRE5HZmYm69evp2PHjkDt+nxSSD1DaWlpAERERDjti4iIwDAMMjIyarpZp23MmDH4+PgQExPD/fffz0svvcQdd9wB1J1ztdls3HvvvbzxxhssW7aMF198kf3799OvXz++/vproO6cq7hHVd4/aWlp+Pn5ERQUVGHd0mOJiJyOsWPHkpOTw2OPPQbUrs8nq0uPXo+c7PLf310a9CSPPvoot956K8nJyXz55Zfcdddd5OTkMG7cOHud2n6uCQkJJCQk2B8nJiZy2WWX0blzZ8aPH8/AgQPt+2r7uYp7ner7R+8zEXGFJ554gg8//JCXX36Zbt26OeyrDZ9PCqlnKDIyEqDCvybS09OxWCyEhYXVcKtOX1xcHHFxcQAMGTIEgAkTJjBy5Mg6d67lhYWFMXToUF5//XXy8vLq9LmK61Xl/RMZGUl+fj65ubkEBgY61T3xF4uIyKmYMmUK06dPZ8aMGdx111328tr0+aTL/WeoVatWBAQEsGnTJqd9mzZtonXr1vj7+7uhZdWjZ8+eFBUVsWvXrjp/roZhAOZfhnX9XMW1qvL+KR3rdWLdw4cPk5qa6jAtmojIqZgyZQqTJ09m8uTJPProow77atPnk0LqGbJarQwbNozPPvuM7Oxse/m+fftYtmwZw4cPd2PrztyyZcvw8vKiZcuWdfpcMzIyWLRoEV26dMHf379On6u4XlXeP4MGDcLf3593333X4RjvvvsuFovlpHOxioicaNq0aUyePJnHH3+cSZMmOe2vVZ9PLp3gqg5YvHixsWDBAuPtt982AOOKK64wFixYYCxYsMDIyckxDMMwtm7dagQHBxt9+/Y1Fi9ebHz22WdGp06djEaNGhnJycluPoNTc9tttxkPPvig8a9//ctYvny58cknnxhXXXWVARgPPfSQvV5dONdrrrnGePjhh40FCxYYy5YtM+bMmWO0bdvWsFqtxtKlS+316sK5imtU9+fC9OnTDYvFYjz66KPG8uXLjWeeecbw8/MzbrvtNnecnojUUs8++6wBGIMGDTJWr17ttJWqLZ9PCql/o1mzZgZQ4bZ79257vXXr1hkXXHCBERgYaISGhhqXXnqpsWPHDvc1vIrefvttIzEx0YiKijKsVqsRFhZmnH/++cb777/vVLe2n+vMmTONLl26GDabzfD29jaio6ONyy67zPjpp5+c6tb2cxXXcMXnwosvvmicddZZhq+vrxEXF2dMmjTJOH78eA2dkYjUBeeff36ln00n9kvWhs8ni2H8NRBPRERERMRDaEyqiIiIiHgchVQRERER8TgKqSIiIiLicRRSRURERMTjKKSKiIiIiMdRSBURERERj6OQKiIiIiIeRyFVRERERDyOQqqIiIiIeByFVBERERHxOAqpIiIiIuJx/h8y7FJaDxO7zgAAAABJRU5ErkJggg==",
+      "text/plain": [
+       "<Figure size 800x500 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Top5 invalidity\n",
+    "CompoundT5 = [79.28, 71.2, 24.4, 18.9, 20.2]\n",
+    "ReactionT5 = [0.08, 0.06, 0.06, 0.06, 0.1]\n",
+    "T5Chem = [56.56, 65.17, 43.11, 36.95, 30.74]\n",
+    "\n",
+    "\n",
+    "# plot\n",
+    "import matplotlib.pyplot as plt\n",
+    "fig, ax = plt.subplots(1, figsize=(8, 5))\n",
+    "\n",
+    "\n",
+    "ax.plot([10,30,50,100,200], ReactionT5, \"o-\", label='ReactionT5', color='red', alpha=0.7)\n",
+    "ax.plot([10,30,50,100,200], CompoundT5, \"s--\", label='CompoundT5', color='blue', alpha=0.7)\n",
+    "ax.plot([10,30,50,100,200], T5Chem, \"v:\", label='T5Chem', color='green', alpha=0.7)\n",
+    "\n",
+    "\n",
+    "# plt.ylim(0, 35)\n",
+    "ax.set_xticks([10,30,50,100,200])\n",
+    "ax.set_xticklabels([10,30,50,100,200], fontsize=12)\n",
+    "ax.set_yticks([0, 20, 40, 60, 80, 100])\n",
+    "ax.set_yticklabels([0, 20, 40, 60, 80, 100], fontsize=12)\n",
+    "# plt.tight_layout()\n",
+    "ax.legend(loc=\"best\", fontsize=12)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "reactiont5",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  },
+  "varInspector": {
+   "cols": {
+    "lenName": 16,
+    "lenType": 16,
+    "lenVar": 40
+   },
+   "kernels_config": {
+    "python": {
+     "delete_cmd_postfix": "",
+     "delete_cmd_prefix": "del ",
+     "library": "var_list.py",
+     "varRefreshCmd": "print(var_dic_list())"
+    },
+    "r": {
+     "delete_cmd_postfix": ") ",
+     "delete_cmd_prefix": "rm(",
+     "library": "var_list.r",
+     "varRefreshCmd": "cat(var_dic_list()) "
+    }
+   },
+   "types_to_exclude": [
+    "module",
+    "function",
+    "builtin_function_or_method",
+    "instance",
+    "_Feature"
+   ],
+   "window_display": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

task_retrosynthesis/calculate_accuracy.py

@@ -0,0 +1,134 @@
+import argparse
+import os
+import sys
+import warnings
+
+import pandas as pd
+import rdkit
+from rdkit import Chem
+from transformers import AutoTokenizer
+
+rdkit.RDLogger.DisableLog("rdApp.*")
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from utils import canonicalize, seed_everything
+
+warnings.filterwarnings("ignore")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Script for reaction retrosynthesis prediction."
+    )
+    parser.add_argument(
+        "--input_data",
+        type=str,
+        required=True,
+        help="Path to the input data.",
+    )
+    parser.add_argument(
+        "--target_data",
+        type=str,
+        required=True,
+        help="Path to the target data.",
+    )
+    parser.add_argument(
+        "--target_col",
+        type=str,
+        required=True,
+        help="Name of target column.",
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        type=str,
+        default="sagawa/ReactionT5v2-retrosynthesis",
+        help="Name or path of the finetuned model for prediction. Can be a local model or one from Hugging Face.",
+    )
+    parser.add_argument(
+        "--num_beams", type=int, default=5, help="Number of beams used for beam search."
+    )
+    parser.add_argument(
+        "--seed", type=int, default=42, help="Seed for reproducibility."
+    )
+    return parser.parse_args()
+
+
+def remove_space(row):
+    for i in range(5):
+        row[f"{i}th"] = row[f"{i}th"].replace(" ", "")
+    return row
+
+
+if __name__ == "__main__":
+    CFG = parse_args()
+
+    seed_everything(seed=CFG.seed)
+
+    tokenizer = AutoTokenizer.from_pretrained(
+        os.path.abspath(CFG.model_name_or_path)
+        if os.path.exists(CFG.model_name_or_path)
+        else CFG.model_name_or_path,
+        return_tensors="pt",
+    )
+
+    df = pd.read_csv(CFG.input_data)
+    df[[f"{i}th" for i in range(CFG.num_beams)]] = df[
+        [f"{i}th" for i in range(CFG.num_beams)]
+    ].fillna(" ")
+    df["target"] = pd.read_csv(CFG.target_data)[CFG.target_col].values
+    df = df.apply(remove_space, axis=1)
+
+    top_k_invalidity = CFG.num_beams
+
+    top1, top2, top3, top5 = [], [], [], []
+    invalidity = []
+
+    for idx, row in df.iterrows():
+        target = canonicalize(row["target"])
+        if canonicalize(row["0th"]) == target:
+            top1.append(1)
+            top2.append(1)
+            top3.append(1)
+            top5.append(1)
+        elif canonicalize(row["1th"]) == target:
+            top1.append(0)
+            top2.append(1)
+            top3.append(1)
+            top5.append(1)
+        elif canonicalize(row["2th"]) == target:
+            top1.append(0)
+            top2.append(0)
+            top3.append(1)
+            top5.append(1)
+        elif canonicalize(row["3th"]) == target:
+            top1.append(0)
+            top2.append(0)
+            top3.append(0)
+            top5.append(1)
+        elif canonicalize(row["4th"]) == target:
+            top1.append(0)
+            top2.append(0)
+            top3.append(0)
+            top5.append(1)
+        else:
+            top1.append(0)
+            top2.append(0)
+            top3.append(0)
+            top5.append(0)
+
+        input_compound = row["input"]
+        output = [row[f"{i}th"] for i in range(top_k_invalidity)]
+        inval_score = 0
+        for ith, out in enumerate(output):
+            mol = Chem.MolFromSmiles(out.rstrip("."))
+            if not isinstance(mol, Chem.rdchem.Mol):
+                inval_score += 1
+        invalidity.append(inval_score)
+    print(CFG.input_data)
+    print(f"Top 1 accuracy: {sum(top1) / len(top1)}")
+    print(f"Top 2 accuracy: {sum(top2) / len(top2)}")
+    print(f"Top 3 accuracy: {sum(top3) / len(top3)}")
+    print(f"Top 5 accuracy: {sum(top5) / len(top5)}")
+    print(
+        f"Top {top_k_invalidity} Invalidity: {sum(invalidity) / (len(invalidity) * top_k_invalidity) * 100}"
+    )

task_retrosynthesis/finetune.py

@@ -0,0 +1,278 @@
+import argparse
+import os
+import sys
+import warnings
+
+import datasets
+import pandas as pd
+import torch
+from datasets import Dataset, DatasetDict
+from transformers import (
+    AutoModelForSeq2SeqLM,
+    AutoTokenizer,
+    DataCollatorForSeq2Seq,
+    EarlyStoppingCallback,
+    Seq2SeqTrainer,
+    Seq2SeqTrainingArguments,
+)
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from train import preprocess_df
+from utils import filter_out, get_accuracy_score, preprocess_dataset, seed_everything
+
+# Suppress warnings and disable progress bars
+warnings.filterwarnings("ignore")
+datasets.utils.logging.disable_progress_bar()
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--train_data_path",
+        type=str,
+        required=True,
+        help="The path to data used for training. CSV file that contains ['REACTANT', 'PRODUCT'] columns is expected.",
+    )
+    parser.add_argument(
+        "--valid_data_path",
+        type=str,
+        required=True,
+        help="The path to data used for validation. CSV file that contains ['REACTANT', 'PRODUCT'] columns is expected.",
+    )
+    parser.add_argument(
+        "--similar_reaction_data_path",
+        type=str,
+        required=False,
+        help="Path to similar data CSV.",
+    )
+    parser.add_argument(
+        "--output_dir", type=str, default="t5", help="Path of the output directory."
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        type=str,
+        required=False,
+        default="sagawa/ReactionT5v2-retrosynthesis",
+        help="The name of a pretrained model or path to a model which you want to finetune on your dataset. You can use your local models or models uploaded to hugging face.",
+    )
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        default=False,
+        required=False,
+        help="Use debug mode.",
+    )
+    parser.add_argument(
+        "--epochs",
+        type=int,
+        default=20,
+        required=False,
+        help="Number of epochs for training.",
+    )
+    parser.add_argument("--lr", type=float, default=2e-5, help="Learning rate.")
+    parser.add_argument("--batch_size", type=int, default=32, help="Batch size.")
+    parser.add_argument(
+        "--input_max_length",
+        type=int,
+        default=150,
+        required=False,
+        help="Max input token length.",
+    )
+    parser.add_argument(
+        "--target_max_length",
+        type=int,
+        default=150,
+        required=False,
+        help="Max target token length.",
+    )
+    parser.add_argument(
+        "--eval_beams",
+        type=int,
+        default=5,
+        help="Number of beams used for beam search during evaluation.",
+    )
+    parser.add_argument(
+        "--target_column",
+        type=str,
+        default="REACTANT",
+        help="Target column name.",
+    )
+    parser.add_argument(
+        "--weight_decay",
+        type=float,
+        default=0.01,
+        required=False,
+        help="weight_decay used for trainer",
+    )
+    parser.add_argument(
+        "--evaluation_strategy",
+        type=str,
+        default="epoch",
+        required=False,
+        help="Evaluation strategy used during training. Select from 'no', 'steps', or 'epoch'. If you select 'steps', also give --eval_steps.",
+    )
+    parser.add_argument(
+        "--eval_steps",
+        type=int,
+        required=False,
+        help="Number of update steps between two evaluations",
+    )
+    parser.add_argument(
+        "--save_strategy",
+        type=str,
+        default="epoch",
+        required=False,
+        help="Save strategy used during training. Select from 'no', 'steps', or 'epoch'. If you select 'steps', also give --save_steps.",
+    )
+    parser.add_argument(
+        "--save_steps",
+        type=int,
+        required=False,
+        default=500,
+        help="Number of steps between two saving",
+    )
+    parser.add_argument(
+        "--logging_strategy",
+        type=str,
+        default="epoch",
+        required=False,
+        help="Logging strategy used during training. Select from 'no', 'steps', or 'epoch'. If you select 'steps', also give --logging_steps.",
+    )
+    parser.add_argument(
+        "--logging_steps",
+        type=int,
+        required=False,
+        default=500,
+        help="Number of steps between two logging",
+    )
+    parser.add_argument(
+        "--save_total_limit",
+        type=int,
+        default=3,
+        required=False,
+        help="Limit of the number of saved checkpoints. If limit is reached, the oldest checkpoint will be deleted.",
+    )
+    parser.add_argument(
+        "--fp16",
+        action="store_true",
+        default=False,
+        required=False,
+        help="Use fp16 during training",
+    )
+    parser.add_argument(
+        "--disable_tqdm",
+        action="store_true",
+        default=False,
+        required=False,
+        help="Disable tqdm during training",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=42,
+        required=False,
+        help="Set seed for reproducibility.",
+    )
+    parser.add_argument(
+        "--sampling_num",
+        type=int,
+        default=-1,
+        help="Number of samples used for training. If you want to use all samples, set -1.",
+    )
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    CFG = parse_args()
+    CFG.disable_tqdm = True
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    seed_everything(seed=CFG.seed)
+
+    train = preprocess_df(
+        filter_out(pd.read_csv(CFG.train_data_path), ["REACTANT", "PRODUCT"])
+    )
+    valid = preprocess_df(
+        filter_out(pd.read_csv(CFG.valid_data_path), ["REACTANT", "PRODUCT"])
+    )
+
+    if CFG.sampling_num > 0:
+        train = train.sample(n=CFG.sampling_num, random_state=CFG.seed).reset_index(
+            drop=True
+        )
+
+    if CFG.similar_reaction_data_path:
+        similar = preprocess_df(
+            filter_out(
+                pd.read_csv(CFG.similar_reaction_data_path), ["REACTANT", "PRODUCT"]
+            )
+        )
+        print(len(train))
+        train = pd.concat([train, similar], ignore_index=True)
+        print(len(train))
+
+    dataset = DatasetDict(
+        {
+            "train": Dataset.from_pandas(train[["input", "REACTANT"]]),
+            "validation": Dataset.from_pandas(valid[["input", "REACTANT"]]),
+        }
+    )
+
+    # load tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(
+        os.path.abspath(CFG.model_name_or_path)
+        if os.path.exists(CFG.model_name_or_path)
+        else CFG.model_name_or_path,
+        return_tensors="pt",
+    )
+    CFG.tokenizer = tokenizer
+
+    # load model
+    model = AutoModelForSeq2SeqLM.from_pretrained(
+        os.path.abspath(CFG.model_name_or_path) if os.path.exists(CFG.model_name_or_path) else CFG.model_name_or_path
+    )
+    tokenized_datasets = dataset.map(
+        lambda examples: preprocess_dataset(examples, CFG),
+        batched=True,
+        remove_columns=dataset["train"].column_names,
+        load_from_cache_file=False,
+    )
+
+    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
+
+    args = Seq2SeqTrainingArguments(
+        CFG.output_dir,
+        evaluation_strategy=CFG.evaluation_strategy,
+        eval_steps=CFG.eval_steps,
+        save_strategy=CFG.save_strategy,
+        save_steps=CFG.save_steps,
+        logging_strategy=CFG.logging_strategy,
+        logging_steps=CFG.logging_steps,
+        learning_rate=CFG.lr,
+        per_device_train_batch_size=CFG.batch_size,
+        per_device_eval_batch_size=CFG.batch_size,
+        weight_decay=CFG.weight_decay,
+        save_total_limit=CFG.save_total_limit,
+        num_train_epochs=CFG.epochs,
+        predict_with_generate=True,
+        fp16=CFG.fp16,
+        disable_tqdm=CFG.disable_tqdm,
+        push_to_hub=False,
+        load_best_model_at_end=True,
+    )
+
+    model.config.eval_beams = CFG.eval_beams
+    model.config.max_length = CFG.target_max_length
+    trainer = Seq2SeqTrainer(
+        model,
+        args,
+        train_dataset=tokenized_datasets["train"],
+        eval_dataset=tokenized_datasets["validation"],
+        data_collator=data_collator,
+        tokenizer=tokenizer,
+        compute_metrics=lambda eval_preds: get_accuracy_score(eval_preds, CFG),
+        callbacks=[EarlyStoppingCallback(early_stopping_patience=10)],
+    )
+
+    trainer.train(resume_from_checkpoint=False)
+    trainer.save_model("./best_model")

task_retrosynthesis/generate_embedding.py

@@ -0,0 +1,131 @@
+import argparse
+import os
+import sys
+import warnings
+
+import numpy as np
+import pandas as pd
+import torch
+from torch.utils.data import DataLoader
+from transformers import AutoTokenizer, T5EncoderModel
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from generation_utils import ReactionT5Dataset
+from train import preprocess_df, preprocess_USPTO
+from utils import filter_out, seed_everything
+
+warnings.filterwarnings("ignore")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--input_data",
+        type=str,
+        required=True,
+        help="Path to the input data.",
+    )
+    parser.add_argument(
+        "--test_data",
+        type=str,
+        required=False,
+        help="Path to the test data. If provided, the duplicates will be removed from the input data.",
+    )
+    parser.add_argument(
+        "--input_max_length",
+        type=int,
+        default=400,
+        help="Maximum token length of input.",
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        type=str,
+        default="sagawa/ReactionT5v2-retrosynthesis",
+        help="Name or path of the finetuned model for prediction. Can be a local model or one from Hugging Face.",
+    )
+    parser.add_argument(
+        "--batch_size", type=int, default=5, help="Batch size for prediction."
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="./",
+        help="Directory where predictions are saved.",
+    )
+    parser.add_argument(
+        "--debug", action="store_true", default=False, help="Use debug mode."
+    )
+    parser.add_argument(
+        "--seed", type=int, default=42, help="Seed for reproducibility."
+    )
+    return parser.parse_args()
+
+
+def create_embedding(dataloader, model, device):
+    outputs_mean = []
+    model.eval()
+    model.to(device)
+    for inputs in dataloader:
+        inputs = {k: v.to(CFG.device) for k, v in inputs.items()}
+        with torch.no_grad():
+            output = model(**inputs)
+        last_hidden_states = output[0]
+        input_mask_expanded = (
+            inputs["attention_mask"]
+            .unsqueeze(-1)
+            .expand(last_hidden_states.size())
+            .float()
+        )
+        sum_embeddings = torch.sum(last_hidden_states * input_mask_expanded, 1)
+        sum_mask = input_mask_expanded.sum(1)
+        sum_mask = torch.clamp(sum_mask, min=1e-6)
+        mean_embeddings = sum_embeddings / sum_mask
+        outputs_mean.append(mean_embeddings.detach().cpu().numpy())
+
+    return outputs_mean
+
+
+if __name__ == "__main__":
+    CFG = parse_args()
+    CFG.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    if not os.path.exists(CFG.output_dir):
+        os.makedirs(CFG.output_dir)
+
+    seed_everything(seed=CFG.seed)
+
+    CFG.tokenizer = AutoTokenizer.from_pretrained(
+        os.path.abspath(CFG.model_name_or_path)
+        if os.path.exists(CFG.model_name_or_path)
+        else CFG.model_name_or_path,
+        return_tensors="pt",
+    )
+    model = T5EncoderModel.from_pretrained(CFG.model_name_or_path).to(CFG.device)
+    model.eval()
+
+    input_data = filter_out(pd.read_csv(CFG.input_data), ["REACTANT", "PRODUCT"])
+    input_data = preprocess_df(input_data, drop_duplicates=False)
+
+    if CFG.test_data:
+        input_data_copy = preprocess_USPTO(input_data.copy())
+        test_data = filter_out(pd.read_csv(CFG.test_data), ["REACTANT", "PRODUCT"])
+        USPTO_test = preprocess_USPTO(test_data)
+        input_data = input_data[
+            ~input_data_copy["pair"].isin(USPTO_test["pair"])
+        ].reset_index(drop=True)
+
+    input_data.to_csv(os.path.join(CFG.output_dir, "input_data.csv"), index=False)
+    dataset = ReactionT5Dataset(CFG, input_data)
+    dataloader = DataLoader(
+        dataset,
+        batch_size=CFG.batch_size,
+        shuffle=False,
+        num_workers=4,
+        pin_memory=True,
+        drop_last=False,
+    )
+
+    outputs = create_embedding(dataloader, model, CFG.device)
+    outputs = np.concatenate(outputs, axis=0)
+
+    np.save(os.path.join(CFG.output_dir, "embedding_mean.npy"), outputs)

task_retrosynthesis/get_distance.py

@@ -0,0 +1,74 @@
+import argparse
+import os
+import sys
+import warnings
+
+import numpy as np
+import pandas as pd
+import torch
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from utils import seed_everything
+
+warnings.filterwarnings("ignore")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Search for similar reactions.")
+    parser.add_argument(
+        "--input_data",
+        type=str,
+        required=True,
+        help="Path to the input data.",
+    )
+    parser.add_argument(
+        "--target_embedding",
+        type=str,
+        required=True,
+        help="Path to the target embedding.",
+    )
+    parser.add_argument(
+        "--query_embedding",
+        type=str,
+        required=True,
+        help="Path to the target embedding.",
+    )
+    parser.add_argument("--batch_size", type=int, default=64, help="Batch size.")
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="./",
+        help="Directory where results are saved.",
+    )
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    config = parse_args()
+    seed_everything(42)
+
+    target_embedding = np.load(config.target_embedding)
+    query_embedding = np.load(config.query_embedding)
+
+    target_embedding = torch.tensor(target_embedding, dtype=torch.float32).cuda()
+    query_embedding = torch.tensor(query_embedding, dtype=torch.float32).cuda()
+
+    target_embedding = torch.nn.functional.normalize(target_embedding, p=2, dim=1)
+    query_embedding = torch.nn.functional.normalize(query_embedding, p=2, dim=1)
+
+    batch_size = config.batch_size
+    distances = []
+
+    for i in range(0, query_embedding.shape[0], batch_size):
+        print(f"Processing batch {i // batch_size}...")
+        batch = query_embedding[i : i + batch_size]
+        similarity = torch.matmul(batch, target_embedding.T)
+        distance, _ = torch.max(similarity, dim=1)
+        distances.append(distance.cpu().tolist())
+
+    distances = np.concatenate(distances)
+
+    df = pd.read_csv(config.input_data)
+    df["distance"] = distances
+    df.to_csv(os.path.join(config.output_dir, "distance.csv"), index=False)

task_retrosynthesis/prediction.py

@@ -0,0 +1,143 @@
+import argparse
+import gc
+import os
+import sys
+import warnings
+
+import pandas as pd
+import torch
+from torch.utils.data import DataLoader
+from tqdm import tqdm
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from generation_utils import (
+    ReactionT5Dataset,
+    decode_output,
+    save_multiple_predictions,
+)
+from train import preprocess_df
+from utils import seed_everything
+
+warnings.filterwarnings("ignore")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Script for reaction retrosynthesis prediction."
+    )
+    parser.add_argument(
+        "--input_data",
+        type=str,
+        required=True,
+        help="Path to the input data.",
+    )
+    parser.add_argument(
+        "--input_max_length",
+        type=int,
+        default=400,
+        help="Maximum token length of input.",
+    )
+    parser.add_argument(
+        "--output_min_length",
+        type=int,
+        default=1,
+        help="Minimum token length of output.",
+    )
+    parser.add_argument(
+        "--output_max_length",
+        type=int,
+        default=300,
+        help="Maximum token length of output.",
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        type=str,
+        default="sagawa/ReactionT5v2-retrosynthesis",
+        help="Name or path of the finetuned model for prediction. Can be a local model or one from Hugging Face.",
+    )
+    parser.add_argument(
+        "--num_beams", type=int, default=5, help="Number of beams used for beam search."
+    )
+    parser.add_argument(
+        "--num_return_sequences",
+        type=int,
+        default=5,
+        help="Number of predictions returned. Must be less than or equal to num_beams.",
+    )
+    parser.add_argument(
+        "--batch_size", type=int, default=5, help="Batch size for prediction."
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="./",
+        help="Directory where predictions are saved.",
+    )
+    parser.add_argument(
+        "--debug", action="store_true", default=False, help="Use debug mode."
+    )
+    parser.add_argument(
+        "--seed", type=int, default=42, help="Seed for reproducibility."
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    CFG = parse_args()
+    CFG.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    if not os.path.exists(CFG.output_dir):
+        os.makedirs(CFG.output_dir)
+
+    seed_everything(seed=CFG.seed)
+
+    CFG.tokenizer = AutoTokenizer.from_pretrained(
+        os.path.abspath(CFG.model_name_or_path)
+        if os.path.exists(CFG.model_name_or_path)
+        else CFG.model_name_or_path,
+        return_tensors="pt",
+    )
+    model = AutoModelForSeq2SeqLM.from_pretrained(
+        os.path.abspath(CFG.model_name_or_path)
+        if os.path.exists(CFG.model_name_or_path)
+        else CFG.model_name_or_path
+    ).to(CFG.device)
+    model.eval()
+
+    input_data = pd.read_csv(CFG.input_data)
+    input_data = preprocess_df(input_data, drop_duplicates=False)
+    dataset = ReactionT5Dataset(CFG, input_data)
+    dataloader = DataLoader(
+        dataset,
+        batch_size=CFG.batch_size,
+        shuffle=False,
+        num_workers=4,
+        pin_memory=True,
+        drop_last=False,
+    )
+
+    all_sequences, all_scores = [], []
+    for inputs in tqdm(dataloader, total=len(dataloader)):
+        inputs = {k: v.to(CFG.device) for k, v in inputs.items()}
+        with torch.no_grad():
+            output = model.generate(
+                **inputs,
+                min_length=CFG.output_min_length,
+                max_length=CFG.output_max_length,
+                num_beams=CFG.num_beams,
+                num_return_sequences=CFG.num_return_sequences,
+                return_dict_in_generate=True,
+                output_scores=True,
+            )
+        sequences, scores = decode_output(output, CFG)
+        all_sequences.extend(sequences)
+        if scores:
+            all_scores.extend(scores)
+        del output
+        torch.cuda.empty_cache()
+        gc.collect()
+
+    output_df = save_multiple_predictions(input_data, all_sequences, all_scores, CFG)
+
+    output_df.to_csv(os.path.join(CFG.output_dir, "output.csv"), index=False)

task_retrosynthesis/train.py

@@ -0,0 +1,305 @@
+import argparse
+import os
+import sys
+import warnings
+from pathlib import Path
+
+import datasets
+import pandas as pd
+import torch
+from datasets import Dataset, DatasetDict
+from transformers import (
+    AutoModelForSeq2SeqLM,
+    AutoTokenizer,
+    DataCollatorForSeq2Seq,
+    EarlyStoppingCallback,
+    Seq2SeqTrainer,
+    Seq2SeqTrainingArguments,
+)
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from utils import (
+    add_new_tokens,
+    filter_out,
+    get_accuracy_score,
+    preprocess_dataset,
+    seed_everything,
+)
+
+# Suppress warnings and disable progress bars
+warnings.filterwarnings("ignore")
+datasets.utils.logging.disable_progress_bar()
+
+
+def parse_args():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Training script for reaction prediction model."
+    )
+    parser.add_argument(
+        "--train_data_path", type=str, required=True, help="Path to training data CSV."
+    )
+    parser.add_argument(
+        "--valid_data_path",
+        type=str,
+        required=True,
+        help="Path to validation data CSV.",
+    )
+    parser.add_argument("--test_data_path", type=str, help="Path to test data CSV.")
+    parser.add_argument(
+        "--USPTO_test_data_path",
+        type=str,
+        help="The path to data used for USPTO testing. CSV file that contains ['REACTANT', 'PRODUCT'] columns is expected.",
+    )
+    parser.add_argument(
+        "--output_dir", type=str, default="t5", help="Path of the output directory."
+    )
+    parser.add_argument(
+        "--pretrained_model_name_or_path",
+        type=str,
+        required=True,
+        help="Pretrained model path or name.",
+    )
+    parser.add_argument(
+        "--debug", action="store_true", default=False, help="Enable debug mode."
+    )
+    parser.add_argument(
+        "--epochs",
+        type=int,
+        default=5,
+        help="Number of epochs.",
+    )
+    parser.add_argument("--lr", type=float, default=1e-3, help="Learning rate.")
+    parser.add_argument("--batch_size", type=int, default=16, help="Batch size.")
+    parser.add_argument(
+        "--input_max_length",
+        type=int,
+        default=400,
+        help="Max input token length.",
+    )
+    parser.add_argument(
+        "--target_max_length",
+        type=int,
+        default=150,
+        help="Max target token length.",
+    )
+    parser.add_argument(
+        "--eval_beams",
+        type=int,
+        default=5,
+        help="Number of beams used for beam search during evaluation.",
+    )
+    parser.add_argument(
+        "--target_column",
+        type=str,
+        default="REACTANT",
+        help="Target column name.",
+    )
+    parser.add_argument(
+        "--weight_decay",
+        type=float,
+        default=0.01,
+        help="Weight decay.",
+    )
+    parser.add_argument(
+        "--evaluation_strategy",
+        type=str,
+        default="epoch",
+        help="Evaluation strategy used during training. Select from 'no', 'steps', or 'epoch'. If you select 'steps', also give --eval_steps.",
+    )
+    parser.add_argument(
+        "--eval_steps",
+        type=int,
+        help="Evaluation steps.",
+    )
+    parser.add_argument(
+        "--save_strategy",
+        type=str,
+        default="epoch",
+        help="Save strategy used during training. Select from 'no', 'steps', or 'epoch'. If you select 'steps', also give --save_steps.",
+    )
+    parser.add_argument(
+        "--save_steps",
+        type=int,
+        default=500,
+        help="Save steps.",
+    )
+    parser.add_argument(
+        "--logging_strategy",
+        type=str,
+        default="epoch",
+        help="Logging strategy used during training. Select from 'no', 'steps', or 'epoch'. If you select 'steps', also give --logging_steps.",
+    )
+    parser.add_argument(
+        "--logging_steps",
+        type=int,
+        default=500,
+        help="Logging steps.",
+    )
+    parser.add_argument(
+        "--save_total_limit",
+        type=int,
+        default=2,
+        help="Limit of saved checkpoints.",
+    )
+    parser.add_argument(
+        "--fp16",
+        action="store_true",
+        default=False,
+        help="Enable fp16 training.",
+    )
+    parser.add_argument(
+        "--disable_tqdm",
+        action="store_true",
+        default=False,
+        help="Disable tqdm.",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=42,
+        help="Random seed.",
+    )
+
+    return parser.parse_args()
+
+
+def preprocess_df(df, drop_duplicates=True):
+    """Preprocess the dataframe by filling NaNs, dropping duplicates, and formatting the input."""
+    for col in ["REACTANT", "PRODUCT", "CATALYST", "REAGENT", "SOLVENT"]:
+        if col not in df.columns:
+            df[col] = None
+        df[col] = df[col].fillna(" ")
+
+    if drop_duplicates:
+        df = (
+            df[["REACTANT", "PRODUCT", "CATALYST", "REAGENT", "SOLVENT"]]
+            .drop_duplicates()
+            .reset_index(drop=True)
+        )
+    df["input"] = df["PRODUCT"]
+
+    return df
+
+
+def preprocess_USPTO(df):
+    df["REACTANT"] = df["REACTANT"].apply(lambda x: str(sorted(x.split("."))))
+    df["PRODUCT"] = df["PRODUCT"].apply(lambda x: str(sorted(x.split("."))))
+
+    df["pair"] = df["REACTANT"] + " - " + df["PRODUCT"].astype(str)
+
+    return df
+
+
+if __name__ == "__main__":
+    CFG = parse_args()
+    CFG.disable_tqdm = True
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    seed_everything(seed=CFG.seed)
+
+    train = preprocess_df(
+        filter_out(pd.read_csv(CFG.train_data_path), ["REACTANT", "PRODUCT"])
+    )
+    valid = preprocess_df(
+        filter_out(pd.read_csv(CFG.valid_data_path), ["REACTANT", "PRODUCT"])
+    )
+    if CFG.USPTO_test_data_path:
+        train_copy = preprocess_USPTO(train.copy())
+        USPTO_test = preprocess_USPTO(pd.read_csv(CFG.USPTO_test_data_path))
+        train = train[~train_copy["pair"].isin(USPTO_test["pair"])].reset_index(
+            drop=True
+        )
+    train["pair"] = train["REACTANT"] + " - " + train["PRODUCT"]
+    valid["pair"] = valid["REACTANT"] + " - " + valid["PRODUCT"]
+    valid = valid[~valid["pair"].isin(train["pair"])].reset_index(drop=True)
+    train.to_csv("train.csv", index=False)
+    valid.to_csv("valid.csv", index=False)
+
+    if CFG.test_data_path:
+        test = preprocess_df(
+            filter_out(pd.read_csv(CFG.test_data_path), ["REACTANT", "PRODUCT"])
+        )
+        test["pair"] = test["REACTANT"] + " - " + test["PRODUCT"]
+        test = test[~test["pair"].isin(train["pair"])].reset_index(drop=True)
+        test = test.drop_duplicates(subset=["pair"]).reset_index(drop=True)
+        test.to_csv("test.csv", index=False)
+
+    dataset = DatasetDict(
+        {
+            "train": Dataset.from_pandas(train[["input", "REACTANT"]]),
+            "validation": Dataset.from_pandas(valid[["input", "REACTANT"]]),
+        }
+    )
+
+    # load tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(
+        os.path.abspath(CFG.pretrained_model_name_or_path)
+        if os.path.exists(CFG.pretrained_model_name_or_path)
+        else CFG.pretrained_model_name_or_path,
+        return_tensors="pt",
+    )
+    tokenizer = add_new_tokens(
+        tokenizer,
+        Path(__file__).resolve().parent.parent / "data" / "additional_tokens.txt",
+    )
+    tokenizer.add_special_tokens(
+        {
+            "additional_special_tokens": tokenizer.additional_special_tokens
+            + ["REACTANT:", "REAGENT:"]
+        }
+    )
+    CFG.tokenizer = tokenizer
+
+    model = AutoModelForSeq2SeqLM.from_pretrained(
+        os.path.abspath(CFG.pretrained_model_name_or_path) if os.path.exists(CFG.pretrained_model_name_or_path) else CFG.pretrained_model_name_or_path
+    )
+    model.resize_token_embeddings(len(tokenizer))
+
+    tokenized_datasets = dataset.map(
+        lambda examples: preprocess_dataset(examples, CFG),
+        batched=True,
+        remove_columns=dataset["train"].column_names,
+        load_from_cache_file=False,
+    )
+
+    data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
+
+    args = Seq2SeqTrainingArguments(
+        CFG.output_dir,
+        evaluation_strategy=CFG.evaluation_strategy,
+        eval_steps=CFG.eval_steps,
+        save_strategy=CFG.save_strategy,
+        save_steps=CFG.save_steps,
+        logging_strategy=CFG.logging_strategy,
+        logging_steps=CFG.logging_steps,
+        learning_rate=CFG.lr,
+        per_device_train_batch_size=CFG.batch_size,
+        per_device_eval_batch_size=CFG.batch_size,
+        weight_decay=CFG.weight_decay,
+        save_total_limit=CFG.save_total_limit,
+        num_train_epochs=CFG.epochs,
+        predict_with_generate=True,
+        fp16=CFG.fp16,
+        disable_tqdm=CFG.disable_tqdm,
+        push_to_hub=False,
+        load_best_model_at_end=True,
+    )
+
+    model.config.eval_beams = CFG.eval_beams
+    model.config.max_length = CFG.target_max_length
+    trainer = Seq2SeqTrainer(
+        model,
+        args,
+        train_dataset=tokenized_datasets["train"],
+        eval_dataset=tokenized_datasets["validation"],
+        data_collator=data_collator,
+        tokenizer=tokenizer,
+        compute_metrics=lambda eval_preds: get_accuracy_score(eval_preds, CFG),
+        callbacks=[EarlyStoppingCallback(early_stopping_patience=10)],
+    )
+
+    try:
+        trainer.train(resume_from_checkpoint=True)
+    except:
+        trainer.train(resume_from_checkpoint=None)
+    trainer.save_model("./best_model")

task_yield/convert_to_PreTrainedModel.py

@@ -0,0 +1,77 @@
+import argparse
+import glob
+import os
+import sys
+
+import torch
+from transformers import AutoConfig, AutoTokenizer
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from models import ReactionT5Yield
+
+
+def parse_args():
+    """
+    Parse command line arguments.
+    """
+    parser = argparse.ArgumentParser(
+        description="ReactionT5Yield model impremented with nn.Module with transformers' PreTrainedModel"
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        type=str,
+        help="The name of a finetuned model or path to a model which you want to convert. You can use your local models or models uploaded to hugging face.",
+    )
+    parser.add_argument(
+        "--base_model_name_or_path",
+        type=str,
+        help="The name of the base model of the finetuned model",
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="./",
+        help="Directory to save the prediction.",
+    )
+    parser.add_argument(
+        "--fc_dropout",
+        type=float,
+        default=0.0,
+    )
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    CFG = parse_args()
+
+    if not os.path.exists(CFG.output_dir):
+        os.makedirs(CFG.output_dir)
+
+    CFG.tokenizer = AutoTokenizer.from_pretrained(
+        CFG.model_name_or_path, return_tensors="pt"
+    )
+
+    model = ReactionT5Yield(
+        CFG,
+        config_path=os.path.join(CFG.model_name_or_path, "config.pth"),
+        pretrained=False,
+    )
+    pth_files = glob.glob(os.path.join(CFG.model_name_or_path, "*.pth"))
+    for pth_file in pth_files:
+        state = torch.load(
+            pth_file,
+            map_location=torch.device("cpu"),
+        )
+        try:
+            model.load_state_dict(state)
+            break
+        except:
+            pass
+
+    config = AutoConfig.from_pretrained(CFG.base_model_name_or_path)
+    config.vocab_size = len(CFG.tokenizer)
+
+    CFG.tokenizer.save_pretrained(CFG.output_dir)
+    torch.save(model.state_dict(), os.path.join(CFG.output_dir, "pytorch_model.bin"))
+    config.save_pretrained(CFG.output_dir)

task_yield/finetune.py

@@ -0,0 +1,219 @@
+import argparse
+import os
+import subprocess
+import sys
+import warnings
+
+import pandas as pd
+import torch
+from datasets.utils.logging import disable_progress_bar
+from transformers import AutoTokenizer
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from train import preprocess_df, train_loop
+from utils import get_logger, seed_everything
+
+# Suppress warnings and logging
+warnings.filterwarnings("ignore")
+disable_progress_bar()
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+
+def parse_args():
+    """
+    Parse command line arguments.
+    """
+    parser = argparse.ArgumentParser(
+        description="Training script for ReactionT5Yield model."
+    )
+
+    parser.add_argument(
+        "--train_data_path",
+        type=str,
+        required=True,
+        help="Path to training data CSV file.",
+    )
+    parser.add_argument(
+        "--valid_data_path",
+        type=str,
+        required=True,
+        help="Path to validation data CSV file.",
+    )
+    parser.add_argument(
+        "--similar_reaction_data_path",
+        type=str,
+        required=False,
+        help="Path to similar data CSV.",
+    )
+    parser.add_argument(
+        "--pretrained_model_name_or_path",
+        type=str,
+        default="sagawa/CompoundT5",
+        help="Pretrained model name or path.",
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        type=str,
+        help="The model's name or path used for fine-tuning.",
+    )
+    parser.add_argument(
+        "--download_pretrained_model",
+        action="store_true",
+        default=False,
+        required=False,
+        help="Download pretrained model from hugging face hub and use it for fine-tuning.",
+    )
+    parser.add_argument("--debug", action="store_true", help="Enable debug mode.")
+    parser.add_argument(
+        "--epochs", type=int, default=200, help="Number of training epochs."
+    )
+    parser.add_argument(
+        "--patience", type=int, default=10, help="Early stopping patience."
+    )
+    parser.add_argument("--lr", type=float, default=1e-5, help="Learning rate.")
+    parser.add_argument("--batch_size", type=int, default=32, help="Batch size.")
+    parser.add_argument(
+        "--input_max_length", type=int, default=300, help="Maximum input token length."
+    )
+    parser.add_argument(
+        "--num_workers", type=int, default=4, help="Number of data loading workers."
+    )
+    parser.add_argument(
+        "--fc_dropout",
+        type=float,
+        default=0.0,
+        help="Dropout rate after fully connected layers.",
+    )
+    parser.add_argument(
+        "--eps", type=float, default=1e-6, help="Epsilon for Adam optimizer."
+    )
+    parser.add_argument(
+        "--weight_decay", type=float, default=0.05, help="Weight decay for optimizer."
+    )
+    parser.add_argument(
+        "--max_grad_norm",
+        type=int,
+        default=1000,
+        help="Maximum gradient norm for clipping.",
+    )
+    parser.add_argument(
+        "--gradient_accumulation_steps",
+        type=int,
+        default=1,
+        help="Gradient accumulation steps.",
+    )
+    parser.add_argument(
+        "--num_warmup_steps", type=int, default=0, help="Number of warmup steps."
+    )
+    parser.add_argument(
+        "--batch_scheduler", action="store_true", help="Use batch scheduler."
+    )
+    parser.add_argument(
+        "--print_freq", type=int, default=100, help="Logging frequency."
+    )
+    parser.add_argument(
+        "--use_amp",
+        action="store_true",
+        help="Use automatic mixed precision for training.",
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="./",
+        help="Directory to save the trained model.",
+    )
+    parser.add_argument(
+        "--seed", type=int, default=42, help="Random seed for reproducibility."
+    )
+    parser.add_argument(
+        "--sampling_num",
+        type=int,
+        default=-1,
+        help="Number of samples used for training. If you want to use all samples, set -1.",
+    )
+    parser.add_argument(
+        "--sampling_frac",
+        type=float,
+        default=-1.0,
+        help="Ratio of samples used for training. If you want to use all samples, set -1.0.",
+    )
+    parser.add_argument(
+        "--checkpoint",
+        type=str,
+        help="Path to the checkpoint file for resuming training.",
+    )
+
+    return parser.parse_args()
+
+
+def download_pretrained_model():
+    """
+    Download the pretrained model from Hugging Face.
+    """
+    subprocess.run(
+        "wget https://huggingface.co/sagawa/ReactionT5v2-yield/resolve/main/CompoundT5_best.pth",
+        shell=True,
+    )
+    subprocess.run(
+        "wget https://huggingface.co/sagawa/ReactionT5v2-yield/resolve/main/config.pth",
+        shell=True,
+    )
+    subprocess.run(
+        "wget https://huggingface.co/sagawa/ReactionT5v2-yield/resolve/main/special_tokens_map.json",
+        shell=True,
+    )
+    subprocess.run(
+        "wget https://huggingface.co/sagawa/ReactionT5v2-yield/resolve/main/tokenizer.json",
+        shell=True,
+    )
+    subprocess.run(
+        "wget https://huggingface.co/sagawa/ReactionT5v2-yield/resolve/main/tokenizer_config.json",
+        shell=True,
+    )
+
+
+if __name__ == "__main__":
+    CFG = parse_args()
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    CFG.device = device
+    if not os.path.exists(CFG.output_dir):
+        os.makedirs(CFG.output_dir)
+    seed_everything(seed=CFG.seed)
+
+    if CFG.download_pretrained_model:
+        download_pretrained_model()
+        CFG.model_name_or_path = "."
+
+    train = pd.read_csv(CFG.train_data_path).drop_duplicates().reset_index(drop=True)
+    valid = pd.read_csv(CFG.valid_data_path).drop_duplicates().reset_index(drop=True)
+    train = preprocess_df(train, CFG)
+    valid = preprocess_df(valid, CFG)
+
+    if CFG.sampling_num > 0:
+        train = train.sample(n=CFG.sampling_num, random_state=CFG.seed).reset_index(
+            drop=True
+        )
+    elif CFG.sampling_frac > 0 and CFG.sampling_frac < 1:
+        train = train.sample(frac=CFG.sampling_frac, random_state=CFG.seed).reset_index(
+            drop=True
+        )
+
+    if CFG.similar_reaction_data_path:
+        similar = preprocess_df(pd.read_csv(CFG.similar_reaction_data_path), CFG)
+        print(len(train))
+        train = pd.concat([train, similar], ignore_index=True)
+        print(len(train))
+
+    LOGGER = get_logger(os.path.join(CFG.output_dir, "train"))
+    CFG.logger = LOGGER
+
+    tokenizer = AutoTokenizer.from_pretrained(
+        os.path.abspath(CFG.model_name_or_path)
+        if os.path.exists(CFG.model_name_or_path)
+        else CFG.model_name_or_path,
+        return_tensors="pt",
+    )
+    tokenizer.save_pretrained(CFG.output_dir)
+    CFG.tokenizer = tokenizer
+
+    train_loop(train, valid, CFG)

task_yield/generate_embedding.py

@@ -0,0 +1,138 @@
+import argparse
+import os
+import sys
+
+import numpy as np
+import pandas as pd
+import torch
+from torch.utils.data import DataLoader
+from transformers import AutoTokenizer
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from generation_utils import ReactionT5Dataset
+from models import ReactionT5Yield2
+from train import preprocess_df
+from utils import filter_out, seed_everything
+
+
+def parse_args():
+    """
+    Parse command line arguments.
+    """
+    parser = argparse.ArgumentParser(
+        description="Prediction script for ReactionT5Yield model."
+    )
+
+    parser.add_argument(
+        "--input_data",
+        type=str,
+        required=True,
+        help="Data as a string or CSV file that contains an 'input' column. The format of the string or contents of the column are like 'REACTANT:{reactants of the reaction}PRODUCT:{products of the reaction}'. If there are multiple reactants, concatenate them with '.'.",
+    )
+    parser.add_argument(
+        "--test_data",
+        type=str,
+        required=False,
+        help="Path to the test data. If provided, the duplicates will be removed from the input data.",
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        type=str,
+        default="sagawa/ReactionT5v2-yield",
+        help="Name or path of the finetuned model for prediction. Can be a local model or one from Hugging Face.",
+    )
+    parser.add_argument("--debug", action="store_true", help="Use debug mode.")
+    parser.add_argument(
+        "--input_max_length",
+        type=int,
+        default=400,
+        help="Maximum token length of input.",
+    )
+    parser.add_argument(
+        "--batch_size", type=int, default=5, required=False, help="Batch size."
+    )
+    parser.add_argument(
+        "--num_workers", type=int, default=4, help="Number of data loading workers."
+    )
+    parser.add_argument(
+        "--fc_dropout",
+        type=float,
+        default=0.0,
+        help="Dropout rate after fully connected layers.",
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="./",
+        help="Directory where predictions are saved.",
+    )
+    parser.add_argument(
+        "--seed", type=int, default=42, help="Random seed for reproducibility."
+    )
+
+    return parser.parse_args()
+
+
+def create_embedding(dataloader, model, device):
+    outputs = []
+    model.eval()
+    model.to(device)
+    for inputs in dataloader:
+        inputs = {k: v.to(device) for k, v in inputs.items()}
+        with torch.no_grad():
+            output = model.generate_embedding(inputs)
+
+        outputs.append(output.detach().cpu().numpy())
+
+    return outputs
+
+
+if __name__ == "__main__":
+    CFG = parse_args()
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    CFG.device = device
+
+    if not os.path.exists(CFG.output_dir):
+        os.makedirs(CFG.output_dir)
+
+    seed_everything(seed=CFG.seed)
+
+    CFG.tokenizer = AutoTokenizer.from_pretrained(
+        os.path.abspath(CFG.model_name_or_path)
+        if os.path.exists(CFG.model_name_or_path)
+        else CFG.model_name_or_path,
+        return_tensors="pt",
+    )
+
+    model = ReactionT5Yield2.from_pretrained(CFG.model_name_or_path).to(CFG.device)
+    model.eval()
+
+    input_data = filter_out(
+        pd.read_csv(CFG.input_data), ["YIELD", "REACTANT", "PRODUCT"]
+    )
+    input_data = preprocess_df(input_data, CFG, drop_duplicates=False)
+    if CFG.test_data:
+        test_data = filter_out(
+            pd.read_csv(CFG.test_data), ["YIELD", "REACTANT", "PRODUCT"]
+        )
+        test_data = preprocess_df(test_data, CFG, drop_duplicates=False)
+        # Remove duplicates from the input data
+        input_data = input_data[
+            ~input_data["input"].isin(test_data["input"])
+        ].reset_index(drop=True)
+    input_data.to_csv(os.path.join(CFG.output_dir, "input_data.csv"), index=False)
+    dataset = ReactionT5Dataset(CFG, input_data)
+    dataloader = DataLoader(
+        dataset,
+        batch_size=CFG.batch_size,
+        shuffle=False,
+        num_workers=CFG.num_workers,
+        pin_memory=True,
+        drop_last=False,
+    )
+
+    outputs = create_embedding(dataloader, model, CFG.device)
+    outputs = np.concatenate(outputs, axis=0)
+
+    np.save(os.path.join(CFG.output_dir, "embedding_mean.npy"), outputs)

task_yield/get_distance.py

@@ -0,0 +1,80 @@
+import argparse
+import os
+import sys
+import warnings
+
+import numpy as np
+import pandas as pd
+import torch
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from utils import seed_everything
+
+warnings.filterwarnings("ignore")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Search for similar reactions.")
+    parser.add_argument(
+        "--input_data",
+        type=str,
+        required=True,
+        help="Path to the input data.",
+    )
+    parser.add_argument(
+        "--target_embedding",
+        type=str,
+        required=True,
+        help="Path to the target embedding.",
+    )
+    parser.add_argument(
+        "--query_embedding",
+        type=str,
+        required=True,
+        help="Path to the target embedding.",
+    )
+    parser.add_argument(
+        "--top_k",
+        type=int,
+        default=1,
+        help="Number of similar reactions to retrieve.",
+    )
+    parser.add_argument("--batch_size", type=int, default=64, help="Batch size.")
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="./",
+        help="Directory where results are saved.",
+    )
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    config = parse_args()
+    seed_everything(42)
+
+    target_embedding = np.load(config.target_embedding)
+    query_embedding = np.load(config.query_embedding)
+
+    target_embedding = torch.tensor(target_embedding, dtype=torch.float32).cuda()
+    query_embedding = torch.tensor(query_embedding, dtype=torch.float32).cuda()
+
+    target_embedding = torch.nn.functional.normalize(target_embedding, p=2, dim=1)
+    query_embedding = torch.nn.functional.normalize(query_embedding, p=2, dim=1)
+
+    batch_size = config.batch_size
+    distances = []
+
+    for i in range(0, query_embedding.shape[0], batch_size):
+        print(f"Processing batch {i // batch_size}...")
+        batch = query_embedding[i : i + batch_size]
+        similarity = torch.matmul(batch, target_embedding.T)
+        distance, _ = torch.max(similarity, dim=1)
+        distances.append(distance.cpu().tolist())
+
+    distances = np.concatenate(distances)
+
+    df = pd.read_csv(config.input_data)
+    df["distance"] = distances
+    df.to_csv(os.path.join(config.output_dir, "distance.csv"), index=False)

task_yield/prediction.py

@@ -0,0 +1,173 @@
+import argparse
+import glob
+import logging
+import os
+import sys
+import warnings
+
+import numpy as np
+import pandas as pd
+import torch
+from datasets.utils.logging import disable_progress_bar
+from torch.utils.data import DataLoader
+from tqdm import tqdm
+from transformers import AutoTokenizer
+
+# Suppress warnings and logging
+warnings.filterwarnings("ignore")
+logging.disable(logging.WARNING)
+disable_progress_bar()
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+# Append the utils module path
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from finetune import download_pretrained_model
+from generation_utils import ReactionT5Dataset
+from models import ReactionT5Yield
+from train import preprocess_df
+from utils import seed_everything
+
+
+def parse_args():
+    """
+    Parse command line arguments.
+    """
+    parser = argparse.ArgumentParser(
+        description="Prediction script for ReactionT5Yield model."
+    )
+
+    parser.add_argument(
+        "--input_data",
+        type=str,
+        required=True,
+        help="Data as a CSV file that contains an 'input' column. The format of the contents of the column are like 'REACTANT:{reactants of the reaction}PRODUCT:{products of the reaction}'. If there are multiple reactants, concatenate them with '.'.",
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        type=str,
+        help="Name or path of the finetuned model for prediction. Can be a local model or one from Hugging Face.",
+    )
+    parser.add_argument(
+        "--download_pretrained_model",
+        action="store_true",
+        help="Download finetuned model from hugging face hub and use it for prediction.",
+    )
+    parser.add_argument("--debug", action="store_true", help="Use debug mode.")
+    parser.add_argument(
+        "--input_max_length",
+        type=int,
+        default=300,
+        help="Maximum token length of input.",
+    )
+    parser.add_argument(
+        "--batch_size", type=int, default=5, required=False, help="Batch size."
+    )
+    parser.add_argument(
+        "--num_workers", type=int, default=4, help="Number of data loading workers."
+    )
+    parser.add_argument(
+        "--fc_dropout",
+        type=float,
+        default=0.0,
+        help="Dropout rate after fully connected layers.",
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="./",
+        help="Directory where predictions are saved.",
+    )
+    parser.add_argument(
+        "--seed", type=int, default=42, help="Random seed for reproducibility."
+    )
+
+    return parser.parse_args()
+
+
+def inference_fn(test_loader, model, cfg):
+    """
+    Inference function.
+
+    Args:
+        test_loader (DataLoader): DataLoader for test data.
+        model (nn.Module): Model for inference.
+        cfg (argparse.Namespace): Configuration object.
+
+    Returns:
+        np.ndarray: Predictions.
+    """
+    model.eval()
+    model.to(cfg.device)
+    preds = []
+
+    for inputs in tqdm(test_loader, total=len(test_loader)):
+        inputs = {k: v.to(cfg.device) for k, v in inputs.items()}
+        with torch.no_grad():
+            y_preds = model(inputs)
+        preds.append(y_preds.to("cpu").numpy())
+
+    return np.concatenate(preds)
+
+
+if __name__ == "__main__":
+    CFG = parse_args()
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    CFG.device = device
+
+    if not os.path.exists(CFG.output_dir):
+        os.makedirs(CFG.output_dir)
+
+    seed_everything(seed=CFG.seed)
+
+    if CFG.model_name_or_path is None:
+        CFG.download_pretrained_model = True
+
+    if CFG.download_pretrained_model:
+        download_pretrained_model()
+        CFG.model_name_or_path = "."
+
+    CFG.tokenizer = AutoTokenizer.from_pretrained(
+        os.path.abspath(CFG.model_name_or_path)
+        if os.path.exists(CFG.model_name_or_path)
+        else CFG.model_name_or_path,
+        return_tensors="pt",
+    )
+
+    model = ReactionT5Yield(
+        CFG,
+        config_path=os.path.join(CFG.model_name_or_path, "config.pth"),
+        pretrained=False,
+    )
+    pth_files = glob.glob(os.path.join(CFG.model_name_or_path, "*.pth"))
+    for pth_file in pth_files:
+        state = torch.load(
+            pth_file,
+            map_location=torch.device("cpu"),
+        )
+        try:
+            model.load_state_dict(state)
+            break
+        except:
+            pass
+
+    test_ds = pd.read_csv(CFG.input_data)
+    test_ds = preprocess_df(test_ds, CFG, drop_duplicates=False)
+
+    test_dataset = ReactionT5Dataset(CFG, test_ds)
+    test_loader = DataLoader(
+        test_dataset,
+        batch_size=CFG.batch_size,
+        shuffle=False,
+        num_workers=CFG.num_workers,
+        pin_memory=True,
+        drop_last=False,
+    )
+
+    prediction = inference_fn(test_loader, model, CFG)
+
+    test_ds["prediction"] = prediction * 100
+    test_ds["prediction"] = test_ds["prediction"].clip(0, 100)
+    test_ds.to_csv(
+        os.path.join(CFG.output_dir, "yield_prediction_output.csv"), index=False
+    )

task_yield/prediction_with_PreTrainedModel.py

@@ -0,0 +1,119 @@
+import argparse
+import logging
+import os
+import sys
+import warnings
+
+import pandas as pd
+import torch
+from datasets.utils.logging import disable_progress_bar
+from torch.utils.data import DataLoader
+from transformers import AutoTokenizer
+
+# Suppress warnings and logging
+warnings.filterwarnings("ignore")
+logging.disable(logging.WARNING)
+disable_progress_bar()
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+# Append the utils module path
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from generation_utils import ReactionT5Dataset
+from models import ReactionT5Yield2
+from prediction import inference_fn
+from train import preprocess_df
+from utils import seed_everything
+
+
+def parse_args():
+    """
+    Parse command line arguments.
+    """
+    parser = argparse.ArgumentParser(
+        description="Prediction script for ReactionT5Yield model."
+    )
+
+    parser.add_argument(
+        "--input_data",
+        type=str,
+        required=True,
+        help="Data as a CSV file that contains an 'input' column. The format of the contents of the column are like 'REACTANT:{reactants of the reaction}PRODUCT:{products of the reaction}'. If there are multiple reactants, concatenate them with '.'.",
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        type=str,
+        default="sagawa/ReactionT5v2-yield",
+        help="Name or path of the finetuned model for prediction. Can be a local model or one from Hugging Face.",
+    )
+    parser.add_argument("--debug", action="store_true", help="Use debug mode.")
+    parser.add_argument(
+        "--input_max_length",
+        type=int,
+        default=400,
+        help="Maximum token length of input.",
+    )
+    parser.add_argument(
+        "--batch_size", type=int, default=5, required=False, help="Batch size."
+    )
+    parser.add_argument(
+        "--num_workers", type=int, default=4, help="Number of data loading workers."
+    )
+    parser.add_argument(
+        "--fc_dropout",
+        type=float,
+        default=0.0,
+        help="Dropout rate after fully connected layers.",
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="./",
+        help="Directory where predictions are saved.",
+    )
+    parser.add_argument(
+        "--seed", type=int, default=42, help="Random seed for reproducibility."
+    )
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    CFG = parse_args()
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    CFG.device = device
+
+    if not os.path.exists(CFG.output_dir):
+        os.makedirs(CFG.output_dir)
+
+    seed_everything(seed=CFG.seed)
+
+    CFG.tokenizer = AutoTokenizer.from_pretrained(
+        os.path.abspath(CFG.model_name_or_path)
+        if os.path.exists(CFG.model_name_or_path)
+        else CFG.model_name_or_path,
+        return_tensors="pt",
+    )
+
+    model = ReactionT5Yield2.from_pretrained(CFG.model_name_or_path)
+
+    test_ds = pd.read_csv(CFG.input_data)
+    test_ds = preprocess_df(test_ds, CFG, drop_duplicates=False)
+
+    test_dataset = ReactionT5Dataset(CFG, test_ds)
+    test_loader = DataLoader(
+        test_dataset,
+        batch_size=CFG.batch_size,
+        shuffle=False,
+        num_workers=CFG.num_workers,
+        pin_memory=True,
+        drop_last=False,
+    )
+
+    prediction = inference_fn(test_loader, model, CFG)
+
+    test_ds["prediction"] = prediction
+    test_ds["prediction"] = test_ds["prediction"].clip(0, 100)
+    test_ds.to_csv(
+        os.path.join(CFG.output_dir, "yield_prediction_output.csv"), index=False
+    )

task_yield/train.py

@@ -0,0 +1,570 @@
+import argparse
+import gc
+import glob
+import os
+import sys
+import time
+import warnings
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+from datasets.utils.logging import disable_progress_bar
+from sklearn.metrics import mean_squared_error, r2_score
+from torch.optim import AdamW
+from torch.utils.data import DataLoader, Dataset
+from transformers import AutoTokenizer, get_linear_schedule_with_warmup
+
+# Append the utils module path
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from generation_utils import prepare_input
+from models import ReactionT5Yield
+from rdkit import RDLogger
+from utils import (
+    AverageMeter,
+    add_new_tokens,
+    canonicalize,
+    filter_out,
+    get_logger,
+    get_optimizer_params,
+    seed_everything,
+    space_clean,
+    timeSince,
+)
+
+# Suppress warnings and logging
+warnings.filterwarnings("ignore")
+RDLogger.DisableLog("rdApp.*")
+disable_progress_bar()
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+
+def parse_args():
+    """
+    Parse command line arguments.
+    """
+    parser = argparse.ArgumentParser(
+        description="Training script for ReactionT5Yield model."
+    )
+
+    parser.add_argument(
+        "--train_data_path",
+        type=str,
+        required=True,
+        help="Path to training data CSV file.",
+    )
+    parser.add_argument(
+        "--valid_data_path",
+        type=str,
+        required=True,
+        help="Path to validation data CSV file.",
+    )
+    parser.add_argument(
+        "--test_data_path",
+        type=str,
+        help="Path to testing data CSV file.",
+    )
+    parser.add_argument(
+        "--CN_test_data_path",
+        type=str,
+        help="Path to CN testing data CSV file.",
+    )
+    parser.add_argument(
+        "--pretrained_model_name_or_path",
+        type=str,
+        default="sagawa/CompoundT5",
+        help="Pretrained model name or path.",
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        type=str,
+        help="The model's name or path used for fine-tuning.",
+    )
+    parser.add_argument("--debug", action="store_true", help="Enable debug mode.")
+    parser.add_argument(
+        "--epochs", type=int, default=5, help="Number of training epochs."
+    )
+    parser.add_argument(
+        "--patience", type=int, default=10, help="Early stopping patience."
+    )
+    parser.add_argument("--lr", type=float, default=5e-4, help="Learning rate.")
+    parser.add_argument("--batch_size", type=int, default=5, help="Batch size.")
+    parser.add_argument(
+        "--input_max_length", type=int, default=400, help="Maximum input token length."
+    )
+    parser.add_argument(
+        "--num_workers", type=int, default=4, help="Number of data loading workers."
+    )
+    parser.add_argument(
+        "--fc_dropout",
+        type=float,
+        default=0.0,
+        help="Dropout rate after fully connected layers.",
+    )
+    parser.add_argument(
+        "--eps", type=float, default=1e-6, help="Epsilon for Adam optimizer."
+    )
+    parser.add_argument(
+        "--weight_decay", type=float, default=0.05, help="Weight decay for optimizer."
+    )
+    parser.add_argument(
+        "--max_grad_norm",
+        type=int,
+        default=1000,
+        help="Maximum gradient norm for clipping.",
+    )
+    parser.add_argument(
+        "--gradient_accumulation_steps",
+        type=int,
+        default=1,
+        help="Gradient accumulation steps.",
+    )
+    parser.add_argument(
+        "--num_warmup_steps", type=int, default=0, help="Number of warmup steps."
+    )
+    parser.add_argument(
+        "--batch_scheduler", action="store_true", help="Use batch scheduler."
+    )
+    parser.add_argument(
+        "--print_freq", type=int, default=100, help="Logging frequency."
+    )
+    parser.add_argument(
+        "--use_amp",
+        action="store_true",
+        help="Use automatic mixed precision for training.",
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="./",
+        help="Directory to save the trained model.",
+    )
+    parser.add_argument(
+        "--seed", type=int, default=42, help="Random seed for reproducibility."
+    )
+    parser.add_argument(
+        "--sampling_num",
+        type=int,
+        default=-1,
+        help="Number of samples used for training. If you want to use all samples, set -1.",
+    )
+    parser.add_argument(
+        "--sampling_frac",
+        type=float,
+        default=-1.0,
+        help="Ratio of samples used for training. If you want to use all samples, set -1.0.",
+    )
+    parser.add_argument(
+        "--checkpoint",
+        type=str,
+        help="Path to the checkpoint file for resuming training.",
+    )
+
+    return parser.parse_args()
+
+
+def preprocess_df(df, cfg, drop_duplicates=True):
+    """
+    Preprocess the input DataFrame for training.
+
+    Args:
+        df (pd.DataFrame): Input DataFrame.
+        cfg (argparse.Namespace): Configuration object.
+
+    Returns:
+        pd.DataFrame: Preprocessed DataFrame.
+    """
+    if "YIELD" in df.columns:
+        # if max yield is 100, then normalize to [0, 1]
+        if df["YIELD"].max() >= 100:
+            df["YIELD"] = df["YIELD"].clip(0, 100) / 100
+    else:
+        df["YIELD"] = None
+
+    for col in ["REACTANT", "PRODUCT", "CATALYST", "REAGENT", "SOLVENT"]:
+        if col not in df.columns:
+            df[col] = None
+        df[col] = df[col].fillna(" ")
+
+    df["REAGENT"] = df["CATALYST"] + "." + df["REAGENT"]
+
+    for col in ["REAGENT", "REACTANT", "PRODUCT"]:
+        df[col] = df[col].apply(lambda x: space_clean(x))
+        df[col] = df[col].apply(lambda x: canonicalize(x) if x != " " else " ")
+        df = df[~df[col].isna()].reset_index(drop=True)
+        df[col] = df[col].apply(lambda x: ".".join(sorted(x.split("."))))
+
+    df["input"] = (
+        "REACTANT:"
+        + df["REACTANT"]
+        + "REAGENT:"
+        + df["REAGENT"]
+        + "PRODUCT:"
+        + df["PRODUCT"]
+    )
+    if drop_duplicates:
+        df = df.loc[df[["input", "YIELD"]].drop_duplicates().index].reset_index(
+            drop=True
+        )
+
+    if cfg.debug:
+        df = df.head(1000)
+
+    return df
+
+
+def preprocess_CN(df):
+    """
+    Preprocess the CN test DataFrame.
+
+    Args:
+        df (pd.DataFrame): Input DataFrame.
+
+    Returns:
+        pd.DataFrame: Preprocessed DataFrame.
+    """
+    df["REACTANT"] = df["REACTANT"].apply(lambda x: ".".join(sorted(x.split("."))))
+    df["REAGENT"] = df["REAGENT"].apply(lambda x: ".".join(sorted(x.split("."))))
+    df["PRODUCT"] = df["PRODUCT"].apply(lambda x: ".".join(sorted(x.split("."))))
+    df["input"] = (
+        "REACTANT:"
+        + df["REACTANT"]
+        + "REAGENT:"
+        + df["REAGENT"]
+        + "PRODUCT:"
+        + df["PRODUCT"]
+    )
+    df["pair"] = df["input"]
+    return df
+
+
+class TrainDataset(Dataset):
+    """
+    Dataset class for training.
+    """
+
+    def __init__(self, cfg, df):
+        self.cfg = cfg
+        self.inputs = df["input"].values
+        self.labels = df["YIELD"].values
+
+    def __len__(self):
+        return len(self.labels)
+
+    def __getitem__(self, item):
+        inputs = prepare_input(self.cfg, self.inputs[item])
+        label = torch.tensor(self.labels[item], dtype=torch.float)
+        return inputs, label
+
+
+def save_checkpoint(state, filename="checkpoint.pth.tar"):
+    """
+    Save model checkpoint.
+
+    Args:
+        state (dict): Checkpoint state.
+        filename (str): Filename to save the checkpoint.
+    """
+    torch.save(state, filename)
+
+
+def train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, cfg):
+    """
+    Training function for one epoch.
+
+    Args:
+        train_loader (DataLoader): DataLoader for training data.
+        model (nn.Module): Model to be trained.
+        criterion (nn.Module): Loss function.
+        optimizer (Optimizer): Optimizer.
+        epoch (int): Current epoch.
+        scheduler (Scheduler): Learning rate scheduler.
+        cfg (argparse.Namespace): Configuration object.
+
+    Returns:
+        float: Average training loss.
+    """
+    model.train()
+    scaler = torch.amp.GradScaler(enabled=cfg.use_amp)
+    losses = AverageMeter()
+    start = time.time()
+
+    for step, (inputs, labels) in enumerate(train_loader):
+        inputs = {k: v.to(cfg.device) for k, v in inputs.items()}
+        labels = labels.to(cfg.device)
+        batch_size = labels.size(0)
+
+        with torch.autocast(cfg.device, enabled=cfg.use_amp):
+            y_preds = model(inputs)
+        loss = criterion(y_preds.view(-1, 1), labels.view(-1, 1))
+
+        if cfg.gradient_accumulation_steps > 1:
+            loss /= cfg.gradient_accumulation_steps
+
+        losses.update(loss.item(), batch_size)
+        scaler.scale(loss).backward()
+
+        grad_norm = torch.nn.utils.clip_grad_norm_(
+            model.parameters(), cfg.max_grad_norm
+        )
+
+        if (step + 1) % cfg.gradient_accumulation_steps == 0:
+            scaler.step(optimizer)
+            scaler.update()
+            optimizer.zero_grad()
+
+            if cfg.batch_scheduler:
+                scheduler.step()
+
+        if step % cfg.print_freq == 0 or step == (len(train_loader) - 1):
+            print(
+                f"Epoch: [{epoch + 1}][{step}/{len(train_loader)}] "
+                f"Elapsed {timeSince(start, float(step + 1) / len(train_loader))} "
+                f"Loss: {losses.val:.4f}({losses.avg:.4f}) "
+                f"Grad: {grad_norm:.4f} "
+                f"LR: {scheduler.get_lr()[0]:.8f}"
+            )
+
+    return losses.avg
+
+
+def valid_fn(valid_loader, model, cfg):
+    """
+    Validation function.
+
+    Args:
+        valid_loader (DataLoader): DataLoader for validation data.
+        model (nn.Module): Model to be validated.
+        cfg (argparse.Namespace): Configuration object.
+
+    Returns:
+        tuple: Validation loss and R^2 score.
+    """
+    model.eval()
+    start = time.time()
+    label_list = []
+    pred_list = []
+
+    for step, (inputs, labels) in enumerate(valid_loader):
+        inputs = {k: v.to(cfg.device) for k, v in inputs.items()}
+        with torch.no_grad():
+            y_preds = model(inputs)
+        label_list.extend(labels.tolist())
+        pred_list.extend(y_preds.tolist())
+
+        if step % cfg.print_freq == 0 or step == (len(valid_loader) - 1):
+            print(
+                f"EVAL: [{step}/{len(valid_loader)}] "
+                f"Elapsed {timeSince(start, float(step + 1) / len(valid_loader))} "
+                f"RMSE Loss: {np.sqrt(mean_squared_error(label_list, pred_list)):.4f} "
+                f"R^2 Score: {r2_score(label_list, pred_list):.4f}"
+            )
+
+    return mean_squared_error(label_list, pred_list), r2_score(label_list, pred_list)
+
+
+def train_loop(train_ds, valid_ds, cfg):
+    """
+    Training loop.
+
+    Args:
+        train_ds (pd.DataFrame): Training data.
+        valid_ds (pd.DataFrame): Validation data.
+    """
+    train_dataset = TrainDataset(cfg, train_ds)
+    valid_dataset = TrainDataset(cfg, valid_ds)
+
+    train_loader = DataLoader(
+        train_dataset,
+        batch_size=cfg.batch_size,
+        shuffle=True,
+        num_workers=cfg.num_workers,
+        pin_memory=True,
+        drop_last=True,
+    )
+    valid_loader = DataLoader(
+        valid_dataset,
+        batch_size=cfg.batch_size,
+        shuffle=False,
+        num_workers=cfg.num_workers,
+        pin_memory=True,
+        drop_last=False,
+    )
+
+    if not cfg.model_name_or_path:
+        model = ReactionT5Yield(cfg, config_path=None, pretrained=True)
+        torch.save(model.config, os.path.join(cfg.output_dir, "config.pth"))
+    else:
+        model = ReactionT5Yield(
+            cfg,
+            config_path=os.path.join(cfg.model_name_or_path, "config.pth"),
+            pretrained=False,
+        )
+        torch.save(model.config, os.path.join(cfg.output_dir, "config.pth"))
+        pth_files = glob.glob(os.path.join(cfg.model_name_or_path, "*.pth"))
+        for pth_file in pth_files:
+            state = torch.load(
+                pth_file, map_location=torch.device("cpu"), weights_only=False
+            )
+            try:
+                model.load_state_dict(state)
+                break
+            except:
+                pass
+    model.to(cfg.device)
+
+    optimizer_parameters = get_optimizer_params(
+        model, encoder_lr=cfg.lr, decoder_lr=cfg.lr, weight_decay=cfg.weight_decay
+    )
+    optimizer = AdamW(optimizer_parameters, lr=cfg.lr, eps=cfg.eps, betas=(0.9, 0.999))
+
+    num_train_steps = int(len(train_ds) / cfg.batch_size * cfg.epochs)
+    scheduler = get_linear_schedule_with_warmup(
+        optimizer,
+        num_warmup_steps=cfg.num_warmup_steps,
+        num_training_steps=num_train_steps,
+    )
+
+    criterion = nn.MSELoss(reduction="mean")
+    best_loss = float("inf")
+    start_epoch = 0
+    es_count = 0
+
+    if cfg.checkpoint:
+        checkpoint = torch.load(cfg.checkpoint)
+        model.load_state_dict(checkpoint["state_dict"])
+        optimizer.load_state_dict(checkpoint["optimizer"])
+        scheduler.load_state_dict(checkpoint["scheduler"])
+        best_loss = checkpoint["loss"]
+        start_epoch = checkpoint["epoch"] + 1
+        es_count = checkpoint["es_count"]
+        del checkpoint
+
+    for epoch in range(start_epoch, cfg.epochs):
+        start_time = time.time()
+
+        avg_loss = train_fn(
+            train_loader, model, criterion, optimizer, epoch, scheduler, cfg
+        )
+        val_loss, val_r2_score = valid_fn(valid_loader, model, cfg)
+
+        elapsed = time.time() - start_time
+
+        cfg.logger.info(
+            f"Epoch {epoch + 1} - avg_train_loss: {avg_loss:.4f}  val_rmse_loss: {val_loss:.4f}  val_r2_score: {val_r2_score:.4f}  time: {elapsed:.0f}s"
+        )
+
+        if val_loss < best_loss:
+            es_count = 0
+            best_loss = val_loss
+            cfg.logger.info(
+                f"Epoch {epoch + 1} - Save Lowest Loss: {best_loss:.4f} Model"
+            )
+            torch.save(
+                model.state_dict(),
+                os.path.join(
+                    cfg.output_dir,
+                    f"{cfg.pretrained_model_name_or_path.split('/')[-1]}_best.pth",
+                ),
+            )
+        else:
+            es_count += 1
+            if es_count >= cfg.patience:
+                print("Early stopping")
+                break
+
+        save_checkpoint(
+            {
+                "epoch": epoch,
+                "state_dict": model.state_dict(),
+                "optimizer": optimizer.state_dict(),
+                "scheduler": scheduler.state_dict(),
+                "loss": best_loss,
+                "es_count": es_count,
+            },
+            filename=os.path.join(cfg.output_dir, "checkpoint.pth.tar"),
+        )
+
+    torch.cuda.empty_cache()
+    gc.collect()
+
+
+if __name__ == "__main__":
+    CFG = parse_args()
+    CFG.batch_scheduler = True
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    CFG.device = device
+    if not os.path.exists(CFG.output_dir):
+        os.makedirs(CFG.output_dir)
+    seed_everything(seed=CFG.seed)
+
+    train = preprocess_df(
+        filter_out(pd.read_csv(CFG.train_data_path), ["YIELD", "REACTANT", "PRODUCT"]),
+        CFG,
+    )
+    valid = preprocess_df(
+        filter_out(pd.read_csv(CFG.valid_data_path), ["YIELD", "REACTANT", "PRODUCT"]),
+        CFG,
+    )
+
+    if CFG.CN_test_data_path:
+        train_copy = preprocess_CN(train.copy())
+        CN_test = preprocess_CN(pd.read_csv(CFG.CN_test_data_path))
+
+        print(len(train))
+        train = train[~train_copy["pair"].isin(CN_test["pair"])].reset_index(drop=True)
+        print(len(train))
+
+    train["pair"] = train["input"] + " - " + train["YIELD"].astype(str)
+    valid["pair"] = valid["input"] + " - " + valid["YIELD"].astype(str)
+    valid = valid[~valid["pair"].isin(train["pair"])].reset_index(drop=True)
+
+    if CFG.sampling_num > 0:
+        train = train.sample(n=CFG.sampling_num, random_state=CFG.seed).reset_index(
+            drop=True
+        )
+    elif CFG.sampling_frac > 0:
+        train = train.sample(frac=CFG.sampling_frac, random_state=CFG.seed).reset_index(
+            drop=True
+        )
+
+    train.to_csv("train.csv", index=False)
+    valid.to_csv("valid.csv", index=False)
+
+    if CFG.test_data_path:
+        test = filter_out(
+            pd.read_csv(CFG.test_data_path), ["YIELD", "REACTANT", "PRODUCT"]
+        )
+        test = preprocess_df(test, CFG)
+        test["pair"] = test["input"] + " - " + test["YIELD"].astype(str)
+        test = test[~test["pair"].isin(train["pair"])].reset_index(drop=True)
+        test = test.drop_duplicates(subset=["pair"]).reset_index(drop=True)
+        test.to_csv("test.csv", index=False)
+
+    LOGGER = get_logger(os.path.join(CFG.output_dir, "train"))
+    CFG.logger = LOGGER
+
+    # load tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(
+        os.path.abspath(CFG.model_name_or_path)
+        if os.path.exists(CFG.model_name_or_path)
+        else CFG.model_name_or_path,
+        return_tensors="pt",
+    )
+    tokenizer = add_new_tokens(
+        tokenizer,
+        Path(__file__).resolve().parent.parent / "data" / "additional_tokens.txt",
+    )
+
+    tokenizer.add_special_tokens(
+        {
+            "additional_special_tokens": tokenizer.additional_special_tokens
+            + ["REACTANT:", "PRODUCT:", "REAGENT:"]
+        }
+    )
+    tokenizer.save_pretrained(CFG.output_dir)
+    CFG.tokenizer = tokenizer
+
+    train_loop(train, valid, CFG)

task_yield/visualize_embedding.ipynb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5187f4fb3a6d7fc19873902ca53a1699152cdc5cb50e79bd946bb430b7be154d
+size 10491206

utils.py

@@ -0,0 +1,277 @@
+import math
+import os
+import pickle
+import random
+import time
+
+import numpy as np
+import torch
+from rdkit import Chem
+
+
+def seed_everything(seed=42):
+    random.seed(seed)
+    os.environ["PYTHONHASHSEED"] = str(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.backends.cudnn.deterministic = True
+
+
+def space_clean(row):
+    row = row.replace(". ", "").replace(" .", "").replace("  ", " ")
+    return row
+
+
+def canonicalize(smiles):
+    try:
+        new_smiles = Chem.MolToSmiles(Chem.MolFromSmiles(smiles), canonical=True)
+    except:
+        new_smiles = None
+    return new_smiles
+
+
+def canonicalize_str(smiles):
+    """Try to canonicalize the molecule, return empty string if fails."""
+    if "%" in smiles:
+        return smiles
+    else:
+        try:
+            return canonicalize(smiles)
+        except:
+            return ""
+
+
+def uncanonicalize(smiles):
+    try:
+        new_smiles = []
+        for smiles_i in smiles.split("."):
+            mol = Chem.MolFromSmiles(smiles_i)
+            atom_indices = list(range(mol.GetNumAtoms()))
+            random.shuffle(atom_indices)
+            new_smiles_i = Chem.MolToSmiles(
+                mol, rootedAtAtom=atom_indices[0], canonical=False
+            )
+            new_smiles.append(new_smiles_i)
+        smiles = ".".join(new_smiles)
+    except:
+        smiles = None
+    return smiles
+
+
+def remove_atom_mapping(smi):
+    mol = Chem.MolFromSmiles(smi)
+    [a.SetAtomMapNum(0) for a in mol.GetAtoms()]
+    smi = Chem.MolToSmiles(mol, canonical=True)
+    return canonicalize(smi)
+
+
+def get_logger(filename="train"):
+    from logging import INFO, FileHandler, Formatter, StreamHandler, getLogger
+
+    logger = getLogger(__name__)
+    logger.setLevel(INFO)
+    handler1 = StreamHandler()
+    handler1.setFormatter(Formatter("%(message)s"))
+    handler2 = FileHandler(filename=f"{filename}.log")
+    handler2.setFormatter(Formatter("%(message)s"))
+    logger.addHandler(handler1)
+    logger.addHandler(handler2)
+    return logger
+
+
+class AverageMeter(object):
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        self.val = 0
+        self.avg = 0
+        self.sum = 0
+        self.count = 0
+
+    def update(self, val, n=1):
+        self.val = val
+        self.sum += val * n
+        self.count += n
+        self.avg = self.sum / self.count
+
+
+def asMinutes(s):
+    m = math.floor(s / 60)
+    s -= m * 60
+    return "%dm %ds" % (m, s)
+
+
+def timeSince(since, percent):
+    now = time.time()
+    s = now - since
+    es = s / (percent)
+    rs = es - s
+    return "%s (remain %s)" % (asMinutes(s), asMinutes(rs))
+
+
+def get_optimizer_params(model, encoder_lr, decoder_lr, weight_decay=0.0):
+    no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
+    optimizer_parameters = [
+        {
+            "params": [
+                p
+                for n, p in model.model.named_parameters()
+                if not any(nd in n for nd in no_decay)
+            ],
+            "lr": encoder_lr,
+            "weight_decay": weight_decay,
+        },
+        {
+            "params": [
+                p
+                for n, p in model.model.named_parameters()
+                if any(nd in n for nd in no_decay)
+            ],
+            "lr": encoder_lr,
+            "weight_decay": 0.0,
+        },
+        {
+            "params": [p for n, p in model.named_parameters() if "model" not in n],
+            "lr": decoder_lr,
+            "weight_decay": 0.0,
+        },
+    ]
+    return optimizer_parameters
+
+
+def to_cpu(obj):
+    if torch.is_tensor(obj):
+        return obj.to("cpu")
+    elif isinstance(obj, dict):
+        return {k: to_cpu(v) for k, v in obj.items()}
+    elif (
+        isinstance(obj, list)
+        or isinstance(obj, tuple)
+        or isinstance(obj, set)
+        or isinstance(obj, torch.Tensor)
+    ):
+        return [to_cpu(v) for v in obj]
+    else:
+        return obj
+
+
+def get_accuracy_score(eval_preds, cfg):
+    preds, labels = eval_preds
+    if isinstance(preds, tuple):
+        preds = preds[0]
+
+    decoded_preds = cfg.tokenizer.batch_decode(preds, skip_special_tokens=True)
+
+    labels = np.where(labels != -100, labels, cfg.tokenizer.pad_token_id)
+    decoded_labels = cfg.tokenizer.batch_decode(labels, skip_special_tokens=True)
+
+    decoded_preds = [
+        canonicalize_str(pred.strip().replace(" ", "")) for pred in decoded_preds
+    ]
+    decoded_labels = [
+        [canonicalize_str(label.strip().replace(" ", ""))] for label in decoded_labels
+    ]
+
+    score = 0
+    for i in range(len(decoded_preds)):
+        if decoded_preds[i] == decoded_labels[i][0]:
+            score += 1
+    score /= len(decoded_preds)
+    return {"accuracy": score}
+
+
+def get_accuracy_score_multitask(eval_preds, cfg):
+    preds, labels = eval_preds
+    if isinstance(preds, tuple):
+        preds = preds[0]
+
+    special_tokens = cfg.tokenizer.special_tokens_map
+    special_tokens = [
+        special_tokens["eos_token"],
+        special_tokens["pad_token"],
+        special_tokens["unk_token"],
+    ] + list(
+        set(special_tokens["additional_special_tokens"])
+        - set(
+            [
+                "0%",
+                "10%",
+                "20%",
+                "30%",
+                "40%",
+                "50%",
+                "60%",
+                "70%",
+                "80%",
+                "90%",
+                "100%",
+            ]
+        )
+    )
+
+    decoded_preds = cfg.tokenizer.batch_decode(preds, skip_special_tokens=False)
+    for special_token in special_tokens:
+        decoded_preds = [pred.replace(special_token, "") for pred in decoded_preds]
+
+    labels = np.where(labels != -100, labels, cfg.tokenizer.pad_token_id)
+    decoded_labels = cfg.tokenizer.batch_decode(labels, skip_special_tokens=False)
+    for special_token in special_tokens:
+        decoded_labels = [pred.replace(special_token, "") for pred in decoded_labels]
+
+    decoded_preds = [
+        canonicalize_str(pred.strip().replace(" ", "")) for pred in decoded_preds
+    ]
+    decoded_labels = [
+        [canonicalize_str(label.strip().replace(" ", ""))] for label in decoded_labels
+    ]
+
+    score = 0
+    for i in range(len(decoded_preds)):
+        if decoded_preds[i] == decoded_labels[i][0]:
+            score += 1
+    score /= len(decoded_preds)
+    return {"accuracy": score}
+
+
+def preprocess_dataset(examples, cfg):
+    inputs = examples["input"]
+    targets = examples[cfg.target_column]
+    model_inputs = cfg.tokenizer(
+        inputs, max_length=cfg.input_max_length, truncation=True
+    )
+    labels = cfg.tokenizer(targets, max_length=cfg.target_max_length, truncation=True)
+    model_inputs["labels"] = labels["input_ids"]
+    return model_inputs
+
+
+def filter_out(df, col_names):
+    for col_name in col_names:
+        df = df[~df[col_name].isna()].reset_index(drop=True)
+    return df
+
+
+def save_pickle(path: str, contents):
+    """Saves contents to a pickle file."""
+    with open(path, "wb") as f:
+        pickle.dump(contents, f)
+
+
+def load_pickle(path: str):
+    """Loads contents from a pickle file."""
+    with open(path, "rb") as f:
+        return pickle.load(f)
+
+
+def add_new_tokens(tokenizer, file_path):
+    """
+    Adds new tokens to the tokenizer from a file.
+    The file should contain one token per line.
+    """
+    with open(file_path, "r") as f:
+        new_tokens = [line.strip() for line in f if line.strip()]
+
+    tokenizer.add_tokens(new_tokens)
+
+    return tokenizer