diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..c20a366d71c80ada7069b82a1fe429281f3fd156 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +fourm/utils/clip/bpe_simple_vocab_16e6.txt.gz filter=lfs diff=lfs merge=lfs -text diff --git a/fourm/data/__init__.py b/fourm/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..21a57787b5cdf7c2df14ee73a2f4db7ff11e66af --- /dev/null +++ b/fourm/data/__init__.py @@ -0,0 +1,4 @@ +from .image_augmenter import * +from .modality_transforms import * +from .unified_datasets import build_fm_pretraining_dataset, build_fm_transfer_dataset, build_wds_fm_pretraining_dataloader, build_wds_divae_dataloader, build_huggingface_pretraining_dataloader, build_mixture_dataloader +from .pretrain_utils import * diff --git a/fourm/data/dataset_utils.py b/fourm/data/dataset_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9b643ef6d7ebcf6a8c67d6c62d5dfd0bbf673af2 --- /dev/null +++ b/fourm/data/dataset_utils.py @@ -0,0 +1,85 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np +from torch.utils.data import Dataset + + +class RepeatedDatasetWrapper(Dataset): + def __init__(self, original_dataset, num_repeats): + """ + Dataset wrapper that repeats the original dataset n times. + + Args: + original_dataset (torch.utils.data.Dataset): The original dataset to be repeated. + num_repeats (int): The number of times the dataset should be repeated. + """ + self.original_dataset = original_dataset + self.num_repeats = num_repeats + + def __getitem__(self, index): + """ + Retrieve the item at the given index. + + Args: + index (int): The index of the item to be retrieved. + """ + original_index = index % len(self.original_dataset) + return self.original_dataset[original_index] + + def __len__(self): + """ + Get the length of the dataset after repeating it n times. + + Returns: + int: The length of the dataset. + """ + return len(self.original_dataset) * self.num_repeats + + +class SubsampleDatasetWrapper(Dataset): + def __init__(self, original_dataset, dataset_size, seed=0, return_orig_idx=False): + """ + Dataset wrapper that randomly subsamples the original dataset. + + Args: + original_dataset (torch.utils.data.Dataset): The original dataset to be subsampled. + dataset_size (int): The size of the subsampled dataset. + seed (int): The seed to use for selecting the subset of indices of the original dataset. + return_orig_idx (bool): Whether to return the original index of the item in the original dataset. + """ + self.original_dataset = original_dataset + self.dataset_size = dataset_size or len(original_dataset) + self.return_orig_idx = return_orig_idx + np.random.seed(seed) + self.indices = np.random.permutation(len(self.original_dataset))[:self.dataset_size] + + def __getitem__(self, index): + """ + Retrieve the item at the given index. + + Args: + index (int): The index of the item to be retrieved. + """ + original_index = self.indices[index] + sample = self.original_dataset[original_index] + return sample, original_index if self.return_orig_idx else sample + + def __len__(self): + """ + Get the length of the dataset after subsampling it. + + Returns: + int: The length of the dataset. + """ + return len(self.indices) diff --git a/fourm/data/image_augmenter.py b/fourm/data/image_augmenter.py new file mode 100644 index 0000000000000000000000000000000000000000..3e1edcccfca31f40ae293cd26749cd2087417cef --- /dev/null +++ b/fourm/data/image_augmenter.py @@ -0,0 +1,186 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import random +from abc import ABC, abstractmethod + +import numpy as np +import torchvision + +from fourm.utils import to_2tuple + + +class AbstractImageAugmenter(ABC): + """Abstract class for image augmenters. + """ + + @abstractmethod + def __call__(self, mod_dict, crop_settings): + pass + + +class RandomCropImageAugmenter(AbstractImageAugmenter): + + def __init__(self, target_size=224, hflip=0.5, crop_scale=(0.2, 1.0), crop_ratio=(0.75, 1.3333), main_domain='rgb'): + + self.target_size = to_2tuple(target_size) + self.hflip = hflip + self.crop_scale = crop_scale + self.crop_ratio = crop_ratio + self.main_domain = main_domain + + def __call__(self, mod_dict, crop_settings): + + if crop_settings is not None: + raise ValueError("Crop settings are provided but not used by this augmenter.") + + image = mod_dict[self.main_domain] if self.main_domain is not None else mod_dict[list(mod_dict.keys())[0]] + # With torchvision 0.13+, can also be: orig_size = TF.get_dimensions(image) + orig_width, orig_height = image.size + orig_size = (orig_height, orig_width) + + top, left, h, w = torchvision.transforms.RandomResizedCrop.get_params( + image, scale=self.crop_scale, ratio=self.crop_ratio + ) + crop_coords = top, left, h, w + flip = random.random() < self.hflip + rand_aug_idx = None + + return crop_coords, flip, orig_size, self.target_size, rand_aug_idx + +class NoImageAugmenter(AbstractImageAugmenter): # this is for non-image modalities like poses where we don't do any augs, e.g. during tokenization + + def __init__(self, no_aug=True, main_domain='human_poses'): + self.target_size = None #to_2tuple(target_size) + self.no_aug = no_aug + self.main_domain = main_domain + + def __call__(self, mod_dict, crop_settings): + # # With torchvision 0.13+, can also be: orig_size = TF.get_dimensions(image) + orig_size = (224, 224) + + rand_aug_idx = 0 + top, left, h, w, flip = 0, 0, 224, 224, 0 + crop_coords = (top, left, h, w) + + return crop_coords, flip, orig_size, self.target_size, rand_aug_idx + +class PreTokenizedImageAugmenter(AbstractImageAugmenter): + + def __init__(self, target_size, no_aug=False, main_domain='rgb'): + self.target_size = to_2tuple(target_size) + self.no_aug = no_aug + self.main_domain = main_domain + + def __call__(self, mod_dict, crop_settings): + # With torchvision 0.13+, can also be: orig_size = TF.get_dimensions(image) + if self.main_domain in mod_dict and 'tok' not in self.main_domain: + image = mod_dict[self.main_domain] if self.main_domain is not None else mod_dict[list(mod_dict.keys())[0]] + orig_width, orig_height = image.size + orig_size = (orig_height, orig_width) + else: + orig_size = None + + rand_aug_idx = 0 if self.no_aug else np.random.randint(len(crop_settings)) + top, left, h, w, flip = crop_settings[rand_aug_idx] + crop_coords = (top, left, h, w) + + return crop_coords, flip, orig_size, self.target_size, rand_aug_idx + + +class CenterCropImageAugmenter(AbstractImageAugmenter): + def __init__(self, target_size, hflip=0.0, main_domain='rgb'): + self.target_size = to_2tuple(target_size) + self.hflip = hflip + self.main_domain = main_domain + + def __call__(self, mod_dict, crop_settings=None): + image = mod_dict[self.main_domain] if self.main_domain is not None else mod_dict[list(mod_dict.keys())[0]] + orig_width, orig_height = image.size + orig_size = (orig_height, orig_width) + + if orig_height > orig_width: + h = w = orig_width + top = (orig_height - orig_width) // 2 + left = 0 + else: + h = w = orig_height + top = 0 + left = (orig_width - orig_height) // 2 + + crop_coords = (top, left, h, w) + flip = random.random() < self.hflip + rand_aug_idx = None + + return crop_coords, flip, orig_size, self.target_size, rand_aug_idx + + +class PaddingImageAugmenter(AbstractImageAugmenter): + def __init__(self, target_size, hflip=0.0, main_domain='rgb'): + self.target_size = to_2tuple(target_size) + self.hflip = hflip + self.main_domain = main_domain + + def __call__(self, mod_dict, crop_settings): + image = mod_dict[self.main_domain] if self.main_domain is not None else mod_dict[list(mod_dict.keys())[0]] + orig_width, orig_height = image.size + orig_size = (orig_height, orig_width) + + h = w = max(orig_width, orig_height) + top = left = 0 + crop_coords = (top, left, h, w) + flip = random.random() < self.hflip + rand_aug_idx = None + + return crop_coords, flip, orig_size, self.target_size, rand_aug_idx + + +class ScaleJitteringImageAugmenter(AbstractImageAugmenter): + def __init__(self, target_size, hflip=0.0, scale=(0.1, 2.0), main_domain='rgb'): + self.target_size = to_2tuple(target_size) + self.hflip = hflip + self.scale = scale + self.main_domain = main_domain + + def scale_jitter(self, orig_height, orig_width): + rand_scale = np.random.uniform(self.scale[0], self.scale[1]) + max_hw = max(orig_height, orig_width) + h = w = round(max_hw / rand_scale) + top = round(max(0, np.random.uniform(0, orig_height - h))) + left = round(max(0, np.random.uniform(0, orig_width - w))) + + return top, left, h, w + + def __call__(self, mod_dict, crop_settings): + + if crop_settings is not None: + raise ValueError("Crop settings are provided but not used by this augmenter.") + + image = mod_dict[self.main_domain] if self.main_domain is not None else mod_dict[list(mod_dict.keys())[0]] + # With torchvision 0.13+, can also be: orig_size = TF.get_dimensions(image) + orig_width, orig_height = image.size + orig_size = (orig_height, orig_width) + + crop_coords = self.scale_jitter(orig_height, orig_width) + flip = random.random() < self.hflip + rand_aug_idx = None + + return crop_coords, flip, orig_size, self.target_size, rand_aug_idx + + +class EmptyAugmenter(AbstractImageAugmenter): + def __init__(self): + pass + + def __call__(self, mod_dict, crop_settings): + return None, None, None, None, None \ No newline at end of file diff --git a/fourm/data/masking.py b/fourm/data/masking.py new file mode 100644 index 0000000000000000000000000000000000000000..860861ab71aa25c962677bf67fd1730054a5a34b --- /dev/null +++ b/fourm/data/masking.py @@ -0,0 +1,747 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +import random +from typing import Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +from einops import rearrange +from tokenizers import Tokenizer +from torch.distributions import Dirichlet + +from fourm.data.modality_transforms import get_transform_key +from fourm.utils import to_2tuple +from fourm.utils.tokenizer import get_sentinel_to_id_mapping + + +def sample_cosine(min_val: float = 0, max_val: float =1) -> float: + """Sample a value from a cosine distribution between min_val and max_val + + Args: + min_val: Minimum value + max_val: Maximum value + + Returns: + Sampled value + """ + + return min_val + 0.5 * (max_val - min_val) * (1 + math.cos(math.pi * random.uniform(0, 1))) + + +def sample_uniform(min_val: float = 0, max_val: float =1) -> float: + """Sample a value from a uniform distribution between min_val and max_val + + Args: + min_val: Minimum value + max_val: Maximum value + + Returns: + Sampled value + """ + + return random.uniform(min_val, max_val) + + +def simple_span_masking(sequence: List[int], sentinel_to_id: Dict[int, int], keep_prob: float) -> Tuple[List[int], List[int]]: + """Span masking for a sequence + + Args: + sequence: Sequence to mask + sentinel_to_id: Mapping from sentinel to id + keep_prob: Probability of keeping a token + + Returns: + Masked input sequence and masked target sequence + """ + sequence_length = len(sequence) + # 0 for keep, 1 for mask + masks = torch.where(torch.rand(sequence_length) <= keep_prob, 0, 1).bool().tolist() + + input_sequence = [] + target_sequence = [] + + prev_mask = False + sentinel_count = 0 + for token, mask in zip(sequence, masks): + if mask: + if not prev_mask: + sentinel_count += 1 + input_sequence.append(sentinel_to_id[sentinel_count]) + target_sequence.append(sentinel_to_id[sentinel_count]) + prev_mask = True + target_sequence.append(token) + else: + prev_mask = False + input_sequence.append(token) + + target_sequence.append(sentinel_to_id[sentinel_count + 1]) + return input_sequence, target_sequence + + +def chunk_span_masking(sequence_chunks: List[List[int]], sentinel_to_id: Dict[int, int], keep_prob: float) -> Tuple[List[int], List[int]]: + """Span masking where masking is performed at the chunk level. + + Args: + sequence_chunks: Sequence chunks to mask + sentinel_to_id: Mapping from sentinel to id + keep_prob: Probability of keeping a token + + Returns: + Masked input sequence and masked target sequence + """ + chunk_length = len(sequence_chunks) + # 0 for keep, 1 for mask + masks = torch.where(torch.rand(chunk_length) <= keep_prob, 0, 1).bool().tolist() + + input_sequence = [] + target_sequence = [] + + prev_mask = False + sentinel_count = 0 + for chunk, mask in zip(sequence_chunks, masks): + if mask: + if not prev_mask: + sentinel_count += 1 + input_sequence.append(sentinel_to_id[sentinel_count]) + target_sequence.append(sentinel_to_id[sentinel_count]) + prev_mask = True + target_sequence.extend(chunk) + else: + prev_mask = False + input_sequence.extend(chunk) + + target_sequence.append(sentinel_to_id[sentinel_count + 1]) + return input_sequence, target_sequence + + + +class UnifiedMasking(object): + def __init__(self, + modality_info: Dict, + text_tokenizer: Optional[Tokenizer], + input_tokens_range: Union[int, Tuple[int, int]], + target_tokens_range: Optional[Union[int, Tuple[int, int]]], + max_tries: int = 100, + sampling_weights: Optional[List[float]] = None,): + """Performs masking on a dict of modalities (both image based and sequence based modalities) + + Args: + modality_info: Dict with the modalities and their corresponding information + text_tokenizer: Tokenizer to use for text modalities + input_tokens_range: Range of number of tokens to mask in the input + target_tokens_range: Range of number of tokens to mask in the target + max_tries: Maximum number of tries to find a valid token budgets + sampling_weights: Sampling weights for the mixture of Dirichlet distributions + """ + self.input_tokens_range = to_2tuple(input_tokens_range) + self.target_tokens_range = to_2tuple(target_tokens_range) if target_tokens_range is not None else None + self.modality_info = modality_info + self.num_modalities = len(modality_info) + self.max_tries = max_tries + self.min_tokens = torch.tensor([mod['min_tokens'] for mod in modality_info.values()]) + self.max_tokens = torch.tensor([mod['max_tokens'] for mod in modality_info.values()]) + self.mod_is_img = torch.tensor([mod['type'] == 'img' for mod in modality_info.values()]) + + # Dirichlet sampling (supports a mixture of multiple Dirichlet distributions) + eps = 1e-9 + input_alphas = torch.tensor([mod["input_alphas"] for mod in modality_info.values()]) + input_alphas = rearrange(input_alphas, "nmod nmix -> nmix nmod") + self.input_dirichlets = [Dirichlet(torch.clamp(input_alpha, min=eps)) for input_alpha in input_alphas] + target_alphas = torch.tensor([mod["target_alphas"] for mod in modality_info.values()]) + target_alphas = rearrange(target_alphas, "nmod nmix -> nmix nmod") + self.target_dirichlets = [Dirichlet(torch.clamp(target_alpha, min=eps)) for target_alpha in target_alphas] + assert(len(self.input_dirichlets) == len(self.target_dirichlets)) + self.num_dirichlets = len(self.input_dirichlets) + if sampling_weights is not None: + assert len(sampling_weights) == self.num_dirichlets + self.sampling_weights = torch.tensor(sampling_weights) + else: + self.sampling_weights = None + + self.text_tokenizer = text_tokenizer + self.keep_prob_decay_factor = 0.9 + self.sentinel_to_id = get_sentinel_to_id_mapping(text_tokenizer) + self.sentinel_ids = set(self.sentinel_to_id.values()) + self.pad_id = text_tokenizer.token_to_id("[PAD]") + self.eos_id = text_tokenizer.token_to_id("[EOS]") + + def input_token_budget(self, num_input_tokens, dir_idx=0): + """Sample a token budget for the input + + Args: + num_input_tokens: Number of tokens in the input + + Returns: + Token budget for the input + """ + # Get the number of tokens for each modality + for i in range(self.max_tries): + input_token_budget = (self.input_dirichlets[dir_idx].sample() * num_input_tokens).floor().int() + diff = num_input_tokens - input_token_budget.sum() + # Adds the remaining tokens by sampling from the Dirichlet and taking the argmax + # This avoids adding tokens to modalities that shouldn't be sampled (i.e. with alphas ~=0) + input_token_budget += torch.bincount(self.input_dirichlets[dir_idx].sample_n(diff).argmax(dim=-1), minlength=len(input_token_budget)) + + # If token budget is over max tokens for a given modality, set it to max + input_token_budget = torch.clamp(input_token_budget, max=self.max_tokens) + + if (input_token_budget >= self.min_tokens).all(): + return input_token_budget.tolist() + + print(f"More than max tries for input!") + return input_token_budget.tolist() + + def target_token_budget(self, input_token_budget, num_target_tokens, dir_idx=0): + """Sample a token budget for the target + + Args: + input_token_budget: Token budget for the input + num_target_tokens: Number of tokens in the target + + Returns: + Token budget for the target + """ + # We don't reduce the number of tokens for sequence based tasks + max_tokens_remaining = torch.where(self.mod_is_img, self.max_tokens - torch.tensor(input_token_budget), self.max_tokens) + max_tokens_remaining = torch.max(self.min_tokens, max_tokens_remaining) + for i in range(self.max_tries): + target_token_budget = (self.target_dirichlets[dir_idx].sample() * num_target_tokens).floor().int() + diff = num_target_tokens - target_token_budget.sum() + # Adds the remaining tokens by sampling from the Dirichlet and taking the argmax + # This avoids adding tokens to modalities that shouldn't be sampled (i.e. with alphas ~=0) + target_token_budget += torch.bincount(self.target_dirichlets[dir_idx].sample_n(diff).argmax(dim=-1), minlength=len(target_token_budget)) + + # If token budget is over max tokens for a given modality, set it to max + target_token_budget = torch.clamp(target_token_budget, max=max_tokens_remaining) + + if (target_token_budget >= self.min_tokens).all(): + return target_token_budget.tolist() + + print(f"More than max tries for target!") + return target_token_budget.tolist() + + def image_mask(self, tensor: torch.Tensor, num_tokens: int, input_budget: int, target_budget: int): + """Applies input and target masking to an image tensor + + Args: + tensor: Image tensor + num_tokens: Number of tokens in the tensor + input_budget: Token budget for the input + target_budget: Token budget for the target + + Returns: + Dictionary containing the masked image tensor, the input mask, the target mask, and the decoder attention mask + """ + noise = torch.rand(num_tokens) + ids_shuffle = torch.argsort(noise, dim=0) + + input_mask = torch.ones(num_tokens, dtype=torch.bool) + input_mask[:input_budget] = 0 + input_mask = torch.gather(input_mask, dim=0, index=ids_shuffle) + + if target_budget is None: + target_mask = ~input_mask + else: + target_mask = torch.ones(num_tokens, dtype=torch.bool) + target_mask[input_budget:input_budget + target_budget] = 0 + target_mask = torch.gather(target_mask, dim=0, index=ids_shuffle) + + decoder_attention_mask = torch.zeros(num_tokens, dtype=torch.int) + first_mask_token = torch.argmin(target_mask + torch.arange(target_mask.shape[0], device=target_mask.device) * 1e-6) + decoder_attention_mask[first_mask_token] = (~target_mask).sum() # Equiv. to target budget + + return {"tensor": tensor, "input_mask": input_mask, "target_mask": target_mask, "decoder_attention_mask": decoder_attention_mask} + + def sequence_token_mask(self, sequence_ids: str, max_tokens: int, input_budget: int, target_budget: int, keep_scheme: str, vocab_offset: int): + """Applies input and target masking to a sequence of tokens (e.g. DINOv2 global tokens) + The keep probability is sampled from a cosine schedule and does not depend on the number of tokens in the sequence. + If the keep probability results in a sequence that is too long, then it is lowered until the sequence is short enough. + + Args: + sequence_ids: Sequence ids + max_tokens: Maximum number of tokens in the sequence + input_budget: Token budget for the input + target_budget: Token budget for the target + keep_scheme: Scheme for sampling the keep probability + vocab_offset: Offset to avoid overlap with sentinel tokens + + Returns: + Dictionary containing the masked sequence tensor, the input mask, the target mask, and the decoder attention mask + """ + seq_ids = sequence_ids + seq_ids = seq_ids + vocab_offset # Avoid overlap with sentinel tokens (needs to be substracted after decoding) + + # If input budget is 0, treat it as if the whole sequence is completely masked + if input_budget == 0: + keep_prob = 0. + input_seq_ids = [] + _, target_seq_ids = simple_span_masking(seq_ids, self.sentinel_to_id, keep_prob) + else: + if keep_scheme == 'random': + keep_prob = sample_uniform(0, 1) + elif keep_scheme == 'all': + keep_prob = 1.0 + elif keep_scheme == 'binary': + keep_prob = random.choice([0., 1.]) + else: + raise ValueError(f"Invalid keep scheme for sequence masking: {keep_scheme}") + + input_seq_ids, target_seq_ids = simple_span_masking(seq_ids, self.sentinel_to_id, keep_prob) + # Keep lowering the keep_prob while we are over-budget + while len(input_seq_ids) > input_budget: + keep_prob = keep_prob * self.keep_prob_decay_factor + input_seq_ids, target_seq_ids = simple_span_masking(seq_ids, self.sentinel_to_id, keep_prob) + + # Span masking can add up to (max_tokens + 1) * 2 tokens for input + target + max_length = (max_tokens + 1) * 2 + tensor = torch.ones(max_length, dtype=torch.int) * self.pad_id + input_mask = torch.ones(max_length, dtype=torch.bool) + target_mask = torch.ones(max_length, dtype=torch.bool) + decoder_attention_mask = torch.zeros(max_length, dtype=torch.int) + + # Set input and input mask + tensor[:len(input_seq_ids)] = torch.tensor(input_seq_ids, dtype=torch.int) + input_mask[:len(input_seq_ids)] = 0 + + if target_budget is None or len(target_seq_ids) <= target_budget: + tensor[input_budget:input_budget + len(target_seq_ids)] = torch.tensor(target_seq_ids, dtype=torch.int) + target_mask[input_budget:input_budget + len(target_seq_ids)] = 0 + decoder_attention_mask[input_budget:input_budget + len(target_seq_ids)] = 1 + else: + # Randomly choose sentinel token. + sentinel_indices = [i for i, token_id in enumerate(target_seq_ids) if token_id in self.sentinel_ids] + # If there is more than 1 sentinel, avoid sampling the very last one which indicates the end of the sequence + chosen_sentinel = np.random.randint(max(1, len(sentinel_indices) - 1)) + # If length starting at this token g.t. budget, truncate until budget is reached + if len(target_seq_ids) - sentinel_indices[chosen_sentinel] >= target_budget: + target_seq_ids = target_seq_ids[sentinel_indices[chosen_sentinel]:sentinel_indices[chosen_sentinel] + target_budget] + # Otherwise, select earliest sentinel token such that we don't go over budget + # Note: We could also use the randomly chosen sentinel token, but that would waste budget + else: + for idx in sentinel_indices: + if len(target_seq_ids) - idx <= target_budget: + target_seq_ids = target_seq_ids[idx:] + break + + tensor[input_budget:input_budget + len(target_seq_ids)] = torch.tensor(target_seq_ids, dtype=torch.int) + target_mask[input_budget:input_budget + len(target_seq_ids)] = 0 + decoder_attention_mask[input_budget:input_budget + len(target_seq_ids)] = 1 + + return {"tensor": tensor, "input_mask": input_mask, "target_mask": target_mask, "decoder_attention_mask": decoder_attention_mask} + + def sequence_mask(self, sequence: Union[str, List[str]], max_tokens: int, input_budget: int, target_budget: int, keep_scheme: str): + """Applies input and target masking to a sequence + + The keep probability is sampled from a cosine schedule and does not depend on the number of tokens in the sequence. + If the keep probability results in a sequence that is too long, then it is lowered until the sequence is short enough. + + Args: + sequence: Sequence, can be either a str or list of strings + max_tokens: Maximum number of tokens in the sequence + input_budget: Token budget for the input + target_budget: Token budget for the target + keep_scheme: Scheme for sampling the keep probability + + Returns: + Dictionary containing the masked sequence tensor, the input mask, the target mask, and the decoder attention mask + """ + if isinstance(sequence, str): + # Tokenize the sequence and get the ids + seq_ids: List[int] = self.text_tokenizer.encode(sequence).ids + # Add EOS to all sequences + seq_ids.append(self.eos_id) + # Truncate sequence + seq_ids = seq_ids[:max_tokens] + + # Use default span masking + span_masking_fn = simple_span_masking + + elif isinstance(sequence, list): + # Tokenize the sequence chunks and get the ids + encoded_seq_chunks = self.text_tokenizer.encode_batch(sequence) + seq_ids: List[List[int]] = [seq.ids for seq in encoded_seq_chunks] + # Add EOS as an extra chunk + seq_ids.append([self.eos_id]) + # Truncate sequence to keep all chunks below max token length + cumulative_token_count = np.cumsum(np.array([len(chunk) for chunk in seq_ids])) + seq_ids = [chunk for (chunk, token_count) in zip(seq_ids, cumulative_token_count) if token_count <= max_tokens] + + # Span mask over chunks + span_masking_fn = chunk_span_masking + + else: + raise ValueError(f"Invalid sequence: {sequence}") + + + # If input budget is 0, treat it as if the whole sequence is completely masked + if input_budget == 0: + keep_prob = 0. + input_seq_ids = [] + _, target_seq_ids = span_masking_fn(seq_ids, self.sentinel_to_id, keep_prob) + else: + if keep_scheme == 'random': + keep_prob = sample_uniform(0, 1) + elif keep_scheme == 'all': + keep_prob = 1.0 + elif keep_scheme == 'binary': + keep_prob = random.choice([0., 1.]) + else: + raise ValueError(f"Invalid keep scheme for sequence masking: {keep_scheme}") + + input_seq_ids, target_seq_ids = span_masking_fn(seq_ids, self.sentinel_to_id, keep_prob) + # Keep lowering the keep_prob while we are over-budget + while len(input_seq_ids) > input_budget: + keep_prob = keep_prob * self.keep_prob_decay_factor + input_seq_ids, target_seq_ids = span_masking_fn(seq_ids, self.sentinel_to_id, keep_prob) + + # Span masking can add up to (max_tokens + 1) * 2 tokens for input + target + max_length = (max_tokens + 1) * 2 + tensor = torch.ones(max_length, dtype=torch.int) * self.pad_id + input_mask = torch.ones(max_length, dtype=torch.bool) + target_mask = torch.ones(max_length, dtype=torch.bool) + decoder_attention_mask = torch.zeros(max_length, dtype=torch.int) + + # Set input and input mask + tensor[:len(input_seq_ids)] = torch.tensor(input_seq_ids, dtype=torch.int) + input_mask[:len(input_seq_ids)] = 0 + + if target_budget is None or len(target_seq_ids) <= target_budget: + tensor[input_budget:input_budget + len(target_seq_ids)] = torch.tensor(target_seq_ids, dtype=torch.int) + target_mask[input_budget:input_budget + len(target_seq_ids)] = 0 + decoder_attention_mask[input_budget:input_budget + len(target_seq_ids)] = 1 + else: + # Randomly choose sentinel token. + sentinel_indices = [i for i, token_id in enumerate(target_seq_ids) if token_id in self.sentinel_ids] + # If there is more than 1 sentinel, avoid sampling the very last one which indicates the end of the sequence + chosen_sentinel = np.random.randint(max(1, len(sentinel_indices) - 1)) + # If length starting at this token g.t. budget, truncate until budget is reached + if len(target_seq_ids) - sentinel_indices[chosen_sentinel] >= target_budget: + target_seq_ids = target_seq_ids[sentinel_indices[chosen_sentinel]:sentinel_indices[chosen_sentinel] + target_budget] + # Otherwise, select earliest sentinel token such that we don't go over budget + # Note: We could also use the randomly chosen sentinel token, but that would waste budget + else: + for idx in sentinel_indices: + if len(target_seq_ids) - idx <= target_budget: + target_seq_ids = target_seq_ids[idx:] + break + + tensor[input_budget:input_budget + len(target_seq_ids)] = torch.tensor(target_seq_ids, dtype=torch.int) + target_mask[input_budget:input_budget + len(target_seq_ids)] = 0 + decoder_attention_mask[input_budget:input_budget + len(target_seq_ids)] = 1 + + return {"tensor": tensor, "input_mask": input_mask, "target_mask": target_mask, "decoder_attention_mask": decoder_attention_mask} + + + def sequence_emb_mask_span(self, emb_tensor: torch.Tensor, max_tokens: int, input_budget: int, target_budget: int, keep_scheme: str): + """Applies input masking to an sequence embedding tensor, target masking is not supported with sequence embeddings + + Args: + emb_tensor: Sequence embedding tensor + max_tokens: Maximum number of tokens in the sequence + input_budget: Token budget for the input + target_budget: Token budget for the target (unused for now) + keep_scheme: Scheme for sampling the keep probability + + Returns: + Dictionary containing the masked sequence embedding tensor, the input mask, the target mask, and the decoder attention mask + """ + # Only supported as input modality now + + # Make fake seq ids for sequence embeddings to reuse simple_span_masking function + fake_seq_ids = [] + emb_dict = {} + id_num = len(self.sentinel_ids) + emb_ind = 0 + while(len(fake_seq_ids) < len(emb_tensor)): + if id_num not in self.sentinel_ids: # replace with T5 sentinel_id + fake_seq_ids.append(id_num) + emb_dict[id_num] = emb_tensor[emb_ind, :] + emb_ind += 1 + id_num += 1 + + # Truncate sequence + fake_seq_ids = fake_seq_ids[:max_tokens] + + # If input budget is 0, treat it as if the whole sequence is completely masked + if input_budget == 0: + keep_prob = 0. + fake_input_seq_ids = [] + _, fake_target_seq_ids = simple_span_masking(fake_seq_ids, self.sentinel_to_id, keep_prob) + else: + if keep_scheme == 'random': + keep_prob = sample_uniform(0, 1) + elif keep_scheme == 'all': + keep_prob = 1.0 + elif keep_scheme == 'binary': + keep_prob = random.choice([0., 1.]) + else: + raise ValueError(f"Invalid keep scheme for sequence masking: {keep_scheme}") + + fake_input_seq_ids, fake_target_seq_ids = simple_span_masking(fake_seq_ids, self.sentinel_to_id, keep_prob) + # Keep lowering the keep_prob while we are over-budget + while len(fake_input_seq_ids) > input_budget: + keep_prob = keep_prob * self.keep_prob_decay_factor + fake_input_seq_ids, fake_target_seq_ids = simple_span_masking(fake_seq_ids, self.sentinel_to_id, keep_prob) + + # Span masking can add up to max_tokens tokens for input + max_length = max_tokens + tensor = torch.zeros((max_length, emb_tensor.shape[1]), dtype=torch.float32) + input_mask = torch.ones(max_length, dtype=torch.bool) + target_mask = torch.ones(max_length, dtype=torch.bool) + decoder_attention_mask = torch.zeros(max_length, dtype=torch.int) + + # Put tensor values back based on the fake seq ids + for i_, fake_id in enumerate(fake_input_seq_ids): + if fake_id in self.sentinel_ids: + tensor[i_, :] = torch.zeros_like(emb_tensor[0,:]) # TODO replace to learned embeddings later + else: + tensor[i_, :] = emb_dict[fake_id] + + # Set input and input mask + input_mask[:len(fake_input_seq_ids)] = 0 + + return {"tensor": tensor, "input_mask": input_mask, "target_mask": target_mask, "decoder_attention_mask": decoder_attention_mask} + + + def __call__(self, mod_dict): + """Applies input and target masking to a dictionary of modalities + + Args: + mod_dict: Dictionary of modalities + + Returns: + Dictionary containing the masked modalities + """ + if self.sampling_weights is not None: + # Sample masking scheme according to a list of weights + dir_idx = torch.multinomial(self.sampling_weights, 1).item() + else: + # Randomly sample masking scheme + dir_idx = random.randint(0, self.num_dirichlets - 1) + + num_input_tokens = random.randint(*self.input_tokens_range) + num_target_tokens = random.randint(*self.target_tokens_range) if self.target_tokens_range is not None else None + + input_token_budget = self.input_token_budget(num_input_tokens, dir_idx) + + if num_target_tokens is not None: + target_token_budget = self.target_token_budget(input_token_budget, num_target_tokens, dir_idx) + else: + target_token_budget = [None] * self.num_modalities + + masked_mod_dict = {} + for (mod_name, mod_info), input_budget, target_budget in zip(self.modality_info.items(), input_token_budget, target_token_budget): + mod_type = mod_info['type'] + mod_name_load = mod_name if mod_name in mod_dict else get_transform_key(mod_name) + if mod_type == 'img': + masked_mod_dict[mod_name] = self.image_mask(mod_dict[mod_name_load], mod_info['max_tokens'], input_budget, target_budget) + elif mod_type == 'seq': + keep_scheme = 'random' if ('keep' not in mod_info) else mod_info['keep'][dir_idx] + masked_mod_dict[mod_name] = self.sequence_mask(mod_dict[mod_name_load], mod_info['max_tokens'], input_budget, target_budget, keep_scheme) + elif mod_type == 'seq_token': + keep_scheme = 'random' if ('keep' not in mod_info) else mod_info['keep'][dir_idx] + vocab_offset = mod_info.get('vocab_offset', 0) # Check if any space is allocated to sentinel tokens and other special tokens + masked_mod_dict[mod_name] = self.sequence_token_mask(mod_dict[mod_name_load], mod_info['max_tokens'], input_budget, target_budget, keep_scheme, vocab_offset=vocab_offset) + elif mod_type == "seq_emb": + keep_scheme = 'random' if ('keep' not in mod_info) else mod_info['keep'][dir_idx] + masked_mod_dict[mod_name] = self.sequence_emb_mask_span(mod_dict[mod_name_load], mod_info['max_tokens'], input_budget, target_budget, keep_scheme) + else: + raise ValueError(f"Invalid modality type: {mod_type}") + + return masked_mod_dict + + +class TransferMasking(object): + def __init__(self, + modality_info: Dict, + text_tokenizer: Optional[Tokenizer], + input_modalities: List[str], + target_modalities: List[str]): + """Performs masking for transfer on a dict of modalities (both image based and sequence based modalities), + by specifying which modalities are inputs and which are targets. + + Args: + modality_info: Dict with the modalities and their corresponding information + text_tokenizer: Tokenizer to use for text modalities + input_modalities: List of modalities to use as input + target_modalities: List of modalities to use as target + """ + self.modality_info = modality_info + self.num_modalities = len(modality_info) + self.min_tokens = torch.tensor([mod['min_tokens'] for mod in modality_info.values()]) + self.max_tokens = torch.tensor([mod['max_tokens'] for mod in modality_info.values()]) + self.mod_is_img = torch.tensor([mod['type'] == 'img' for mod in modality_info.values()]) + + self.input_modalities = set(input_modalities) + self.target_modalities = set(target_modalities) + + # Tokenizer for text modalities + self.text_tokenizer = text_tokenizer + if self.text_tokenizer is not None: + self.keep_prob_decay_factor = 0.9 + self.sentinel_to_id = get_sentinel_to_id_mapping(text_tokenizer) + self.sentinel_ids = set(self.sentinel_to_id.values()) + self.pad_id = text_tokenizer.token_to_id("[PAD]") + self.eos_id = text_tokenizer.token_to_id("[EOS]") + + + def input_image(self, tensor: torch.Tensor, num_tokens: int): + """Applies masking for an image given as input + + Args: + tensor: Image tensor + num_tokens: Number of tokens in the tensor + + Returns: + Dictionary containing the masked image tensor, the input mask, the target mask, and the decoder attention mask + """ + + # Input mask + input_mask = torch.zeros(num_tokens, dtype=torch.bool) + # Target mask + target_mask = torch.ones(num_tokens, dtype=torch.bool) + # Decoder attention mask + decoder_attention_mask = torch.zeros(num_tokens, dtype=torch.int) + + return {"tensor": tensor, "input_mask": input_mask, "target_mask": target_mask, "decoder_attention_mask": decoder_attention_mask} + + def target_image(self, tensor: torch.Tensor, num_tokens: int): + """Applies masking for an image given as target + + Args: + tensor: Image tensor + num_tokens: Number of tokens in the tensor + + Returns: + Dictionary containing the masked image tensor, the input mask, the target mask, and the decoder attention mask + """ + + # Input mask + input_mask = torch.ones(num_tokens, dtype=torch.bool) + # Target mask + target_mask = torch.zeros(num_tokens, dtype=torch.bool) + # Decoder attention mask + decoder_attention_mask = torch.zeros(num_tokens, dtype=torch.int) + decoder_attention_mask[0] = num_tokens + + return {"tensor": tensor, "input_mask": input_mask, "target_mask": target_mask, "decoder_attention_mask": decoder_attention_mask} + + + def input_sequence(self, sequence_str: str, max_tokens: int): + """Applies masking for a sequence given as input + + Args: + sequence_str: Sequence string + max_tokens: Maximum number of tokens in the sequence + + Returns: + Dictionary containing the masked sequence string, the input mask, the target mask, and the decoder attention mask + """ + # Tokenize the text and get the ids + seq_ids = self.text_tokenizer.encode(sequence_str).ids + # Add EOS to all sequences + seq_ids.append(self.eos_id) + # Truncate sequence + seq_ids = seq_ids[:max_tokens] + + keep_prob = 1. + input_seq_ids, target_seq_ids = simple_span_masking(seq_ids, self.sentinel_to_id, keep_prob) + + # Span masking can add up to (max_tokens + 1) * 2 tokens for input + target + max_length = (max_tokens + 1) * 2 + tensor = torch.ones(max_length, dtype=torch.int) * self.pad_id + input_mask = torch.ones(max_length, dtype=torch.bool) + target_mask = torch.ones(max_length, dtype=torch.bool) + decoder_attention_mask = torch.zeros(max_length, dtype=torch.int) + + # Set input and input mask + tensor[:len(input_seq_ids)] = torch.tensor(input_seq_ids, dtype=torch.int) + input_mask[:len(input_seq_ids)] = 0 + + tensor[max_tokens:max_tokens + len(target_seq_ids)] = torch.tensor(target_seq_ids, dtype=torch.int) + target_mask[max_tokens:max_tokens + len(target_seq_ids)] = 0 + decoder_attention_mask[max_tokens:max_tokens + len(target_seq_ids)] = 1 + + + return {"tensor": tensor, "input_mask": input_mask, "target_mask": target_mask, "decoder_attention_mask": decoder_attention_mask} + + + def target_sequence(self, sequence_str: str, max_tokens: int): + """Applies masking for a sequence given as target + + Args: + sequence_str: Sequence string + max_tokens: Maximum number of tokens in the sequence + + Returns: + Dictionary containing the masked sequence string, the input mask, the target mask, and the decoder attention mask + """ + # Tokenize the text and get the ids + seq_ids = self.text_tokenizer.encode(sequence_str).ids + # Add EOS to all sequences + seq_ids.append(self.eos_id) + # Truncate sequence + seq_ids = seq_ids[:max_tokens] + + keep_prob = 0. + input_seq_ids = [] + _, target_seq_ids = simple_span_masking(seq_ids, self.sentinel_to_id, keep_prob) + + # Span masking can add up to (max_tokens + 1) * 2 tokens for input + target + max_length = (max_tokens + 1) * 2 + tensor = torch.ones(max_length, dtype=torch.int) * self.pad_id + input_mask = torch.ones(max_length, dtype=torch.bool) + target_mask = torch.ones(max_length, dtype=torch.bool) + decoder_attention_mask = torch.zeros(max_length, dtype=torch.int) + + # Set input and input mask + tensor[:len(input_seq_ids)] = torch.tensor(input_seq_ids, dtype=torch.int) + input_mask[:len(input_seq_ids)] = 0 + + tensor[max_tokens:max_tokens + len(target_seq_ids)] = torch.tensor(target_seq_ids, dtype=torch.int) + target_mask[max_tokens:max_tokens + len(target_seq_ids)] = 0 + decoder_attention_mask[max_tokens:max_tokens + len(target_seq_ids)] = 1 + + return {"tensor": tensor, "input_mask": input_mask, "target_mask": target_mask, + "decoder_attention_mask": decoder_attention_mask} + + def __call__(self, mod_dict): + """Applies input and target masking to a dictionary of modalities + + Args: + mod_dict: Dictionary of modalities + + Returns: + Dictionary containing the masked modalities + """ + masked_mod_dict = {} + for mod_name, mod_info in self.modality_info.items(): + mod_type = mod_info['type'] + if mod_type == 'img' and mod_name in self.input_modalities: + masked_mod_dict[mod_name] = self.input_image(mod_dict[mod_name], mod_info['max_tokens']) + elif mod_type == 'img' and mod_name in self.target_modalities: + masked_mod_dict[mod_name] = self.target_image(mod_dict[mod_name], mod_info['max_tokens']) + elif mod_type == 'seq' and mod_name in self.input_modalities: + masked_mod_dict[mod_name] = self.input_sequence(mod_dict[mod_name], mod_info['max_tokens']) + elif mod_type == 'seq' and mod_name in self.target_modalities: + masked_mod_dict[mod_name] = self.target_sequence(mod_dict[mod_name], mod_info['max_tokens']) + else: + raise ValueError(f"Invalid modality type: {mod_type} or modality name not in input or target modalities: {mod_name}") + + if 'mask_valid' in mod_dict: + masked_mod_dict['mask_valid'] = mod_dict['mask_valid'] + + return masked_mod_dict \ No newline at end of file diff --git a/fourm/data/modality_info.py b/fourm/data/modality_info.py new file mode 100644 index 0000000000000000000000000000000000000000..a6572e735af6e9421a07767d798189981f04a219 --- /dev/null +++ b/fourm/data/modality_info.py @@ -0,0 +1,427 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from functools import partial + +import fourm.utils.data_constants as data_constants +from fourm.data.modality_transforms import (CaptionTransform, DepthTransform, + DetectionTransform, MaskTransform, + NormalTransform, RGBTransform, + SemsegTransform, TokTransform, + CaptionEmbTransform, MetadataTransform, + HumanPoseTransform, ColorPaletteTransform, + SAMInstanceTokTransform, SAMInstanceTransform) +from fourm.models.decoder_embeddings import (ImageTokenDecoderEmbedding, + SequenceDecoderEmbedding) +from fourm.models.encoder_embeddings import (ImageEncoderEmbedding, + ImageTokenEncoderEmbedding, + SequenceEncoderEmbedding, + SequenceEmbEncoderEmbedding) +from fourm.utils import generate_uint15_hash + +MODALITY_INFO = { + # 4M-7 modalities + 'rgb@224': { + 'input_size': 224, + 'patch_size': 16, + 'encoder_embedding': partial(ImageEncoderEmbedding, num_channels=3), + 'decoder_embedding': None, + 'min_tokens': 0, + 'max_tokens': None, # Will be set to 196 + 'type': 'img', + 'num_channels': 3, + 'id': generate_uint15_hash('rgb@224'), + 'path': 'rgb', + }, + 'rgb': { # used for tokenizer training + 'type': 'img', + 'num_channels': 3, + 'id': generate_uint15_hash('rgb'), + 'path': 'rgb', + }, + 'caption': { + 'vocab_size': 30_000, + 'encoder_embedding': partial(SequenceEncoderEmbedding, vocab_size=30_000, max_length=256, padding_idx=0), + 'decoder_embedding': partial(SequenceDecoderEmbedding, vocab_size=30_000, max_length=256, padding_idx=0), + 'min_tokens': 0, + 'max_tokens': 256, + 'type': 'seq', + 'id': generate_uint15_hash('caption'), + }, + 'det': { + 'vocab_size': 30_000, + 'encoder_embedding': partial(SequenceEncoderEmbedding, vocab_size=30_000, max_length=256, padding_idx=0), + 'decoder_embedding': partial(SequenceDecoderEmbedding, vocab_size=30_000, max_length=256, padding_idx=0), + 'min_tokens': 0, + 'max_tokens': 256, + 'type': 'seq', + 'id': generate_uint15_hash('det'), + }, + 'tok_rgb@224': { + 'input_size': 224, + 'patch_size': 16, + 'vocab_size': 16384, + 'encoder_embedding': partial(ImageTokenEncoderEmbedding, vocab_size=16384), + 'decoder_embedding': partial(ImageTokenDecoderEmbedding, vocab_size=16384), + 'min_tokens': 0, + 'max_tokens': None, # Will be set to 196 + 'type': 'img', + 'id': generate_uint15_hash('tok_rgb@224'), + 'pretokenized': True, + }, + 'tok_depth@224': { + 'input_size': 224, + 'patch_size': 16, + 'vocab_size': 8192, + 'encoder_embedding': partial(ImageTokenEncoderEmbedding, vocab_size=8192), + 'decoder_embedding': partial(ImageTokenDecoderEmbedding, vocab_size=8192), + 'min_tokens': 0, + 'max_tokens': None, # Will be set to 196 + 'type': 'img', + 'id': generate_uint15_hash('tok_depth@224'), + 'pretokenized': True, + }, + 'depth': { # used for tokenizer training + 'type': 'img', + 'num_channels': 1, + 'id': generate_uint15_hash('depth'), + }, + 'tok_normal@224': { + 'input_size': 224, + 'patch_size': 16, + 'vocab_size': 8192, + 'encoder_embedding': partial(ImageTokenEncoderEmbedding, vocab_size=8192), + 'decoder_embedding': partial(ImageTokenDecoderEmbedding, vocab_size=8192), + 'min_tokens': 0, + 'max_tokens': None, # Will be set to 196 + 'type': 'img', + 'id': generate_uint15_hash('tok_normal@224'), + 'pretokenized': True, + }, + 'normal': { # used for tokenizer training + 'type': 'img', + 'num_channels': 3, + 'id': generate_uint15_hash('normal'), + }, + 'tok_semseg@224': { + 'input_size': 224, + 'patch_size': 16, + 'vocab_size': 4096, + 'encoder_embedding': partial(ImageTokenEncoderEmbedding, vocab_size=4096), + 'decoder_embedding': partial(ImageTokenDecoderEmbedding, vocab_size=4096), + 'min_tokens': 0, + 'max_tokens': None, # Will be set to 196 + 'type': 'img', + 'id': generate_uint15_hash('tok_semseg@224'), + 'pretokenized': True, + }, + 'semseg_coco': { # used for tokenizer training + 'type': 'img', + 'num_channels': 64, + 'num_labels': data_constants.COCO_SEMSEG_NUM_CLASSES, + 'id': generate_uint15_hash('semseg_coco'), + }, + 'tok_clip@224': { + 'input_size': 224, + 'patch_size': 16, + 'vocab_size': 8192, + 'encoder_embedding': partial(ImageTokenEncoderEmbedding, vocab_size=8192), + 'decoder_embedding': partial(ImageTokenDecoderEmbedding, vocab_size=8192), + 'min_tokens': 0, + 'max_tokens': None, # Will be set to 196 + 'type': 'img', + 'id': generate_uint15_hash('tok_clip@224'), + 'pretokenized': True, + }, + 'CLIP-B16': { # used for tokenizer training + 'type': 'feature_map', + 'num_channels': 512, + 'id': generate_uint15_hash('CLIP-B16'), + }, + + # 4M-21 modalities + 't5_caption': { + 'encoder_embedding': partial(SequenceEmbEncoderEmbedding, max_length=77, padding_idx=0), + 'decoder_embedding': None, + 'min_tokens': 0, + 'max_tokens': 77, + 'type': 'seq_emb', + 'id': generate_uint15_hash('t5_caption'), + }, + 'metadata': { + 'vocab_size': 30_000, + 'encoder_embedding': partial(SequenceEncoderEmbedding, vocab_size=30_000, max_length=40, padding_idx=0, sincos_pos_emb=True), + 'decoder_embedding': partial(SequenceDecoderEmbedding, vocab_size=30_000, max_length=40, padding_idx=0, sincos_pos_emb=True), + 'min_tokens': 0, + 'max_tokens': 40, # At most 2x19=38 for 19 metadata types, +1 for EOS, +1 for sentinel + 'type': 'seq', + 'id': generate_uint15_hash('metadata'), + 'shared_vocab': ['caption'], + 'path': 'metadata', + }, + 'human_poses': { + 'vocab_size': 30_000, + 'encoder_embedding': partial(SequenceEncoderEmbedding, vocab_size=30_000, max_length=263, padding_idx=0, sincos_pos_emb=True), + 'decoder_embedding': partial(SequenceDecoderEmbedding, vocab_size=30_000, max_length=263, padding_idx=0, sincos_pos_emb=True), + 'min_tokens': 0, + 'max_tokens': 275, #7*39+1 EOS+1 S_1#263, #261 in one of the models, or 263 to have EOS #261+1+1 #238, + 'type': 'seq', + 'num_channels': 207, # for tokenization training, only the pose part is needed + 'id': generate_uint15_hash('human_poses'), + 'shared_vocab': ['caption'], + }, + 'color_palette': { + 'vocab_size': 30_000, + 'encoder_embedding': partial(SequenceEncoderEmbedding, vocab_size=30_000, max_length=23, padding_idx=0, sincos_pos_emb=True), + 'decoder_embedding': partial(SequenceDecoderEmbedding, vocab_size=30_000, max_length=23, padding_idx=0, sincos_pos_emb=True), + 'min_tokens': 0, + 'max_tokens': 23, #7x3=21 for 7 colors, +1 for EOS, +1 for sentinel + 'type': 'seq', + 'id': generate_uint15_hash('color_palette'), + 'shared_vocab': ['caption'], + 'path': 'color_palette', + }, + 'sam_mask': { + 'encoder_embedding': None, + 'decoder_embedding': None, + 'min_tokens': 0, + 'max_tokens': 64, + 'type': 'img', + 'num_channels': 1, + 'id': generate_uint15_hash('sam_mask'), + }, + 'sam_instance': { + 'vocab_size': 30_000, + 'encoder_embedding': partial(SequenceEncoderEmbedding, vocab_size=30_000, max_length=290, padding_idx=0, sincos_pos_emb=True), + 'decoder_embedding': partial(SequenceDecoderEmbedding, vocab_size=30_000, max_length=290, padding_idx=0, sincos_pos_emb=True), + 'min_tokens': 0, + 'max_tokens': 290, + 'type': 'seq', + 'id': generate_uint15_hash('sam_instance'), + 'shared_vocab': ['caption'], + 'pretokenized': True, + }, + 'tok_canny_edge@224': { + 'input_size': 224, + 'patch_size': 16, + 'vocab_size': 8192, + 'encoder_embedding': partial(ImageTokenEncoderEmbedding, vocab_size=8192), + 'decoder_embedding': partial(ImageTokenDecoderEmbedding, vocab_size=8192), + 'min_tokens': 0, + 'max_tokens': None, # Will be set to 196 + 'type': 'img', + 'id': generate_uint15_hash('tok_canny_edge@224'), + 'pretokenized': True, + }, + 'canny_edge': { # used for tokenizer training + 'type': 'img', + 'num_channels': 1, + 'id': generate_uint15_hash('canny_edge'), + }, + 'tok_sam_edge@224': { + 'input_size': 224, + 'patch_size': 16, + 'vocab_size': 8192, + 'encoder_embedding': partial(ImageTokenEncoderEmbedding, vocab_size=8192), + 'decoder_embedding': partial(ImageTokenDecoderEmbedding, vocab_size=8192), + 'min_tokens': 0, + 'max_tokens': None, # Will be set to 196 + 'type': 'img', + 'id': generate_uint15_hash('tok_sam_edge@224'), + 'pretokenized': True, + }, + 'tok_dinov2@224': { + 'input_size': 224, + 'patch_size': 14, + 'vocab_size': 8192, + 'encoder_embedding': partial(ImageTokenEncoderEmbedding, vocab_size=8192), + 'decoder_embedding': partial(ImageTokenDecoderEmbedding, vocab_size=8192), + 'min_tokens': 0, + 'max_tokens': None, # Will be set to 256 + 'type': 'img', + 'id': generate_uint15_hash('tok_dinov2@224'), + 'pretokenized': True, + }, + 'DINOv2-B14': { # used for tokenizer training + 'type': 'feature_map', + 'num_channels': 768, + 'id': generate_uint15_hash('DINOv2-B14'), + }, + 'tok_imagebind@224': { + 'input_size': 224, + 'patch_size': 14, + 'vocab_size': 8192, + 'encoder_embedding': partial(ImageTokenEncoderEmbedding, vocab_size=8192), + 'decoder_embedding': partial(ImageTokenDecoderEmbedding, vocab_size=8192), + 'min_tokens': 0, + 'max_tokens': None, # Will be set to 256 + 'type': 'img', + 'id': generate_uint15_hash('tok_imagebind@224'), + 'pretokenized': True, + }, + 'ImageBind-H14': { # used for tokenizer training + 'type': 'feature_map', + 'num_channels': 1280, + 'id': generate_uint15_hash('ImageBind-H14'), + }, + 'tok_dinov2_global': { + 'vocab_size': 8192, + 'patch_size': 56, + 'encoder_embedding': partial(ImageTokenEncoderEmbedding, vocab_size=8192, sincos_pos_emb=False), + 'decoder_embedding': partial(ImageTokenDecoderEmbedding, vocab_size=8192, sincos_pos_emb=False), + 'min_tokens': 0, + 'max_tokens': 16, + 'type': 'img', + 'id': generate_uint15_hash('tok_dinov2_global'), + 'pretokenized': True, + }, + 'DINOv2-B14-global': { # used for tokenizer training + 'type': 'feature_map', + 'num_channels': 768, + 'id': generate_uint15_hash('DINOv2-B14-global'), + }, + 'tok_imagebind_global': { + 'vocab_size': 8192, + 'patch_size': 56, + 'encoder_embedding': partial(ImageTokenEncoderEmbedding, vocab_size=8192, sincos_pos_emb=False), + 'decoder_embedding': partial(ImageTokenDecoderEmbedding, vocab_size=8192, sincos_pos_emb=False), + 'min_tokens': 0, + 'max_tokens': 16, + 'type': 'img', + 'id': generate_uint15_hash('tok_imagebind_global'), + 'pretokenized': True, + }, + 'ImageBind-H14-global': { # used for tokenizer training + 'type': 'feature_map', + 'num_channels': 1280, + 'id': generate_uint15_hash('ImageBind-H14-global'), + }, + + ### 224->448 super resolution modalities + 'rgb@448': { + 'input_size': 448, + 'patch_size': 16, + 'encoder_embedding': partial(ImageEncoderEmbedding, num_channels=3), + 'decoder_embedding': None, + 'min_tokens': 0, + 'max_tokens': None, # Will be set to 784 + 'type': 'img', + 'num_channels': 3, + 'id': generate_uint15_hash('rgb@448'), + 'path': 'rgb', + }, + 'tok_rgb@448': { + 'input_size': 448, + 'patch_size': 16, + 'vocab_size': 16384, + 'encoder_embedding': partial(ImageTokenEncoderEmbedding, vocab_size=16384), + 'decoder_embedding': partial(ImageTokenDecoderEmbedding, vocab_size=16384), + 'min_tokens': 0, + 'max_tokens': None, # Will be set to 784 + 'type': 'img', + 'id': generate_uint15_hash('tok_rgb@448'), + 'pretokenized': True, + }, + 'tok_depth@448': { + 'input_size': 448, + 'patch_size': 16, + 'vocab_size': 8192, + 'encoder_embedding': partial(ImageTokenEncoderEmbedding, vocab_size=8192), + 'decoder_embedding': partial(ImageTokenDecoderEmbedding, vocab_size=8192), + 'min_tokens': 0, + 'max_tokens': None, # Will be set to 784 + 'type': 'img', + 'id': generate_uint15_hash('tok_depth@448'), + 'pretokenized': True, + }, + 'tok_normal@448': { + 'input_size': 448, + 'patch_size': 16, + 'vocab_size': 8192, + 'encoder_embedding': partial(ImageTokenEncoderEmbedding, vocab_size=8192), + 'decoder_embedding': partial(ImageTokenDecoderEmbedding, vocab_size=8192), + 'min_tokens': 0, + 'max_tokens': None, # Will be set to 784 + 'type': 'img', + 'id': generate_uint15_hash('tok_normal@448'), + 'pretokenized': True, + }, + 'tok_semseg@448': { + 'input_size': 448, + 'patch_size': 16, + 'vocab_size': 4096, + 'encoder_embedding': partial(ImageTokenEncoderEmbedding, vocab_size=4096), + 'decoder_embedding': partial(ImageTokenDecoderEmbedding, vocab_size=4096), + 'min_tokens': 0, + 'max_tokens': None, # Will be set to 784 + 'type': 'img', + 'id': generate_uint15_hash('tok_semseg@448'), + 'pretokenized': True, + }, + 'tok_clip@448': { + 'input_size': 448, + 'patch_size': 16, + 'vocab_size': 8192, + 'encoder_embedding': partial(ImageTokenEncoderEmbedding, vocab_size=8192), + 'decoder_embedding': partial(ImageTokenDecoderEmbedding, vocab_size=8192), + 'min_tokens': 0, + 'max_tokens': None, # Will be set to 784 + 'type': 'img', + 'id': generate_uint15_hash('tok_clip@448'), + 'pretokenized': True, + }, +} + +# Note: @res suffix is ignored for modality transforms +MODALITY_TRANSFORMS = { + # 4M-7 modalities + 'rgb': RGBTransform(imagenet_default_mean_and_std=True), + 'caption': CaptionTransform(aligned_captions=True), + 'det': DetectionTransform(det_threshold=0.6, det_max_instances=None, bbox_order='dist_to_orig', coord_bins=1000, min_visibility=0.0), + 'tok_rgb': TokTransform(), + 'tok_depth': TokTransform(), + 'tok_normal': TokTransform(), + 'tok_semseg': TokTransform(), + 'tok_clip': TokTransform(), + # 4M-21 modalities + 't5_caption': CaptionEmbTransform(), + 'metadata': MetadataTransform(special_vmin=0, special_vmax=999, shuffle=True, random_trunc=False, return_chunks=True), + 'human_poses': HumanPoseTransform(coord_bins=1000), + 'color_palette': ColorPaletteTransform(coord_bins=1000), + 'sam_instance': SAMInstanceTokTransform(image_size=224, points_per_side=7, point_order='random'), + 'tok_canny_edge': TokTransform(), + 'tok_sam_edge': TokTransform(), + 'tok_dinov2': TokTransform(), + 'tok_imagebind': TokTransform(), + 'tok_dinov2_global': TokTransform(), + 'tok_imagebind_global': TokTransform(), + # Other + 'mask_valid': MaskTransform(mask_pool_size=1), +} + +MODALITY_TRANSFORMS_DIVAE = { + 'rgb': RGBTransform(imagenet_default_mean_and_std=False), + 'depth': DepthTransform(standardize_depth=True), + 'normal': NormalTransform(standardize_surface_normals=False), + 'mask_valid': MaskTransform(mask_pool_size=1), + 'semseg_coco': SemsegTransform(shift_idx_by_one=True), + 'canny_edge': RGBTransform(imagenet_default_mean_and_std=False), + 'human_poses': HumanPoseTransform(coord_bins=1000, only_pose=True), + 'sam_mask': SAMInstanceTransform(mask_size=64, max_instance_n=1), +} + +MODALITY_TRANSFORMS_VQCONTROLNET = { + 'rgb': RGBTransform(imagenet_default_mean_and_std=False), + 'mask_valid': MaskTransform(mask_pool_size=1), + 'caption': CaptionTransform(aligned_captions=True), +} diff --git a/fourm/data/modality_transforms.py b/fourm/data/modality_transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..f3c3563cbcfb5c73d633a24772026252a7ef8860 --- /dev/null +++ b/fourm/data/modality_transforms.py @@ -0,0 +1,1387 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import gzip +import json +import random +from pathlib import Path +from typing import Optional, Tuple, List, Dict +from abc import ABC, abstractmethod + +from PIL import Image +import cv2 + +import albumentations as A +import numpy as np +import torch +import torchvision.transforms.functional as TF +import torchvision.transforms as T +from einops import rearrange, repeat, reduce + +from fourm.utils import to_2tuple +from fourm.utils.data_constants import (IMAGENET_DEFAULT_MEAN, + IMAGENET_DEFAULT_STD, IMAGENET_INCEPTION_MEAN, + IMAGENET_SURFACE_NORMAL_STD, IMAGENET_SURFACE_NORMAL_MEAN, + IMAGENET_INCEPTION_STD, SEG_IGNORE_INDEX, PAD_MASK_VALUE) + + +# The @-symbol is used to specify the resolution of a modality. Syntax: modality@resolution +def get_transform_key(mod_name): + return mod_name.split('@')[0] + +def get_transform_resolution(mod_name, default_resolution, to_tuple=True): + res = int(mod_name.split('@')[1]) if '@' in mod_name else default_resolution + return to_2tuple(res) if to_tuple else res + +def get_transform(mod_name, transforms_dict): + return transforms_dict.get(get_transform_key(mod_name), IdentityTransform()) + +def get_pil_resample_mode(resample_mode: str): + """ + Returns the PIL resampling mode for the given resample mode string. + + Args: + resample_mode: Resampling mode string + """ + if resample_mode is None: + return None + elif resample_mode == "bilinear": + return Image.Resampling.BILINEAR if hasattr(Image, 'Resampling') else Image.BILINEAR + elif resample_mode == "bicubic": + return Image.Resampling.BICUBIC if hasattr(Image, 'Resampling') else Image.BICUBIC + elif resample_mode == "nearest": + return Image.Resampling.NEAREST if hasattr(Image, 'Resampling') else Image.NEAREST + else: + raise ValueError(f"Resample mode {resample_mode} is not supported.") + +class UnifiedDataTransform(object): + def __init__(self, transforms_dict, image_augmenter, resample_mode: str = None, add_sizes: bool = False, **kwargs): + """Unified data augmentation for FourM + + Args: + transforms_dict (dict): Dict of transforms for each modality + image_augmenter (AbstractImageAugmenter): Image augmenter + resample_mode (str, optional): Resampling mode for PIL images (default: None -> uses default resampling mode for data type) + One out of ["bilinear", "bicubic", "nearest", None]. + add_sizes (bool, optional): Whether to add crop coordinates and original size to the output dict + """ + + self.transforms_dict = transforms_dict + self.image_augmenter = image_augmenter + self.resample_mode = resample_mode + self.add_sizes = add_sizes + + def unified_image_augment(self, mod_dict, crop_settings): + """Apply the image augmenter to all modalities where it is applicable + + Args: + mod_dict (dict): Dict of modalities + crop_settings (dict): Crop settings + + Returns: + dict: Transformed dict of modalities + """ + + crop_coords, flip, orig_size, target_size, rand_aug_idx = self.image_augmenter(mod_dict, crop_settings) + + mod_dict = { + k: self.transforms_dict[get_transform_key(k)].image_augment( + v, crop_coords=crop_coords, flip=flip, orig_size=orig_size, + target_size=get_transform_resolution(k, target_size), rand_aug_idx=rand_aug_idx, + resample_mode=self.resample_mode + ) + for k, v in mod_dict.items() + } + + if self.add_sizes: + mod_dict["crop_coords"] = torch.tensor(crop_coords) + mod_dict["orig_size"] = torch.tensor(orig_size) + + return mod_dict + + def __call__(self, mod_dict): + """Apply the augmentation to a dict of modalities (both image based and sequence based modalities) + + Args: + mod_dict (dict): Dict of modalities + + Returns: + dict: Transformed dict of modalities + """ + crop_settings = mod_dict.pop("crop_settings", None) + + mod_dict = {k: get_transform(k, self.transforms_dict).preprocess(v) for k, v in mod_dict.items()} + + mod_dict = self.unified_image_augment(mod_dict, crop_settings) + + mod_dict = {k: get_transform(k, self.transforms_dict).postprocess(v) for k, v in mod_dict.items()} + + return mod_dict + + def __repr__(self): + repr = "(UnifiedDataAugmentation,\n" + repr += ")" + return repr + + +class AbstractTransform(ABC): + + @abstractmethod + def load(self, sample): + pass + + @abstractmethod + def preprocess(self, sample): + pass + + @abstractmethod + def image_augment(self, v, crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx: Optional[int], resample_mode: str = None): + pass + + @abstractmethod + def postprocess(self, v): + pass + + +class ImageTransform(AbstractTransform): + + @staticmethod + def pil_loader(path: str) -> Image.Image: + # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835) + # with open(path, 'rb') as f: + # img = Image.open(f) + img = Image.open(path) + return img + + + @staticmethod + def image_hflip(img: Image, flip: bool): + """Crop and resize an image + + :param img: Image to crop and resize + :param flip: Whether to flip the image + :return: Flipped image (if flip = True) + """ + if flip: + img = TF.hflip(img) + return img + + @staticmethod + def image_crop_and_resize(img: Image, crop_coords: Tuple, target_size: Tuple, resample_mode: str = None): + """Crop and resize an image + + :param img: Image to crop and resize + :param crop_coords: Coordinates of the crop (top, left, h, w) + :param target_size: Coordinates of the resize (height, width) + :return: Cropped and resized image + """ + + top, left, h, w = crop_coords + resize_height, resize_width = target_size + img = TF.crop(img, top, left, h, w) + resample_mode = get_pil_resample_mode(resample_mode) + img = img.resize((resize_height, resize_width), resample=resample_mode) + return img + + +class RGBTransform(ImageTransform): + + def __init__(self, imagenet_default_mean_and_std=True, color_jitter=False, color_jitter_strength=0.5): + self.rgb_mean = IMAGENET_INCEPTION_MEAN if not imagenet_default_mean_and_std else IMAGENET_DEFAULT_MEAN + self.rgb_std = IMAGENET_INCEPTION_STD if not imagenet_default_mean_and_std else IMAGENET_DEFAULT_STD + self.color_jitter = color_jitter + self.color_jitter_transform = self.random_color_jitter(color_jitter_strength) + + def random_color_jitter(self, strength=0.5): + # Color Jitter from Pix2Seq and SimCLR + # Source: https://github.com/google-research/pix2seq/blob/main/data/data_utils.py#L114 + t = T.Compose([ + T.RandomApply([T.ColorJitter(brightness=0.8 * strength, contrast=0.8 * strength, saturation=0.8 * strength, hue=0.2 * strength)], p=0.8), + T.RandomApply([T.Grayscale(num_output_channels=3)], p=0.2), + ]) + + return t + + def rgb_to_tensor(self, img): + img = TF.to_tensor(img) + img = TF.normalize(img, mean=self.rgb_mean, std=self.rgb_std) + return img + + def load(self, path): + # TODO: Instead of converting to RGB here, do it either in the preprocess or the postprocess step. Makes it compatible with wds dataloading. + sample = self.pil_loader(path) + return sample + + def preprocess(self, sample): + sample = sample.convert('RGB') + + if self.color_jitter: + sample = self.color_jitter_transform(sample) + + return sample + + def image_augment(self, img, crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx: Optional[int], resample_mode: str = None): + img = self.image_crop_and_resize(img, crop_coords, target_size, resample_mode=resample_mode) + img = self.image_hflip(img, flip) + return img + + def postprocess(self, sample): + sample = self.rgb_to_tensor(sample) + return sample + + +class DepthTransform(ImageTransform): + + def __init__(self, standardize_depth=True): + self.standardize_depth = standardize_depth + + def depth_to_tensor(self, img): + img = torch.Tensor( img / (2 ** 16 - 1.0) ) + img = img.unsqueeze(0) # 1 x H x W + if self.standardize_depth: + img = self.truncated_depth_standardization(img) + return img + + @staticmethod + def truncated_depth_standardization(depth, thresh: float = 0.1): + """Truncated depth standardization + + :param depth: Depth map + :param thresh: Threshold + :return: Robustly standardized depth map + """ + # Flatten depth and remove bottom and top 10% of values + trunc_depth = torch.sort(depth.reshape(-1), dim=0)[0] + trunc_depth = trunc_depth[int(thresh * trunc_depth.shape[0]): int((1 - thresh) * trunc_depth.shape[0])] + return (depth - trunc_depth.mean()) / torch.sqrt(trunc_depth.var() + 1e-6) + + def load(self, path): + sample = self.pil_loader(path) + return sample + + def preprocess(self, sample): + return sample + + def image_augment(self, img, crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx: Optional[int], resample_mode: str = None): + img = self.image_crop_and_resize(img, crop_coords, target_size, resample_mode=resample_mode) + img = self.image_hflip(img, flip) + return img + + def postprocess(self, sample): + sample = np.array(sample) + sample = self.depth_to_tensor(sample) + return sample + + +class NormalTransform(ImageTransform): + + def __init__(self, standardize_surface_normals=False): + self.normal_mean = (0.5, 0.5, 0.5) if not standardize_surface_normals else IMAGENET_SURFACE_NORMAL_MEAN + self.normal_std = (0.5, 0.5, 0.5) if not standardize_surface_normals else IMAGENET_SURFACE_NORMAL_STD + + def normal_to_tensor(self, img): + img = TF.to_tensor(img) + img = TF.normalize(img, mean=self.normal_mean, std=self.normal_std) + return img + + def load(self, path): + sample = self.pil_loader(path) + return sample + + def preprocess(self, sample): + return sample + + def image_hflip(self, img: Image, flip: bool): + if flip: + img = TF.hflip(img) + flipped_np = np.array(img) + flipped_np[:, :, 0] = 255 - flipped_np[:, :, 0] + img = Image.fromarray(flipped_np) + + return img + + def image_augment(self, img, crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx: Optional[int], resample_mode: str = None): + img = self.image_crop_and_resize(img, crop_coords, target_size, resample_mode=resample_mode) + img = self.image_hflip(img, flip) + return img + + def postprocess(self, sample): + sample = self.normal_to_tensor(sample) + return sample + + +class SemsegTransform(ImageTransform): + + def __init__(self, scale_factor=1.0, shift_idx_by_one=False, id_mapping: Optional[Dict] = None, select_channel=None): + self.scale_factor = scale_factor + self.shift_idx_by_one = shift_idx_by_one + self.id_mapping = id_mapping + self.select_channel = select_channel + + def map_semseg_values(self, sample): + sample = np.asarray(sample) + mapping_fn = lambda x: self.id_mapping.get(x, x) + sample = np.vectorize(mapping_fn)(sample) + sample = Image.fromarray(sample, mode='P') + return sample + + def semseg_to_tensor(self, img): + # Rescale to scale factor + if self.scale_factor != 1.0: + target_height, target_width = int(img.height * self.scale_factor), int(img.width * self.scale_factor) + img = img.resize((target_width, target_height)) + # Using pil_to_tensor keeps it in uint8, to_tensor converts it to float (rescaled to [0, 1]) + img = TF.pil_to_tensor(img).to(torch.long).squeeze(0) + # 255->0, 254->0, all else shifted up by one + return img + + def load(self, path): + sample = self.pil_loader(path) + if self.select_channel is not None: + sample = sample.split()[self.select_channel] + return sample + + def preprocess(self, sample): + sample = sample.convert('P') + + if self.id_mapping is not None: + sample = self.map_semseg_values(sample) + + if self.shift_idx_by_one: + sample = np.asarray(sample) + sample = sample + 1 + sample = Image.fromarray(sample, mode='P') + + return sample + + def image_augment(self, img, crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx: Optional[int], resample_mode: str = None): + # Value for padding with TF.crop is always 0. + # Override resampling mode to 'nearest' for semseg + img = self.image_crop_and_resize(img, crop_coords, target_size, resample_mode='nearest') + img = self.image_hflip(img, flip) + return img + + def postprocess(self, sample): + img = self.semseg_to_tensor(sample) + return img + + +class SAMInstanceTransform(AbstractTransform): + + def __init__(self, mask_size=64, max_instance_n=20, bbox_area_threshold=0.0005): + self.mask_size = mask_size + self.max_instance_n = max_instance_n + self.bbox_area_threshold = bbox_area_threshold + + def get_bbox(self, instance): + """ Gets bounding box of the given instance + """ + min_h, max_h = instance[:,:,1].min(), instance[:,:,1].max() + min_w, max_w = instance[:,:,0].min(), instance[:,:,0].max() + return [min_h, min_w, max_h, max_w] + + def extend_instance_points(self, instance, border_fn): + """ Given an instance and a border function `border_fn`, extends the instance points with crossing points between the instance and + the crop borders. The crossing points are obtained using border_fn. + """ + p = instance[:,0] + p_next = np.roll(p, (-1), axis=(0)) + final_points = [] + for x, xn in zip(p, p_next): + final_points.append(x) + for r in border_fn(x, xn): + final_points.append(r.astype(np.int32)) + p = np.stack(final_points) + return p[:,None] + + def remove_redundant_lines(self, orig_instance, instance): + """ Removes the redundant lines added during cropping. + """ + final_points = [] + for p in instance: + distance = cv2.pointPolygonTest(orig_instance, (p[0,0].item(), p[0,1].item()), measureDist=True) + if distance >= 0: + final_points.append(p[0]) + return np.stack(final_points)[:,None] + + def get_border_functions(self, crop_points): + """ Creates and returns a function `fn` using crop region coordinates given in crop_points. + `fn` receives two input points x and xn and returns all the crossing points between the line connecting + x and xn, and the borders of the cropping rectangle. + """ + p = crop_points[:,0] + p_next = np.roll(p, (-1), axis=(0)) + def fn(x, xn): + output = [] + c_diff = p_next - p + x_diff = x - xn + for diff, c in zip(c_diff, p): + A = np.array([ + [diff[0], x_diff[0]], + [diff[1], x_diff[1]] + ]) + b = x - c + try: + lmbda = np.linalg.solve(A, b) + if 0 <= lmbda[0] <= 1 and 0 <= lmbda[1] <= 1: + output.append(lmbda[1] * xn + (1-lmbda[1]) * x) + except: + continue + return output + return fn + + def crop_sample(self, sample, crop_coords): + """ Crop the sample using crop coordinates. + """ + top, left, h, w = crop_coords + crop_region = (left, top, left + w, top + h) + crop_points = np.array([ + [crop_region[0], crop_region[1]], + [crop_region[2], crop_region[1]], + [crop_region[2], crop_region[3]], + [crop_region[0], crop_region[3]], + ])[:,None] + border_functions = self.get_border_functions(crop_points) + cropped_sample = [] + for instance in sample: + instance = self.extend_instance_points(instance, border_functions) + filter_condition = ( + (instance[:, :, 0] > crop_region[0]) & + (instance[:, :, 0] < crop_region[2]) & + (instance[:, :, 1] > crop_region[1]) & + (instance[:, :, 1] < crop_region[3]) + ) + if not np.any(filter_condition): + continue + + instance_copy = instance.copy() + instance_copy[:, :, 0] = np.clip(instance[:, :, 0], a_min=crop_region[0], a_max=crop_region[2]) + instance_copy[:, :, 1] = np.clip(instance[:, :, 1], a_min=crop_region[1], a_max=crop_region[3]) + instance_copy = self.remove_redundant_lines(instance, instance_copy) + instance_copy[:, :, 0] -= crop_region[0] + instance_copy[:, :, 1] -= crop_region[1] + + cropped_sample.append(instance_copy) + return cropped_sample + + def resize_sample(self, sample, original_size, target_size): + """ Resize the sample + """ + width_scale = target_size[1] / original_size[1] + height_scale = target_size[0] / original_size[0] + resized_sample = [] + for instance in sample: + instance_copy = instance.copy() + instance_copy[:, :, 0] = np.round(width_scale * instance_copy[:, :, 0]) + instance_copy[:, :, 1] = np.round(height_scale * instance_copy[:, :, 1]) + resized_sample.append(instance_copy) + return resized_sample + + def remove_tiny_instances(self, sample, image_size): + """ Remove instances that have an area ratio smaller than `bbox_area_threshold`. + """ + filtered_sample = [] + for instance in sample: + min_h, min_w, max_h, max_w = self.get_bbox(instance) + bbox_area_ratio = (max_h - min_h) * (max_w - min_w) / (image_size[0] * image_size[1]) + if bbox_area_ratio < self.bbox_area_threshold: + continue + filtered_sample.append(instance) + return filtered_sample + + def hflip(self, sample, width): + """ Horizontal flipping the instances in a sample. + """ + flipped_sample = [] + for instance in sample: + instance_copy = instance.copy() + instance_copy[:, :, 0] = width - instance_copy[:, :, 0] + flipped_sample.append(instance_copy) + return flipped_sample + + def get_binary_masks(self, sample): + """ Creates the binary mask of each instance in the sample. + """ + if self.max_instance_n is None: + max_instance_n = len(sample) + else: + max_instance_n = self.max_instance_n + masks = np.zeros((max_instance_n, self.mask_size, self.mask_size)) + bboxes = np.zeros((max_instance_n, 4)) + valid = np.full(max_instance_n, False) + for i, instance in enumerate(sample): + bbox = self.get_bbox(instance) + min_h, min_w, max_h, max_w = bbox + instance_copy = instance.copy() + mask = np.zeros((self.mask_size, self.mask_size), dtype=np.uint8) + instance_copy[:,:,0] = (instance_copy[:,:,0] - min_w) / (max_w - min_w) * self.mask_size + instance_copy[:,:,1] = (instance_copy[:,:,1] - min_h) / (max_h - min_h) * self.mask_size + cv2.drawContours(mask, [instance_copy], 0, (255), thickness=cv2.FILLED) + masks[i] = mask / 255.0 + bboxes[i] = np.array(bbox) + valid[i] = True + return masks, bboxes, valid + + def load(self, path): + sample = np.load(path, allow_pickle=True) + return sample + + def preprocess(self, sample): + if self.max_instance_n is None or len(sample) <= self.max_instance_n: + indecies = np.arange(len(sample)) + else: + indecies = np.random.choice(len(sample), size=self.max_instance_n, replace=False) + return [p['points'] for i, p in enumerate(sample) if i in indecies] + + def image_augment(self, v, crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx: Optional[int], resample_mode: str = None): + v = self.crop_sample(v, crop_coords) + _, _, h, w = crop_coords + v = self.resize_sample(v, (h, w), target_size) + v = self.remove_tiny_instances(v, target_size) + if flip: + v = self.hflip(v, target_size[0]) + return v + + def postprocess(self, sample): + sample, bboxes, valid = self.get_binary_masks(sample) + return { + 'instance': torch.from_numpy(sample).to(torch.float32), + 'bbox': torch.from_numpy(bboxes).to(torch.float32), + 'valid': torch.from_numpy(valid) + } + + +class MaskTransform(ImageTransform): + + def __init__(self, mask_pool_size=1): + assert isinstance(mask_pool_size, int) + self.mask_pool_size = mask_pool_size # Use to expand masks + + def mask_to_tensor(self, img): + mask = TF.to_tensor(img) + if self.mask_pool_size > 1: + mask = reduce(mask, 'c (h1 h2) (w1 w2) -> c h1 w1', 'min', h2=self.mask_pool_size, w2=self.mask_pool_size) + mask = repeat(mask, 'c h1 w1 -> c (h1 h2) (w1 w2)', h2=self.mask_pool_size, w2=self.mask_pool_size) + return (mask == 1.0) + + def load(self, path): + sample = self.pil_loader(path) + return sample + + def preprocess(self, sample): + return sample + + def image_augment(self, img, crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx: Optional[int], resample_mode: str = None): + # Override resampling mode to 'nearest' for masks + img = self.image_crop_and_resize(img, crop_coords, target_size, resample_mode='nearest') + img = self.image_hflip(img, flip) + return img + + def postprocess(self, sample): + sample = self.mask_to_tensor(sample) + return sample + + +class TokTransform(AbstractTransform): + + def __init__(self): + pass + + def load(self, path): + sample = np.load(path).astype(int) + return sample + + def preprocess(self, sample): + return sample + + def image_augment(self, v, crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx: Optional[int], resample_mode: str = None): + if rand_aug_idx is None: + raise ValueError("Crop settings / augmentation index are missing but a pre-tokenized modality is being used") + v = torch.tensor(v[rand_aug_idx]) + return v + + def postprocess(self, sample): + return sample + + +class DetectionTransform(AbstractTransform): + + def __init__(self, det_threshold=0.6, det_max_instances=None, bbox_order='dist_to_orig', coord_bins=1000, min_visibility=0.0, return_raw=False): + self.det_threshold = det_threshold + self.det_max_instances = det_max_instances + self.coord_bins = coord_bins + self.min_visibility = min_visibility + self.return_raw = return_raw + + if bbox_order == 'area': + self.bbox_order = self.order_bboxes_by_area + elif bbox_order == 'score': + self.bbox_order = self.order_bboxes_by_score + elif bbox_order == 'random': + self.bbox_order = self.shuffle_bboxes + else: + self.bbox_order = self.order_bboxes_by_dist_to_orig + + @staticmethod + def order_bboxes_by_area(bboxes): + return sorted(bboxes, key=lambda x: (x[2] - x[0]) * (x[3] - x[1]), reverse=True) + + @staticmethod + def order_bboxes_by_dist_to_orig(bboxes): + return sorted(bboxes, key=lambda x: x[0] ** 2 + x[1] ** 2) + + @staticmethod + def order_bboxes_by_score(bboxes): + return sorted(bboxes, key=lambda x: x[5], reverse=True) + + @staticmethod + def shuffle_bboxes(bboxes): + return sorted(bboxes, key=lambda x: random.random()) + + def convert_detection_instance(self, instances): + """Convert instances dict to list of lists where each list takes the form: + [xmin, ymin, xmax, ymax, class_name, score] + """ + + instances = [inst['boxes'] + [inst['class_name'], inst['score']] for inst in instances if inst['score'] >= self.det_threshold] + return instances + + def bboxes_hflip(self, bboxes: List[Tuple], image_size: Tuple, flip: bool): + image_height, image_width = image_size + if flip: + bboxes = [tuple(A.bbox_hflip(bbox[:4], rows=image_height, cols=image_width)) + tuple(bbox[4:]) + for bbox in bboxes] + + return bboxes + + def bboxes_crop_and_resize(self, bboxes: List[Tuple], crop_coords: Tuple, orig_size: Tuple): + """Crop and resize bounding boxes + + Args: + bboxes: Bounding boxes to crop and resize + crop_coords: Coordinates of the crop (top, left, h, w) + orig_size: Size of the original image + + Returns: + Cropped and resized bounding boxes + """ + orig_height, orig_width = orig_size + top, left, h, w = crop_coords + xmin, ymin, xmax, ymax = left, top, left + w, top + h + bboxes = [tuple(A.bbox_crop(bbox[:4], x_min=xmin, y_min=ymin, x_max=xmax, y_max=ymax, rows=orig_height, + cols=orig_width)) + tuple(bbox[4:]) + for bbox in bboxes] + bboxes = A.core.bbox_utils.filter_bboxes(bboxes, rows=h, cols=w, min_visibility=self.min_visibility) + # No need to resize, bounding boxes in albumentations format are scale invariant + + return bboxes + + def order_and_filter_bboxes(self, bboxes): + if self.det_max_instances is not None and len(bboxes) > self.det_max_instances: + bboxes = self.order_bboxes_by_score(bboxes)[:self.det_max_instances] + + return self.bbox_order(bboxes) + + def convert_bboxes_to_string(self, bboxes: List[Tuple]): + """Convert bounding boxes to a string. + xmin, ymin, xmax, ymax are mapped to v0, v1, v2, v3 special tokens. + + Args: + bboxes: Bounding boxes + + Returns: + String representation of the bounding boxes + """ + # Remove score, quantize coordinates + bins = self.coord_bins + + bboxes = [ + [ + f"v0={round(xmin * (bins - 1))}", + f"v1={round(ymin * (bins - 1))}", + f"v2={round(xmax * (bins - 1))}", + f"v3={round(ymax * (bins - 1))}", + cls, + ] + for (xmin, ymin, xmax, ymax, cls, score) in bboxes + ] + # Convert each bounding box to a string + bboxes = [' '.join(b) for b in bboxes] + # Convert the list to a str + return ' '.join(bboxes) + + def load(self, path): + with open(path, 'r') as f: + sample = json.load(f) + + return sample + + def preprocess(self, sample): + instances = sample['instances'] + return self.convert_detection_instance(instances) + + def image_augment(self, bboxes: List[Tuple], crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx=None, resample_mode: str = None): + bboxes = self.bboxes_crop_and_resize(bboxes, crop_coords, orig_size) + bboxes = self.bboxes_hflip(bboxes, target_size, flip) + bboxes = self.order_and_filter_bboxes(bboxes) + return bboxes + + def postprocess(self, bboxes): + if self.return_raw: + return bboxes + bboxes = self.convert_bboxes_to_string(bboxes) + return bboxes + + +class CaptionTransform(AbstractTransform): + + def __init__(self, aligned_captions=True, no_aug=False): + self.aligned_captions = aligned_captions + self.no_aug = no_aug + + def load(self, path): + # Caption can either be stored as .txt or .json.gz (in which case it's a list of dicts) + if path.endswith('.txt'): + sample = Path(path).read_text() + elif path.endswith('.json'): + with open(path, 'r') as f: + sample = json.load(f) + elif path.endswith('.json.gz'): + with gzip.open(path, 'rb') as f: + sample = json.load(f) + return sample + + def preprocess(self, sample): + return sample + + def image_augment(self, val, crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx: Optional[int], resample_mode: str = None): + + if isinstance(val, list) or isinstance(val, tuple): + if self.aligned_captions: + val = val[0] if rand_aug_idx is None else val[rand_aug_idx] + else: + val = random.choice(val) if not self.no_aug else val[0] + + if isinstance(val, dict): + # If each caption is saved as a dict, extract the string + val = val["caption"] + assert isinstance(val, str) + + return val + + def postprocess(self, sample): + return sample + + +class CaptionEmbTransform(AbstractTransform): + + def __init__(self, aligned_captions=True, no_aug=False): + self.aligned_captions = aligned_captions + self.no_aug = no_aug + + def load(self, path): + if path.endswith('.npz'): + sample = np.load(path) + sample = {'emb': sample['emb'], 'mask_valid': sample['mask_valid']} + else: + raise ValueError(f"Invalid file format for caption embedding: {path}") + return sample + + def preprocess(self, sample): + return sample + + def image_augment(self, val, crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx: Optional[int], resample_mode: str = None): + + emb = val['emb'] + mask_valid = val['mask_valid'].astype(bool) + num_sequences = emb.shape[0] + + if num_sequences > 1: + if self.aligned_captions: + if rand_aug_idx is None: + emb, mask_valid = emb[0], mask_valid[0] + else: + emb, mask_valid = emb[rand_aug_idx], mask_valid[rand_aug_idx] + else: + if self.no_aug: + emb, mask_valid = emb[0], mask_valid[0] + else: + rand_idx = random.randint(0, num_sequences - 1) + emb, mask_valid = emb[rand_idx], mask_valid[rand_idx] + else: + emb, mask_valid = emb[0], mask_valid[0] + + emb = emb[mask_valid] # Keep only valid embeddings + + return emb + + def postprocess(self, sample): + return torch.tensor(sample) + + +class MetadataTransform(AbstractTransform): + + def __init__(self, + special_vmin: int = 0, + special_vmax: int = 999, + shuffle: bool = True, + random_trunc: bool = False, + return_chunks: bool = True, + return_raw: bool = False, + image_dim_bin_size: int = 32,): + """Metadata transform that takes in a metadata dictionary and converts + it into a string, or list of strings (for chunked span masking). + Uses special tokens v1 to denote metadata types, and v0 for their values. + + Args: + special_vmin: Minimum value for special tokens + special_vmax: Maximum value for special tokens + shuffle: Whether to shuffle the metadata order + random_trunc: Whether to randomly truncate the returned metadata + return_chunks: Whether to return a list of strings (for chunked span masking), + or a single string with all metadata concatenated + return_raw: Whether to return the raw metadata dictionary + """ + self.special_vmin = special_vmin + self.special_vmax = special_vmax + self.shuffle = shuffle + self.random_trunc = random_trunc + self.return_chunks = return_chunks + self.return_raw = return_raw + self.image_dim_bin_size = image_dim_bin_size + + # Explicit map to make sure that additional entries do not change existing IDs + # TODO: Make this work with other text tokenizers + self.metadata_id_map = { + 'original_width': 'v1=0', + 'original_height': 'v1=1', + 'caption_n_chars': 'v1=2', + 'caption_n_words': 'v1=3', + 'caption_n_sentences': 'v1=4', + 'n_humans': 'v1=5', + 'n_sam_instances': 'v1=6', + 'n_coco_instances': 'v1=7', + 'coco_instance_diversity': 'v1=8', + 'colorfulness': 'v1=9', + 'brightness': 'v1=10', + 'contrast': 'v1=11', + 'saturation': 'v1=12', + 'entropy': 'v1=13', + 'walkability': 'v1=14', + 'objectness': 'v1=15', + 'semantic_diversity': 'v1=16', + 'geometric_complexity': 'v1=17', + 'occlusion_score': 'v1=18', + 'watermark_score': 'v1=19', + 'aesthetic_score': 'v1=20', + } + self.id_metadata_map = {v: k for k, v in self.metadata_id_map.items()} + + # Image-dimension modalities are binned into 32 bins + self.image_dim_modalities = ['original_height', 'original_width'] + + # Integer modalities that don't undergo any scaling (except for truncation) + self.metadata_int_modalities = [ + 'caption_n_chars', 'caption_n_words', 'caption_n_sentences', + 'n_humans', 'n_sam_instances', 'n_coco_instances', + 'coco_instance_diversity', 'semantic_diversity', + ] + + # Bin boundaries for manually defined metadata modalities. + # Lowest and highest bin boundaries are implicitly set to -inf and +inf + self.metadata_manual_bins = { + 'watermark_score': [0.5], + 'aesthetic_score': [4.5, 5.5], + } + + # All other float or integer modalities that are binned into a defined number of bins + # Dictionary entries are (vmin, vmax, num_bins) + self.metadata_min_max_bins = { + 'colorfulness': (0, 150, 50), + 'brightness': (0, 255, 50), + 'contrast': (0, 127, 50), + 'saturation': (0, 255, 50), + 'entropy': (0, 10, 50), + 'walkability': (0, 1, 50), + 'objectness': (0, 1, 50), + 'geometric_complexity': (0, 0.75, 50), + 'occlusion_score': (0, 0.25, 50), + } + + def image_dim_to_string(self, metadata, key, bin_size=32): + value = metadata[key] // bin_size + value = max(self.special_vmin, min(value, self.special_vmax)) + return f"{self.metadata_id_map[key]} v0={value}" + + def int_metadata_to_string(self, metadata, key): + value = max(self.special_vmin, min(metadata[key], self.special_vmax)) + return f"{self.metadata_id_map[key]} v0={value}" + + def float_metadata_to_string(self, metadata, key, vmin, vmax, bins): + value = max(vmin, min(metadata[key], vmax)) + value = (value - vmin) / (vmax - vmin) + value = int(value * (bins-1)) + return f"{self.metadata_id_map[key]} v0={value}" + + def manual_bin_metadata_to_string(self, metadata, key): + value = metadata[key] + bin_idx = 0 + for bin_value in self.metadata_manual_bins[key]: + if value < bin_value: + break + bin_idx += 1 + return f"{self.metadata_id_map[key]} v0={bin_idx}" + + def metadata_to_string(self, metadata, keys: List[str] = None): + keys = list(metadata.keys()) if keys is None else keys + + if self.shuffle: + # Randomly shuffle + random.shuffle(keys) + if self.random_trunc: + # Randomly truncate + keys = keys[:random.randint(1,len(keys))] + + metadata_strings = [] + + for key in keys: + if key in self.image_dim_modalities: + # Image dimension modalities + metadata_str = self.image_dim_to_string(metadata, key, bin_size=self.image_dim_bin_size) + elif key in self.metadata_int_modalities: + # Integer modalities that don't undergo any scaling + metadata_str = self.int_metadata_to_string(metadata, key) + elif key in self.metadata_manual_bins: + # Metadata modalities for which bin boundaries are manually defined + metadata_str = self.manual_bin_metadata_to_string(metadata, key) + else: + # All other modalities + vmin, vmax, bins = self.metadata_min_max_bins[key] + metadata_str = self.float_metadata_to_string(metadata, key, vmin, vmax, bins) + + metadata_strings.append(metadata_str) + + if self.return_chunks: + return metadata_strings + else: + return ' '.join(metadata_strings) + + def load(self, path): + with open(path, 'r') as f: + sample = json.load(f) + return sample + + def preprocess(self, sample): + return sample + + def image_augment(self, val, crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx=None, resample_mode: str = None): + return val + + def postprocess(self, metadata): + if self.return_raw: + return metadata + metadata_str = self.metadata_to_string(metadata) + return metadata_str + + +class HumanPoseTransform(AbstractTransform): + + def __init__(self, coord_bins=1000, only_pose=False, return_raw=False): + self.coord_bins = coord_bins + self.return_raw = return_raw + self.only_pose = only_pose + + def convert_humanpose_instance(self, instances, only_pose=False): + """Convert instances dict to list of lists where each list takes the form: + [human, xmin xmax ymin ymax global val1 val2 ... val10 pose val1 val2 ... val 207 shape val1 val2 ... val10 camera val1 val2 val3 val4] + Like for bounding boxes, xmin, ymin, xmax, and ymax map to v0, v1, v2, and v3 respectively. + """ + if only_pose: # used for tokenizer training for pose + if len(instances) == 0: + return torch.zeros(207) + else: + return torch.from_numpy(np.array(instances['pred_smpl_params']['body_pose'][0]).flatten()).float() + if len(instances) == 0: #empty, i.e. there are no humans + return 'none' + + for k in instances: + if k!='pred_smpl_params': + instances[k] = torch.from_numpy(np.array(instances[k])) + + smpl_params = (instances['pred_smpl_params']) + + for k in smpl_params: + smpl_params[k] = torch.from_numpy(np.array(smpl_params[k])) + + total_num_instances = len(instances['bbox_xyxy']) + instances_converted = [] + for ii in range(total_num_instances): + instances_converted.append(['human'] + (np.array(instances['bbox_xyxy'][ii]).flatten().tolist()) + ['global'] + (np.array(instances['pred_smpl_params']['global_orient'][ii]).flatten().tolist()) + ['pose'] + (instances['pose_tokenized'][ii].flatten().tolist()) + ['shape'] + (instances['pred_smpl_params']['betas'][ii].flatten().tolist()) + ['camera'] + (instances['pred_cam'][ii].flatten().tolist())) + return instances_converted + + def humanposes_crop_and_resize(self, humanposes: List[Tuple], crop_coords: Tuple, orig_size: Tuple,): + """Crop and resize human poses (and their bounding boxes) + """ + orig_height, orig_width = orig_size + top, left, h, w = crop_coords + + humanposes_converted_resized = [] + for instance in humanposes: + bbox_curr = instance[1:5] + bbox_curr = np.array(bbox_curr) + bbox_curr[0::2] = bbox_curr[0::2] / orig_width + bbox_curr[1::2] = bbox_curr[1::2] / orig_height + + xmin, ymin, xmax, ymax = left, top, left + w, top + h + bbox_curr = A.bbox_crop(bbox_curr, x_min=xmin, y_min=ymin, x_max=xmax, y_max=ymax, rows=orig_height, + cols=orig_width) + bbox_curr = np.array(bbox_curr) + if np.all(bbox_curr[1::2]<0) or np.all(bbox_curr[0::2]<0): #bbox is out of range, remove it + continue + if np.all(bbox_curr[1::2]>1.0) or np.all(bbox_curr[0::2]>1.0): #bbox is out of range, remove it + continue + bbox_curr = np.clip(bbox_curr, a_min=0, a_max=1.) + + instance[1:5] = bbox_curr + humanposes_converted_resized.append(instance) + + # now return all instances, or none if there is no instance + if len(humanposes_converted_resized)>0: + pass + else: #no valid masks remains + return 'none' + + humanpose_returned = humanposes_converted_resized + + return humanpose_returned + + def convert_humanposes_to_string(self, all_humanposes: List[Tuple]): + """Convert humanposes to a string + range of global orientation: [-1, 1] + range of object pose: [-1, 1] + range of shape (betas): [-3, 3] + range of camera: [-1, 19] + """ + bins = self.coord_bins + + instance_final_all = '' + + for humanposes in all_humanposes: + human = humanposes[0] + bboxes = humanposes[1:5] + glob = humanposes[5] + global_orient = np.array(humanposes[6:15]) + pose = humanposes[15] + pose_params = np.array(humanposes[16:24]) + shape = humanposes[24] + shape_params = np.array(humanposes[25:35]) + camera = humanposes[35] + camera_params = np.clip(np.array(humanposes[36:]), a_min=-1., a_max=19.) + + bboxes_new = [ + f"v0={round(bboxes[0] * (bins - 1))}", + f"v1={round(bboxes[1] * (bins - 1))}", + f"v2={round(bboxes[2] * (bins - 1))}", + f"v3={round(bboxes[3] * (bins - 1))}"] + + global_orient = 499.5*global_orient + global_orient_new = [] + for ii in range(len(global_orient)): + global_orient_curr = f"v0={round(global_orient[ii]+499.5)}" + global_orient_new.append(global_orient_curr) + + pose_params_new = [] + for ii in range(len(pose_params)): + if pose_params[ii]<512: + pose_params_curr = f"v0={round(pose_params[ii])}" + else: + pose_params_curr = f"v1={round(pose_params[ii] - 512)}" + pose_params_new.append(pose_params_curr) + + shape_params = 166.5*shape_params + shape_params_new = [] + for ii in range(len(shape_params)): + shape_params_curr = f"v0={round(shape_params[ii]+499.5)}" + shape_params_new.append(shape_params_curr) + + camera_params = 49.95*camera_params + camera_params_new = [] + for ii in range(len(camera_params)): + camera_params_curr = f"v0={round(camera_params[ii]+49.95)}" + camera_params_new.append(camera_params_curr) + + #randomly shuffle everything except bbox part of the sequence + all_strings = [[pose]+pose_params_new, [glob] + global_orient_new, [camera] + camera_params_new, [shape] + shape_params_new ] + rand_perm = torch.randperm(4) + instance_final = [human] + bboxes_new + all_strings[rand_perm[0]] + all_strings[rand_perm[1]] + all_strings[rand_perm[2]] + all_strings[rand_perm[3]] + + + instance_final = ', '.join(instance_final) + instance_final = instance_final.replace(",", "") + instance_final_all = instance_final_all + instance_final + ' ' + + return instance_final_all + + def load(self, path): + with open(path, 'r') as f: + sample = json.load(f) + + return sample + + def preprocess(self, sample): + instances = sample + instances = self.convert_humanpose_instance(instances, only_pose=self.only_pose) + return instances + + def image_augment(self, humanposes: List[Tuple], crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx=None, resample_mode: str = None): + if humanposes=='none' or self.only_pose: + return humanposes + humanposes = self.humanposes_crop_and_resize(humanposes, crop_coords, orig_size) + return humanposes + + def postprocess(self, humanposes): + if humanposes=='none' or self.only_pose: + return humanposes if not self.return_raw else [] + if self.return_raw: + return humanposes + humanposes = self.convert_humanposes_to_string(humanposes) + return humanposes + + +class ColorPaletteTransform(AbstractTransform): + + def __init__(self, coord_bins=1000, return_raw=False): + self.coord_bins = coord_bins + self.return_raw = return_raw + + def convert_palette_instance(self, instances): + """Convert colors to v0= v0= ... + """ + length = random.randint(1,7) + instances_converted = np.array(instances[0][str(length)]).flatten().tolist() + return instances_converted + + def palette_hflip(self, palettes: List[Tuple], image_size: Tuple, flip: bool): + + return palettes + + def convert_palettes_to_string(self, all_palettes: List[Tuple]): + """Convert palettes to a string + """ + + colors = [] + len_palettes = len(all_palettes) + colors.append(f"v1={round(len_palettes/3)}") # start with the length of the color palette to avoid confusion + for ii in range(len(all_palettes)): + color_new = f"v0={round(all_palettes[ii])}" + colors.append(color_new) + + instance_final_all = colors + instance_final_all = ', '.join(instance_final_all) + instance_final_all = instance_final_all.replace(",", "") + + return instance_final_all + + def load(self, path): + with open(path, 'r') as f: + sample = json.load(f) + return sample + + def preprocess(self, sample): + if self.return_raw: + return sample + instances = sample + instances = self.convert_palette_instance(instances) + return instances + + def image_augment(self, palettes: List[Tuple], crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx=None, resample_mode: str = None): + return palettes + + def postprocess(self, palettes): + if self.return_raw: + return palettes + palettes = self.convert_palettes_to_string(palettes) + return palettes + + +class SAMInstanceTokTransform(AbstractTransform): + + def __init__(self, image_size=224, points_per_side=7, point_order='random'): + self.H, self.W = to_2tuple(image_size) + self.points_per_h, self.points_per_w = to_2tuple(points_per_side) + assert point_order in ['random', 'grid'] + self.point_order = point_order + + def get_query_points(self): + if self.point_order == 'grid': + # Create and cache grid query points + if not hasattr(self, 'grid_query_points'): + y, x = np.meshgrid(np.linspace(0, self.H, self.points_per_h + 2)[1:-1], np.linspace(0, self.W, self.points_per_w + 2)[1:-1]) + grid = np.stack((x, y), axis=2).astype(np.int32) + self.grid_query_points = grid.reshape(-1, 2) + return self.grid_query_points + elif self.point_order == 'random': + # Randomly sample query points + y = np.random.randint(0, self.H, self.points_per_h) + x = np.random.randint(0, self.W, self.points_per_w) + return np.concatenate((x[:,None], y[:,None]), axis=1) + else: + raise ValueError(f"Query point order mode {self.point_order} is not supported.") + + def get_target_tokens(self, sample, query_points): + instances_coords = [coords[0] for coords in sample['points']] + tokens = sample['token_ids'] + bboxes = sample['bbox'] + + instance_tokens_per_qpoint = dict() + for point in query_points: + point = (int(point[0].item()), int(point[1].item())) + instance_tokens_per_qpoint[point] = [] + for i, (coords, tok, bbox) in enumerate(zip(instances_coords, tokens, bboxes)): + # Calculate the distance from the query point to the instance + distance = cv2.pointPolygonTest(coords, point, measureDist=True) + # If the query point is inside the instance, add its corresponding token + if distance >= 0: + instance_tokens_per_qpoint[point].append((tok, bbox)) + + return instance_tokens_per_qpoint + + def convert_target_tokens_to_string(self, target_tokens): + result_text = [] + query_points = list(target_tokens.keys()) + # Randomly shuffle query points order (mainly for grid order) + random.shuffle(query_points) + for point in query_points: + + # Add query point coordinates to the string + result_text.append('point') + result_text.append(f'v0={point[1]}') + result_text.append(f'v1={point[0]}') + + # Randomly shuffle the order of instance tokens per query point + random.shuffle(target_tokens[point]) + if len(target_tokens[point]) == 0: + # If no instances tokens are found, add 'none' to the string + result_text.append('none') + else: + for tok, bbox in target_tokens[point]: + result_text.append(f'polygon') + + # Add bounding box coordinates to the string + ymin, xmin, ymax, xmax = bbox.astype(np.int32) + result_text.extend([ + f'v0={xmin}', + f'v1={ymin}', + f'v2={xmax}', + f'v3={ymax}', + ]) + + # Add instance tokens ids to the string + for idx in tok.tolist(): + if idx < 512: + result_text.append(f'v0={idx}') + else: + result_text.append(f'v1={idx - 512}') + + return " ".join(result_text) + + def load(self, path): + sample = np.load(path, allow_pickle=True) + return sample + + def preprocess(self, sample): + for s in sample: + s['token_ids'] = s['token_ids'].astype(np.int32) + return sample + + def image_augment(self, v, crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx: Optional[int], resample_mode: str = None): + if rand_aug_idx is None: + raise ValueError("Crop settings / augmentation index are missing but a pre-tokenized modality is being used") + v = v[rand_aug_idx] + return v + + def postprocess(self, sample): + query_points = self.get_query_points() + target_tokens = self.get_target_tokens(sample, query_points) + final_string = self.convert_target_tokens_to_string(target_tokens) + return final_string + + +class CropSettingsTransform(AbstractTransform): + + def load(self, path): + sample = np.load(path) + return sample + + def preprocess(self, sample): + raise NotImplementedError("CropSettingsTransform does not support preprocessing") + + def image_augment(self, val, crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx: Optional[int], resample_mode: str = None): + raise NotImplementedError("CropSettingsTransform is not meant to be used for image augmentation") + + def postprocess(self, sample): + raise NotImplementedError("CropSettingsTransform does not support postprocessing") + + +class IdentityTransform(AbstractTransform): + + def load(self, path): + raise NotImplementedError("IdentityTransform does not support loading") + + def preprocess(self, sample): + return sample + + def image_augment(self, val, crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx: Optional[int], resample_mode: str = None): + return val + + def postprocess(self, sample): + return sample + + +class JSONTransform(AbstractTransform): + + def load(self, path): + if path.endswith('.json'): + with open(path, 'r') as f: + sample = json.load(f) + elif path.endswith('.json.gz'): + with gzip.open(path, 'rb') as f: + sample = json.load(f) + return sample + + def preprocess(self, sample): + return sample + + def image_augment(self, val, crop_coords: Tuple, flip: bool, orig_size: Tuple, target_size: Tuple, + rand_aug_idx: Optional[int], resample_mode: str = None): + return val + + def postprocess(self, sample): + return sample \ No newline at end of file diff --git a/fourm/data/multimodal_dataset_folder.py b/fourm/data/multimodal_dataset_folder.py new file mode 100644 index 0000000000000000000000000000000000000000..07ccf96ceed1f6aa7d533285ee533a61e3c43a8d --- /dev/null +++ b/fourm/data/multimodal_dataset_folder.py @@ -0,0 +1,363 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import os.path +import pickle +import random +from copy import deepcopy +from typing import Any, Callable, Dict, List, Optional, Tuple, cast + +import numpy as np +from torchvision.datasets.vision import VisionDataset + +from fourm.data.modality_transforms import AbstractTransform, get_transform_key + +IMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp', '.jpx', '.npy', '.npz') + +UNIFIED_EXTENSIONS = IMG_EXTENSIONS + ('.json', '.txt', '.json.gz') + + +def has_file_allowed_extension(filename: str, extensions: Tuple[str, ...]) -> bool: + """Checks if a file is an allowed extension. + + Args: + filename (string): path to a file + extensions (tuple of strings): extensions to consider (lowercase) + + Returns: + bool: True if the filename ends with one of given extensions + """ + return filename.lower().endswith(extensions) + + +def is_image_file(filename: str) -> bool: + """Checks if a file is an allowed image extension. + + Args: + filename (string): path to a file + + Returns: + bool: True if the filename ends with a known image extension + """ + return has_file_allowed_extension(filename, IMG_EXTENSIONS) + + +def make_dataset( + directory: str, + class_to_idx: Dict[str, int], + extensions: Optional[Tuple[str, ...]] = None, + is_valid_file: Optional[Callable[[str], bool]] = None, + cache_path: Optional[str] = None, +) -> List[Tuple[str, int]]: + if cache_path is not None and os.path.exists(cache_path): + # Load cached file paths from disk if it exists + with open(cache_path, 'rb') as f: + return pickle.load(f) + instances = [] + directory = os.path.expanduser(directory) + both_none = extensions is None and is_valid_file is None + both_something = extensions is not None and is_valid_file is not None + if both_none or both_something: + raise ValueError("Both extensions and is_valid_file cannot be None or not None at the same time") + if extensions is not None: + def is_valid_file(x: str) -> bool: + return has_file_allowed_extension(x, cast(Tuple[str, ...], extensions)) + is_valid_file = cast(Callable[[str], bool], is_valid_file) + for target_class in sorted(class_to_idx.keys()): + class_index = class_to_idx[target_class] + target_dir = os.path.join(directory, target_class) + if not os.path.isdir(target_dir): + continue + for root, _, fnames in sorted(os.walk(target_dir, followlinks=True)): + for fname in sorted(fnames): + path = os.path.join(root, fname) + if is_valid_file(path): + item = path, class_index + instances.append(item) + if cache_path is not None: + # Cache all file paths s.t. setting up the dataloader is instant in the future + os.makedirs(os.path.dirname(cache_path), exist_ok=True) + with open(cache_path, 'wb') as f: + pickle.dump(instances, f) + return instances + + +class DatasetFolder(VisionDataset): + """A generic data loader where the samples are arranged in this way: :: + + root/class_x/xxx.ext + root/class_x/xxy.ext + root/class_x/xxz.ext + + root/class_y/123.ext + root/class_y/nsdf3.ext + root/class_y/asd932_.ext + + Args: + root (string): Root directory path. + loader (callable): A function to load a sample given its path. + extensions (tuple[string]): A list of allowed extensions. + both extensions and is_valid_file should not be passed. + transform (callable, optional): A function/transform that takes in + a sample and returns a transformed version. + E.g, ``transforms.RandomCrop`` for images. + target_transform (callable, optional): A function/transform that takes + in the target and transforms it. + is_valid_file (callable, optional): A function that takes path of a file + and check if the file is a valid file (used to check of corrupt logs) + both extensions and is_valid_file should not be passed. + + Attributes: + classes (list): List of the class names sorted alphabetically. + class_to_idx (dict): Dict with items (class_name, class_index). + samples (list): List of (sample path, class_index) tuples + targets (list): The class_index value for each image in the dataset + """ + + def __init__( + self, + root: str, + loader: Callable[[str], Any], + extensions: Optional[Tuple[str, ...]] = None, + transform: Optional[Callable] = None, + target_transform: Optional[Callable] = None, + is_valid_file: Optional[Callable[[str], bool]] = None, + ) -> None: + super(DatasetFolder, self).__init__(root, transform=transform, + target_transform=target_transform) + classes, class_to_idx = self._find_classes(self.root) + samples = make_dataset(self.root, class_to_idx, extensions, is_valid_file) + if len(samples) == 0: + msg = "Found 0 logs in subfolders of: {}\n".format(self.root) + if extensions is not None: + msg += "Supported extensions are: {}".format(",".join(extensions)) + raise RuntimeError(msg) + + self.loader = loader + self.extensions = extensions + + self.classes = classes + self.class_to_idx = class_to_idx + self.samples = samples + self.targets = [s[1] for s in samples] + + def _find_classes(self, dir: str) -> Tuple[List[str], Dict[str, int]]: + """ + Finds the class folders in a dataset. + + Args: + dir (string): Root directory path. + + Returns: + tuple: (classes, class_to_idx) where classes are relative to (dir), and class_to_idx is a dictionary. + + Ensures: + No class is a subdirectory of another. + """ + classes = [d.name for d in os.scandir(dir) if d.is_dir()] + classes.sort() + class_to_idx = {cls_name: i for i, cls_name in enumerate(classes)} + return classes, class_to_idx + + def __getitem__(self, index: int) -> Tuple[Any, Any]: + """ + Args: + index (int): Index + + Returns: + tuple: (sample, target) where target is class_index of the target class. + """ + while True: + try: + path, target = self.samples[index] + sample = self.loader(path) + break + except Exception as e: + print(e) + index = random.randint(0, len(self.samples) - 1) + + if self.transform is not None: + sample = self.transform(sample) + if self.target_transform is not None: + target = self.target_transform(target) + + return sample, target + + def __len__(self) -> int: + return len(self.samples) + + +class MultiModalDatasetFolder(VisionDataset): + """A generic multi-modal dataset loader where the samples are arranged in this way: :: + + root/modality_a/class_x/xxx.ext + root/modality_a/class_y/xxy.ext + root/modality_a/class_z/xxz.ext + + root/modality_b/class_x/xxx.ext + root/modality_b/class_y/xxy.ext + root/modality_b/class_z/xxz.ext + + Args: + root (string): Root directory path. + modalities (list): List of modalities as strings + modality_paths (dict): Dict of paths to modalities + modality_transforms (dict): Dict of transforms for each modality + loader (callable): A function to load a sample given its path. + transform (callable, optional): A function/transform that takes in + a sample and returns a transformed version. + E.g, ``transforms.RandomCrop`` for images. + target_transform (callable, optional): A function/transform that takes + in the target and transforms it. + is_valid_file (callable, optional): A function that takes path of a file + and check if the file is a valid file (used to check of corrupt logs) + both extensions and is_valid_file should not be passed. + max_samples (int, optional): Maximum number of samples to load. If None, all samples are loaded. + pre_shuffle (bool, optional): Whether to shuffle the sample during the init. + return_paths (bool, optional): Whether to return the paths of the samples. + cache (bool, optional): Whether to cache the samples in memory. If True, the samples are loaded only once and then cached in memory. + + Attributes: + classes (list): List of the class names sorted alphabetically. + class_to_idx (dict): Dict with items (class_name, class_index). + samples (list): List of (sample path, class_index) tuples + targets (list): The class_index value for each image in the dataset + """ + + def __init__( + self, + root: str, + modalities: List[str], + modality_paths: Dict[str, str], + modality_transforms: Dict[str, AbstractTransform], + transform: Optional[Callable] = None, + target_transform: Optional[Callable] = None, + is_valid_file: Optional[Callable[[str], bool]] = None, + max_samples: Optional[int] = None, + pre_shuffle: bool = False, + cache: bool = False, + return_path: bool = False, + ) -> None: + super(MultiModalDatasetFolder, self).__init__(root, transform=transform, target_transform=target_transform) + self.modalities = modalities + # If modality_paths is not provided, use the default paths + self.modality_paths = modality_paths + for mod in self.modalities: + if mod not in self.modality_paths: + modality_paths[mod] = mod + self.modality_transforms = modality_transforms + self.return_path = return_path + + classes, class_to_idx = self._find_classes(os.path.join(self.root, list(self.modality_paths.values())[0])) + extensions = UNIFIED_EXTENSIONS if is_valid_file is None else None + + samples = { + mod: make_dataset( + os.path.join(self.root, f'{self.modality_paths[mod]}'), + class_to_idx, + extensions, + is_valid_file, + cache_path=os.path.join(self.root, 'dataloader_cache', f'{self.modality_paths[mod]}.pkl') if cache else None) + for mod in self.modalities + } + + for mod, mod_samples in samples.items(): + if len(mod_samples) == 0: + msg = "Found 0 logs in subfolders of: {}\n".format(os.path.join(self.root, f'{self.modality_paths[mod]}')) + if extensions is not None: + msg += "Supported extensions are: {}".format(",".join(extensions)) + raise RuntimeError(msg) + + self.extensions = extensions + + self.classes = classes + self.class_to_idx = class_to_idx + self.samples = samples + + # Select random subset of dataset if so specified + if isinstance(max_samples, int): + total_samples = len(list(self.samples.values())[0]) + np.random.seed(0) + permutation = np.random.permutation(total_samples) + for task in samples: + self.samples[task] = [self.samples[task][i] for i in permutation][:max_samples] + + if pre_shuffle: + total_samples = len(list(self.samples.values())[0]) + np.random.seed(100) + permutation = np.random.permutation(total_samples) + for task in samples: + self.samples[task] = [self.samples[task][i] for i in permutation] + + self.cache = {} + self.imgs = self.samples + + def _find_classes(self, dir: str) -> Tuple[List[str], Dict[str, int]]: + """ + Finds the class folders in a dataset. + + Args: + dir (string): Root directory path. + + Returns: + tuple: (classes, class_to_idx) where classes are relative to (dir), and class_to_idx is a dictionary. + + Ensures: + No class is a subdirectory of another. + """ + classes = [d.name for d in os.scandir(dir) if d.is_dir()] + classes.sort() + class_to_idx = {cls_name: i for i, cls_name in enumerate(classes)} + return classes, class_to_idx + + def get_class_and_file(self, path: str) -> Tuple[str, str]: + """ Extracts the class and file name from a path. """ + class_id, file_name = path.split('/')[-2:] + file_name = file_name.split('.')[0] + return class_id, file_name + + def __getitem__(self, index: int) -> Tuple[Any, Any]: + """ + Args: + index (int): Index + + Returns: + tuple: (sample, target) where target is class_index of the target class. + """ + if index in self.cache: + sample_dict, target = deepcopy(self.cache[index]) + else: + sample_dict = {} + for mod in self.modalities: + path, target = self.samples[mod][index] + sample = self.modality_transforms[get_transform_key(mod)].load(path) + sample_dict[mod] = sample + # self.cache[index] = deepcopy((sample_dict, target)) + + if self.transform is not None: + sample_dict = self.transform(sample_dict) + if self.target_transform is not None: + target = self.target_transform(target) + + sample_dict['class_idx'] = target + + if self.return_path and not index in self.cache: + class_id, file_name = self.get_class_and_file(path) + sample_dict['class_id'] = class_id + sample_dict['file_name'] = file_name + + return sample_dict + + def __len__(self) -> int: + return len(list(self.samples.values())[0]) diff --git a/fourm/data/pretrain_utils.py b/fourm/data/pretrain_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..08f871501b9787f5b5bf2223a14b280f4872135c --- /dev/null +++ b/fourm/data/pretrain_utils.py @@ -0,0 +1,292 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy + +import torch +import yaml + +import fourm.utils as utils + +from fourm.data import (CenterCropImageAugmenter, EmptyAugmenter, + PreTokenizedImageAugmenter,RandomCropImageAugmenter, build_fm_pretraining_dataset, + build_huggingface_pretraining_dataloader, + build_wds_fm_pretraining_dataloader) +from fourm.data.modality_transforms import CaptionTransform +from fourm.data.modality_info import MODALITY_TRANSFORMS + + +def setup_sampling_mod_info(dataset_config, modality_info): + # Subset of modality info for each dataset + + # Input and output modalities for one dataset + in_domains = sorted(dataset_config['in_domains'].split('-')) + out_domains = sorted(dataset_config['out_domains'].split('-')) + all_domains = sorted(list(set(in_domains) | set(out_domains))) + + mod_info = copy.deepcopy(modality_info) + mod_info = {mod: mod_info[mod] for mod in all_domains} + + # Dirichlet concentration parameter (Alpha) + if dataset_config.get('alphas_config', None) is None: + for mod in mod_info: + mod_info[mod]["input_alphas"] = [0.] + mod_info[mod]["target_alphas"] = [0.] + + if 'input_alphas' in dataset_config: + input_alphas = dataset_config['input_alphas'].split('-') + if len(input_alphas) == 1: + input_alphas = [float(input_alphas[0])] * len(in_domains) + else: + input_alphas = [float(alpha) for alpha in input_alphas] + for mod, alpha in zip(in_domains, input_alphas): + mod_info[mod]['input_alphas'] = [alpha] + + if 'target_alphas' in dataset_config: + target_alphas = dataset_config['target_alphas'].split('-') + if len(target_alphas) == 1: + target_alphas = [float(target_alphas[0])] * len(out_domains) + else: + target_alphas = [float(alpha) for alpha in target_alphas] + for mod, alpha in zip(out_domains, target_alphas): + mod_info[mod]["target_alphas"] = [alpha] + + sampling_weights = None + else: + print(f"Loading alphas config from: {dataset_config['alphas_config']}") + with open(dataset_config['alphas_config'], "r") as f: + alphas_config = yaml.safe_load(f) + + if 'sampling_weights' in alphas_config: + sampling_weights = alphas_config['sampling_weights'] + alphas_config = alphas_config['alphas_mixture'] + else: + sampling_weights = None + + for mod in mod_info: + mod_info[mod]["input_alphas"] = alphas_config[mod]["input_alphas"] + mod_info[mod]["target_alphas"] = alphas_config[mod]["target_alphas"] + if modality_info[mod]['type'] in ['seq', 'seq_emb', 'seq_token']: + mod_info[mod]['keep'] = alphas_config[mod]['keep'] + + return mod_info, sampling_weights + +def get_train_dataloader(dataset_config, modality_info, sampling_weights, text_tokenizer, input_size, + num_input_tokens, num_target_tokens, min_input_tokens, min_target_tokens, + num_tasks, num_workers, dataset_batch_size=None, epoch_size=None): + + in_domains = sorted(list(dataset_config['in_domains'].split('-'))) + out_domains = sorted(list(dataset_config['out_domains'].split('-'))) + all_domains = sorted(list(set(in_domains) | set(out_domains))) + + modality_transforms = MODALITY_TRANSFORMS + if 'caption' in modality_transforms: + modality_transforms['caption'] = CaptionTransform( + aligned_captions=dataset_config.get('aligned_captions', True) + ) + + if dataset_config['type'] == 'multimodal': + + is_pretokenized = any([modality_info[mod].get('pretokenized', False) for mod in modality_info]) + if is_pretokenized: + # Multi-modal training data augmentation (uses pre-tokenized data augmentation) + image_augmenter = PreTokenizedImageAugmenter( + target_size=input_size, + no_aug=(not dataset_config.get('tok_train_aug', True)), + main_domain=dataset_config['main_augment_domain'] + ) + else: + image_augmenter = RandomCropImageAugmenter( + target_size=input_size, + hflip=dataset_config.get('hflip'), + crop_scale=tuple(dataset_config.get('crop_scale')), + crop_ratio=tuple(dataset_config.get('crop_ratio')), + ) + + # Input and target token ranges + num_input_tokens = dataset_config.get('num_input_tokens', num_input_tokens) + num_target_tokens = dataset_config.get('num_target_tokens', num_target_tokens) + min_input_tokens = dataset_config.get('min_input_tokens', min_input_tokens) + min_target_tokens = dataset_config.get('min_target_tokens', min_target_tokens) + min_input_tokens = num_input_tokens if min_input_tokens is None else min_input_tokens + min_target_tokens = num_target_tokens if min_target_tokens is None else min_target_tokens + + + if dataset_config['use_wds']: + # Using webdataset + loader = build_wds_fm_pretraining_dataloader( + data_path=dataset_config['data_path'], all_domains=all_domains, + modality_info=modality_info, modality_transforms=modality_transforms, + image_augmenter=image_augmenter, text_tokenizer=text_tokenizer, + input_tokens_range=(min_input_tokens, num_input_tokens), + target_tokens_range=(min_target_tokens, num_target_tokens), + num_gpus=num_tasks, num_workers=num_workers, + batch_size=dataset_batch_size, epoch_size=epoch_size, + modality_name_map=dataset_config.get('modality_name_map', None), + shuffle_buffer_load=dataset_config.get('wds_shuffle_buffer_tar', 1_000), + shuffle_buffer_repeat=dataset_config.get('wds_shuffle_buffer_repeat', 1_000), + n_repeats=dataset_config.get('wds_n_repeats', 1), + sampling_weights=sampling_weights, + ) + else: + dataset_train = build_fm_pretraining_dataset( + data_path=dataset_config['data_path'], + all_domains=all_domains, modality_info=modality_info, modality_transforms=modality_transforms, + image_augmenter=image_augmenter, text_tokenizer=text_tokenizer, + input_tokens_range=(min_input_tokens, num_input_tokens), + target_tokens_range=(min_target_tokens, num_target_tokens) + ) + sampler_train = torch.utils.data.DistributedSampler( + dataset_train, num_replicas=num_tasks, rank=utils.get_rank(), shuffle=True, drop_last=True, + ) + # DataLoader has batch size 1 as it then gets collated through the Mixture dataloader + loader = torch.utils.data.DataLoader( + dataset_train, sampler=sampler_train, + batch_size=1, num_workers=0, + pin_memory=False, drop_last=True, + collate_fn=lambda x: x[0], + ) + + elif dataset_config['type'] == 'huggingface': + + # Input and target token ranges + num_input_tokens = dataset_config.get('num_input_tokens', num_input_tokens) + num_target_tokens = dataset_config.get('num_target_tokens', num_target_tokens) + + if dataset_config.get('use_wds', False): + raise NotImplementedError('Webdataset not yet implemented for huggingface datasets.') + else: + loader = build_huggingface_pretraining_dataloader( + data_path=dataset_config['data_path'], all_domains=all_domains, + modality_info=modality_info, modality_transforms=modality_transforms, + image_augmenter=EmptyAugmenter(), text_tokenizer=text_tokenizer, + input_tokens_range=(num_input_tokens, num_input_tokens), + target_tokens_range=(num_target_tokens, num_target_tokens), + num_gpus=num_tasks, num_workers=num_workers, + batch_size=dataset_batch_size, epoch_size=epoch_size, + split='train', streaming=True, rename_text_to_caption=True, + shuffle_buffer_load=dataset_config.get('shuffle_buffer_load', 1_000), + shuffle_seed=0, + ) + else: + raise NotImplementedError(f'Dataset type {dataset_config["type"]} not implemented.') + + return loader + + +def cfgs_get(key, val_config, dataset_name, train_configs, default=None): + """ Try to retrieve a key from the validation set config. + If it does not exist, default to retrieving it from the train set config + with the same dataset name. + """ + return val_config.get(key, train_configs[dataset_name].get(key, default)) + + +def get_val_dataloader(dataset_config, dataset_name, train_configs, modality_info, sampling_weights, text_tokenizer, + input_size, num_input_tokens, num_target_tokens, min_input_tokens, min_target_tokens, + fixed_eval, fixed_eval_input_tokens, fixed_eval_target_tokens, + dist_eval, num_tasks, num_workers, batch_size, pin_mem): + + in_domains = sorted(list(cfgs_get('in_domains', dataset_config, dataset_name, train_configs).split('-'))) + out_domains = sorted(list(cfgs_get('out_domains', dataset_config, dataset_name, train_configs).split('-'))) + all_domains = sorted(list(set(in_domains) | set(out_domains))) + + modality_transforms = MODALITY_TRANSFORMS + if 'caption' in modality_transforms: + modality_transforms['caption'] = CaptionTransform( + aligned_captions=cfgs_get('aligned_captions', dataset_config, dataset_name, train_configs, True) + ) + + dataset_type = cfgs_get('type', dataset_config, dataset_name, train_configs) + + if dataset_type == 'multimodal': + + main_augment_domain = cfgs_get('main_augment_domain', dataset_config, dataset_name, train_configs) + is_pretokenized = any([modality_info[mod].get('pretokenized', False) for mod in modality_info]) + if is_pretokenized: + eval_image_augmenter = PreTokenizedImageAugmenter( + target_size=input_size, no_aug=True, main_domain=main_augment_domain + ) + else: + eval_image_augmenter = CenterCropImageAugmenter( + target_size=input_size, main_domain=main_augment_domain + ) + + + if fixed_eval: + input_tokens_range=(fixed_eval_input_tokens, fixed_eval_input_tokens) + target_tokens_range=(fixed_eval_target_tokens, fixed_eval_target_tokens) + else: + # Input and target token ranges + num_input_tokens = dataset_config.get('num_input_tokens', num_input_tokens) + num_target_tokens = dataset_config.get('num_target_tokens', num_target_tokens) + min_input_tokens = dataset_config.get('min_input_tokens', min_input_tokens) + min_target_tokens = dataset_config.get('min_target_tokens', min_target_tokens) + min_input_tokens = num_input_tokens if min_input_tokens is None else min_input_tokens + min_target_tokens = num_target_tokens if min_target_tokens is None else min_target_tokens + input_tokens_range = (min_input_tokens, num_input_tokens) + target_tokens_range = (min_target_tokens, num_target_tokens) + + dataset_val = build_fm_pretraining_dataset( + data_path=cfgs_get('data_path', dataset_config, dataset_name, train_configs), + all_domains=all_domains, modality_info=modality_info, modality_transforms=modality_transforms, + image_augmenter=eval_image_augmenter, text_tokenizer=text_tokenizer, + input_tokens_range=input_tokens_range, target_tokens_range=target_tokens_range + ) + + print("Warning: Eval stats may vary slightly as the masking applied on images is random.") + if dist_eval: + if len(dataset_val) % num_tasks != 0: + print('Warning: Enabling distributed evaluation with an eval dataset not divisible by process number. ' + 'This will slightly alter validation results as extra duplicate entries are added to achieve ' + 'equal num of samples per-process.') + sampler_val = torch.utils.data.DistributedSampler( + dataset_val, num_replicas=num_tasks, rank=utils.get_rank(), shuffle=False) + else: + sampler_val = torch.utils.data.SequentialSampler(dataset_val) + loader = torch.utils.data.DataLoader( + dataset_val, sampler=sampler_val, + batch_size=batch_size, + num_workers=num_workers, + pin_memory=pin_mem, + drop_last=False, + ) + + elif dataset_type == 'huggingface': + + if fixed_eval: + input_tokens_range=(fixed_eval_input_tokens, fixed_eval_input_tokens) + target_tokens_range=(fixed_eval_target_tokens, fixed_eval_target_tokens) + else: + # Input and target token ranges + num_input_tokens = dataset_config.get('num_input_tokens', num_input_tokens) + num_target_tokens = dataset_config.get('num_target_tokens', num_target_tokens) + input_tokens_range = (num_input_tokens, num_input_tokens) + target_tokens_range = (num_target_tokens, num_target_tokens) + + loader = build_huggingface_pretraining_dataloader( + data_path=cfgs_get('data_path', dataset_config, dataset_name, train_configs), + all_domains=all_domains, modality_info=modality_info, modality_transforms=modality_transforms, + image_augmenter=EmptyAugmenter(), text_tokenizer=text_tokenizer, + input_tokens_range=input_tokens_range, target_tokens_range=target_tokens_range, + num_gpus=num_tasks, num_workers=num_workers, + batch_size=batch_size, epoch_size=None, + split='validation', streaming=True, rename_text_to_caption=True, + shuffle_buffer_load=cfgs_get('shuffle_buffer_load', dataset_config, dataset_name, train_configs, 1_000), + shuffle_seed=0, + ) + + else: + raise NotImplementedError(f'Dataset type {dataset_type} not implemented.') + + return loader \ No newline at end of file diff --git a/fourm/data/transfer_utils.py b/fourm/data/transfer_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fd4407869077f21bb4926144ba2305f6e42142fd --- /dev/null +++ b/fourm/data/transfer_utils.py @@ -0,0 +1,53 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch + +def convert_samples_to_mod_dict(samples, input_mod, target_mod, num_input_tokens, num_target_tokens): + """Converts a sample (e.g. a batch of RGB images) to a mod dict that can be passed directly to FourM. + Assumes both the input modality and target modality are dense tasks. + """ + + B = samples.shape[0] + device = samples.device + + if input_mod == target_mod: + assert(num_input_tokens == num_target_tokens) + mod_dict = { + input_mod: { + 'tensor': samples, + 'input_mask': torch.zeros((B, num_input_tokens), dtype=torch.bool, device=device), + 'target_mask': torch.zeros((B, num_target_tokens), dtype=torch.bool, device=device), + 'decoder_attention_mask': torch.zeros((B, num_target_tokens), dtype=torch.int, device=device), + }, + } + mod_dict[input_mod]['decoder_attention_mask'][:, 0] = num_target_tokens + + else: + mod_dict = { + input_mod: { + 'tensor': samples, + 'input_mask': torch.zeros((B, num_input_tokens), dtype=torch.bool, device=samples.device), + 'target_mask': torch.ones((B, num_input_tokens), dtype=torch.bool, device=samples.device), + 'decoder_attention_mask': torch.zeros((B, num_input_tokens), dtype=torch.int, device=samples.device), + }, + target_mod: { + 'tensor': torch.zeros((B, num_target_tokens), dtype=torch.long, device=samples.device), + 'input_mask': torch.ones((B, num_target_tokens), dtype=torch.bool, device=samples.device), + 'target_mask': torch.zeros((B, num_target_tokens), dtype=torch.bool, device=samples.device), + 'decoder_attention_mask': torch.ones((B, num_target_tokens), dtype=torch.int, device=samples.device), + }, + } + mod_dict[target_mod]['decoder_attention_mask'][:, 0] = num_target_tokens + + return mod_dict diff --git a/fourm/data/unified_datasets.py b/fourm/data/unified_datasets.py new file mode 100644 index 0000000000000000000000000000000000000000..7369731e6cd7aa999ab921cedbd9ee2afe2dd3ae --- /dev/null +++ b/fourm/data/unified_datasets.py @@ -0,0 +1,557 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy +import io +import itertools +import os +import re +from functools import partial +from typing import Any, Callable, Dict, Iterable, List, Optional + +import braceexpand +import numpy as np +import torch +import webdataset as wds +from PIL import Image +from torch.utils.data import IterableDataset +from torch.utils.data._utils.collate import default_collate +from torchvision import transforms +from webdataset.filters import pipelinefilter, reraise_exception +from webdataset.handlers import warn_and_continue + +try: + # Optionally load huggingface datasets + from datasets import load_dataset + from datasets.distributed import split_dataset_by_node +except ImportError: + print("Huggingface datasets not installed. Please install with `pip install datasets`.") + +from fourm.data.masking import TransferMasking, UnifiedMasking +from fourm.data.modality_transforms import (CropSettingsTransform, IdentityTransform, + MaskTransform, UnifiedDataTransform, + get_transform_key) +from fourm.data.multimodal_dataset_folder import MultiModalDatasetFolder +from fourm.utils.dist import get_rank, get_world_size + + +def build_fm_pretraining_dataset( + data_path, all_domains, modality_info, modality_transforms, + image_augmenter, text_tokenizer, + input_tokens_range, target_tokens_range, + sampling_weights=None): + """Builds the FourM pre-training dataset based on the given arguments. + This function should mainly used for smaller datasets (e.g. validation sets), + while large training sets should be loaded with build_wds_fm_pretraining_dataloader in webdataset format. + + Args: + data_path: Path to the dataset. + all_domains: List of all modalities to be used. + modality_info: Dictionary containing information about the modalities. + modality_transforms: Dictionary containing the transforms for each modality. + image_augmenter: Image augmenter. + text_tokenizer: Text tokenizer (for sequence modalities). + input_tokens_range: Range of the input token budget. + target_tokens_range: Range of the target token budget. + sampling_weights: Sampling weights for the mixture of Dirichlet distributions. + + Returns: + FourM pre-training dataset as a PyTorch Dataset. + """ + + transform = transforms.Compose([ + UnifiedDataTransform(transforms_dict=modality_transforms, image_augmenter=image_augmenter), + UnifiedMasking(modality_info=modality_info, text_tokenizer=text_tokenizer, + input_tokens_range=input_tokens_range, target_tokens_range=target_tokens_range, + sampling_weights=sampling_weights), + ]) + + # Remove vq domains that require a tokenizer + modalities_without_vq = [mod for mod in all_domains if not modality_info[mod].get("requires_tokenizer", False)] + # If we are using a pre-tokenized modality, we default to pre-computed crop settings + if any([modality_info[domain].get("pretokenized", False) for domain in all_domains]): + modalities_without_vq.append("crop_settings") + modality_transforms = copy.deepcopy(modality_transforms) + modality_transforms["crop_settings"] = CropSettingsTransform() + + modality_paths = {mod: modality_info[mod]['path'] for mod in modality_info if modality_info[mod].get('path', None) is not None} + + return MultiModalDatasetFolder(root=data_path, modalities=modalities_without_vq, modality_paths=modality_paths, + modality_transforms=modality_transforms, transform=transform) + + +def build_fm_transfer_dataset( + data_path, modality_info, transform, modality_transforms, all_domains, + load_mask_valid: bool = False, max_samples: Optional[int] = None, + pre_shuffle: bool = False, cache: bool = False): + """Builds the FourM transfer dataset based on the given arguments. + + Args: + data_path: Path to the dataset. + modality_info: Dictionary containing information about the modalities. + transform: Transform to be applied to the dataset. + modality_transforms: Dictionary containing the transforms for each modality. + all_domains: List of all modalities to be used. + load_mask_valid: Whether to load the mask_valid "modality". + max_samples: Maximum number of samples to load. + pre_shuffle: Whether to shuffle the dataset before loading. + cache: Whether to cache the dataset in memory. + + Returns: + FourM transfer dataset as a PyTorch Dataset. + """ + + # Remove vq domains that require a tokenizer + modalities_without_vq = [mod for mod in all_domains if not modality_info[mod].get("requires_tokenizer", False)] + # If we are using a pre-tokenized modality, we default to pre-computed crop settings + if any([modality_info[domain].get("pretokenized", False) for domain in all_domains]): + modalities_without_vq.append("crop_settings") + modality_transforms = copy.deepcopy(modality_transforms) + modality_transforms["crop_settings"] = CropSettingsTransform() + + if load_mask_valid: + modalities_without_vq.append("mask_valid") + modality_transforms = copy.deepcopy(modality_transforms) + modality_transforms["mask_valid"] = MaskTransform() + + modality_paths = {mod: modality_info[mod]['path'] for mod in modality_info if modality_info[mod].get('path', None) is not None} + + return MultiModalDatasetFolder(root=data_path, modalities=modalities_without_vq, modality_paths=modality_paths, + modality_transforms=modality_transforms, transform=transform, max_samples=max_samples, + pre_shuffle=pre_shuffle, cache=cache) + + +### Webdatasets (wds) functions + +def _keyless_map(data, f, handler=reraise_exception): + """Map samples without adding __key__.""" + for sample in data: + try: + result = f(sample) + except Exception as exn: + if handler(exn): + continue + else: + break + if result is None: + continue + yield result + +map = pipelinefilter(_keyless_map) + +def check_dots(s): + if '.gz' in s: + return s.count('.') == 2 + return s.count('.') == 1 + +def remove_ext_with_gz(s): + if s.endswith('.gz'): + s = s.replace(".gz", "") + return os.path.splitext(s)[0] + +def wds_decoder(key, value): + if key == "png" or key.endswith(".png"): + img = Image.open(io.BytesIO(value)) + return img + elif key == "jpg" or key.endswith(".jpg"): + img = Image.open(io.BytesIO(value)) + return img + elif key == "jpeg" or key.endswith(".jpeg"): + img = Image.open(io.BytesIO(value)) + return img + elif key == 'npy' or key.endswith("npy"): + content = np.load(io.BytesIO(value), allow_pickle=True) + # try: + # content = np.load(io.BytesIO(value)) + # except: + # content = np.load(io.BytesIO(value), allow_pickle=True) + return content + elif key == "jpx" or key.endswith('.jpx'): + img = Image.open(io.BytesIO(value)) + return img + elif 'output' in key: + return int(value) + else: + # If not an image, use the basic handlers (.txt, .json, .pickle, .npz, ...) + # See https://github.com/webdataset/webdataset/blob/main/webdataset/autodecode.py + return None + +def repeat_fn(src, n_repeats=5): + """ + Repeat each sample n_repeats times. + E.g. A B C ... repeated 3 times becomes A A A B B B C C C ... + Depending on the downstream application, a shuffle should be added after this. + """ + for sample in src: + for _ in range(n_repeats): + yield sample + +def remove_extensions(sample): + """ + In webdatasets, we identify the type of a given modality by adding an extension + in the form f"{modality_name}.{modality_extension}", e.g. "rgb.jpg" or "caption.json". + This function removes them and returns a dictionary of {f"{modality_name}": modality}. + """ + return {remove_ext_with_gz(k): v for k, v in sample.items()} + +def filter_metadata(sample, metadata=['__key__', '__url__', 'file_name', 'class_name', 'class_idx']): + """ Filters out non-modality entries specified in metadata when loading tar files with webdatasets. """ + return {k: v for k, v in sample.items() if k not in metadata} + +def apply_modality_transforms(sample, modality_transforms): + """ Applies a dictionary of modality-specific transforms to a dictionary of modalities. """ + return {k: (modality_transforms[get_transform_key(k)](v) if k in modality_transforms else v) for k, v in sample.items() } + +def tok_to_int64(sample): + """ + Pre-computed tokens are saved as int16, but we need them as int64 instead. + """ + return {k: (v.astype('int64') if 'tok_' in k else v) for k, v in sample.items()} + +def rename_modalities(sample, modality_paths): + """ + Renames modalities to their corresponding names in modality_paths. + """ + return {out_path: sample[loaded_path] for out_path, loaded_path in modality_paths.items()} + +def extract_modality_names(s): + # Regular expression pattern to match anything enclosed in '{' and '}', and comma separated + pattern = r'\{([^}]*)\}' + match = re.search(pattern, s) + return match.group(1).split(',') if match else [] + +def identity(sample): + """ Identity function that does nothing. """ + return sample + +def multi_tarfile_samples(src_iter: Iterable[Dict[str, Any]], + modality_name_map: Dict[str, str] = None, + handler: Callable[[Exception], bool] = warn_and_continue): + """Webdataset does not support splitting up shards by modality, so we need to do this manually. + Usually, we would need to save all modalities in the same tar file, e.g. shard_root_train/{00000..12345}.tar, + where each shard contains 1000 samples and each sample contains all modalities. + This is not flexible when adding new modalities, so we instead save each modality in a separate tar file, + e.g. shard_root_train_rgb/{00000..12345}.tar, shard_root_train_caption/{00000..12345}.tar, etc., where each shard contains + again 1000 samples, but each sample contains only one modality. All samples in all shards have to be aligned. + + This function takes an iterator over shard URLs, where we use brace expansion to specify multiple tar files per modality. + E.g. shard_root_train_[rgb,caption]/00123.tar will be expanded to shard_root_train_rgb/00123.tar and shard_root_train_caption/00123.tar, + and the samples from these two tar files will be combined into a single sample. + + Args: + src_iter: Iterator over shards that *already brace expanded the shard numbers*, + e.g. {'url': 'shard_root_train_[rgb,caption]/00000.tar'}, {'url': 'shard_root_train_[rgb,caption]/00001.tar'}, ... + This function will also work when no square braces for multiple modalities are used, e.g. {'url': 'shard_root_train/00000.tar'}, ... + It can be a drop-in replacement for wds.tarfile_samples. + modality_name_map: Optional dictionary specifying a mapping from modality folder names to arbitrary other names. + handler: Function that handles exceptions. If it returns True, the shard is skipped. If it returns False, the function exits. + + Yields: + Dictionary of aligned samples from all modalities. + """ + for src in src_iter: + + # Multi tar file URLs use brace expansion with square braces + multi_tar_urls = src['url'].translate(str.maketrans('[]', '{}')) + modality_names = extract_modality_names(multi_tar_urls) + if len(modality_names) == 0: + # Case where multi-modal braceexpand is not used, e.g. shard_dir/shard00000.tar + modality_names = [None] + multi_tar_urls = [multi_tar_urls] + elif len(modality_names) == 1: + # Brace expand doesn't work with a single entry, e.g. shard_dir/[foo]/shard00000.tar + multi_tar_urls = [multi_tar_urls.replace("{", "").replace("}", "")] + else: + # Remaining cases where multiple modalities are specified, e.g. shard_dir/[foo,bar]/shard00000.tar + multi_tar_urls = list(braceexpand.braceexpand(multi_tar_urls)) + + # Create tar iterators for shards of all modalities + tar_iters = [wds.tarfile_samples([{'url': tar_url}]) for tar_url in multi_tar_urls] + + try: + # Loop over these iterators in parallel and combine the tar files from different modalities + for multi_tar_files in zip(*tar_iters): + + merged_dict = {} + merged_dict['__key__'] = multi_tar_files[0]['__key__'] + merged_dict['__url__'] = src['url'] + + for modality_name, modality_dict in zip(modality_names, multi_tar_files): + _key = modality_dict.pop('__key__') + _url = modality_dict.pop('__url__') + + if _key != merged_dict['__key__']: + raise ValueError(f"Divergence detected! Trying to merge keys {_key} of {modality_name} and {merged_dict['__key__']} of merged_dict with modalities {merged_dict.keys()}.") + + tar_is_multimodal = len(modality_dict) > 1 + for k, v in modality_dict.items(): + if tar_is_multimodal or check_dots(k) or modality_name is None: + # We don't change the keys in the following cases: + # 1. The shard contains multiple modalities. Then they *have* to follow the idx.modality_id.ext convention + # 2. If any key contains a dot, this means it already has the idx.modality_id.ext format (idx. is already removed at this stage) + # 3. If the modality name is None, no modality folder was specified (see beginning of function) + merged_dict[k] = v + else: + mapped_name = modality_name if modality_name_map is None else modality_name_map.get(modality_name, modality_name) + merged_dict[f'{mapped_name}.{k}'] = v + + yield merged_dict + + except Exception as e: + print(e) + print(f"Exception occurred while processing {src['url']}.") + if handler(e): + print('Skipping shard...') + continue + else: + break + +def build_wds_fm_pretraining_dataloader( + data_path, all_domains, modality_info, modality_transforms, image_augmenter, + text_tokenizer, input_tokens_range, target_tokens_range, + num_gpus, num_workers, batch_size, epoch_size, sampling_weights=None, modality_name_map=None, + shuffle_buffer_load=1000, shuffle_buffer_repeat=5000, n_repeats=5): + """Builds the WebDataset FourM pre-training dataloader based on the given arguments. + + Args: + data_path: Path to the dataset. + all_domains: List of all modalities to be used. + modality_info: Dictionary containing information about the modalities. + modality_transforms: Dictionary containing the transforms for each modality. + image_augmenter: Image augmenter. + text_tokenizer: Text tokenizer (for sequence modalities). + input_tokens_range: Range of the input token budget. + target_tokens_range: Range of the target token budget. + num_gpus: Number of GPUs. + num_workers: Number of workers. + batch_size: Batch size. + epoch_size: Number of samples per "epoch". (Here, epoch refers to an interrupted training loop without evaluation or checkpointing). + sampling_weights: Sampling weights for the mixture of Dirichlet distributions. + modality_name_map: Optional dictionary specifying a mapping from modality folder names to arbitrary other names. + shuffle_buffer_load: Shuffle buffer size when loading samples from tar files (first shuffle). + shuffle_buffer_repeat: Shuffle buffer size after repeating samples (second shuffle). + n_repeats: Number of times to repeat each sample. + + Returns: + FourM pre-training dataloader as a WebDataset DataLoader. + """ + + modality_paths = {mod: modality_info[mod].get('path', None) or mod for mod in modality_info} + + # Remove vq domains that require a tokenizer + modalities_without_vq = [mod for mod in all_domains if not modality_info[mod].get("requires_tokenizer", False)] + # If we are using a pre-tokenized modality, we default to pre-computed crop settings + if any([modality_info[domain].get("pretokenized", False) for domain in all_domains]): + modalities_without_vq.append("crop_settings") + modality_transforms = copy.deepcopy(modality_transforms) + modality_transforms["crop_settings"] = CropSettingsTransform() + modality_paths["crop_settings"] = "crop_settings" + + # Webdatasets always adds __key__ to the dictionary, so we add a transform that does nothing with it + modality_transforms["__key__"] = IdentityTransform() + + transform = transforms.Compose([ + UnifiedDataTransform(transforms_dict=modality_transforms, image_augmenter=image_augmenter), + UnifiedMasking(modality_info=modality_info, text_tokenizer=text_tokenizer, + input_tokens_range=input_tokens_range, target_tokens_range=target_tokens_range, + sampling_weights=sampling_weights) + ]) + + datapipe = wds.DataPipeline( + # Infinitely sample shards from the shard list with replacement. Each worker is seeded independently. + wds.ResampledShards(data_path), + partial(multi_tarfile_samples, modality_name_map=modality_name_map), # Extract individual samples from single or multi-modal tar files + wds.shuffle(shuffle_buffer_load), # Shuffle with a buffer of given size + wds.decode(wds_decoder), # Decode from bytes to PIL images, numpy arrays, etc. + wds.filters.compose(partial(repeat_fn, n_repeats=n_repeats)), # Repeats each sample n times -> A A A B B B C C C ... + wds.shuffle(shuffle_buffer_repeat), # Shuffle again with a buffer of given size + wds.map(remove_extensions), # Remove "file extensions" from dictionary keys + map(filter_metadata), # Remove non-task keys + map(tok_to_int64), # Convert pre-computed tokens to int64 + map(partial(rename_modalities, modality_paths=modality_paths)), # Rename modalities to their corresponding names in modality_paths + map(transform), # Apply data augmentation and masking + wds.batched(batch_size, collation_fn=default_collate, partial=False) + if batch_size is not None else map(identity), # Batching + ) + + if epoch_size is not None: + batch_size_iter = batch_size if batch_size is not None else 1 + datapipe = datapipe.with_epoch(epoch_size // (num_gpus * num_workers * batch_size_iter)) # Pre-define iterator length + + if batch_size is not None: + # Perform multi-threaded dataloading + return wds.WebLoader(datapipe, num_workers=num_workers, batch_size=None) + else: + return datapipe + + +def build_wds_divae_dataloader( + data_path, modality_info, modality_transforms, image_augmenter, + num_gpus, num_workers, batch_size, epoch_size, shuffle_buffer_load=1000, + shuffle_buffer_repeat=5000, n_repeats=1): + + modality_paths = {mod: modality_info[mod].get('path', None) or mod for mod in modality_info} + + # Webdatasets always adds __key__ to the dictionary, so we add a transform that does nothing with it + modality_transforms["__key__"] = IdentityTransform() + + transform = UnifiedDataTransform(transforms_dict=modality_transforms, image_augmenter=image_augmenter) + + datapipe = wds.DataPipeline( + # Infinitely sample shards from the shard list with replacement. Each worker is seeded independently. + wds.ResampledShards(data_path), + multi_tarfile_samples, # Extract individual samples from single or multi-modal tar files + wds.shuffle(shuffle_buffer_load), # Shuffle with a buffer of given size + wds.decode(wds_decoder), # Decode from bytes to PIL images, numpy arrays, etc. + wds.filters.compose(partial(repeat_fn, n_repeats=n_repeats)), # Repeats each sample n times -> A A A B B B C C C ... + wds.shuffle(shuffle_buffer_repeat), # Shuffle again with a buffer of given size + map(remove_extensions), # Remove "file extensions" from dictionary keys + map(filter_metadata), # Remove non-task keys + map(tok_to_int64), # Convert pre-computed tokens to int64 + map(partial(rename_modalities, modality_paths=modality_paths)), # Rename modalities to their corresponding names in modality_paths + map(transform), # Apply data augmentation and masking + wds.batched(batch_size, collation_fn=default_collate, partial=False) + if batch_size is not None else map(identity), # Batching + ) + + if epoch_size is not None: + batch_size_iter = batch_size if batch_size is not None else 1 + datapipe = datapipe.with_epoch(epoch_size // (num_gpus * num_workers * batch_size_iter)) # Pre-define iterator length + + if batch_size is not None: + # Perform multi-threaded dataloading + return wds.WebLoader(datapipe, num_workers=num_workers, batch_size=None) + else: + return datapipe + + +### Huggingface datasets functions + +def text_to_caption(sample): + """ Rename "text" to "caption". """ + return {'caption': sample['text']} + + +def build_huggingface_pretraining_dataloader( + data_path, all_domains, modality_info, modality_transforms, image_augmenter, + text_tokenizer, input_tokens_range, target_tokens_range, + num_gpus, num_workers, batch_size, epoch_size, split, + streaming=True, rename_text_to_caption=True, shuffle_buffer_load=10_000, shuffle_seed=0): + + # Load huggingface dataset and split samples across workers. Shuffle samples in each worker + dataset = load_dataset(data_path, split=split, streaming=streaming) + dataset = split_dataset_by_node(dataset, rank=get_rank(), world_size=get_world_size()) + dataset = dataset.shuffle(seed=shuffle_seed, buffer_size=shuffle_buffer_load) + + modality_info = {mod: modality_info[mod] for mod in modality_info if mod in all_domains} + + transform = transforms.Compose([ + UnifiedDataTransform(transforms_dict=modality_transforms, image_augmenter=image_augmenter), + UnifiedMasking(modality_info=modality_info, text_tokenizer=text_tokenizer, + input_tokens_range=input_tokens_range, target_tokens_range=target_tokens_range) + ]) + + datapipe = wds.DataPipeline( + dataset, + map(text_to_caption) if rename_text_to_caption else map(identity), # Rename "text" to "caption" + map(filter_metadata), # Remove non-task keys + map(transform), # Apply data augmentation and masking + wds.batched(batch_size, collation_fn=default_collate, partial=False) + if batch_size is not None else map(identity), # Batching + ) + + datapipe.n_shards = dataset.n_shards + num_workers = min(num_workers, dataset.n_shards) + + if epoch_size is not None: + batch_size_iter = batch_size if batch_size is not None else 1 + datapipe = datapipe.with_epoch(epoch_size // (num_gpus * num_workers * batch_size_iter)) # Pre-define iterator length + + if batch_size is not None: + # Perform multi-threaded dataloading + return wds.WebLoader(datapipe, num_workers=num_workers, batch_size=None) + else: + return datapipe + + +### Multi-dataset loading utils +def make_empty_mod_dict(modality_info): + empty_mod_dicts = {} + + for mod_name, mod_info in modality_info.items(): + empty_mod = {} + + # Tensor + if 'num_channels' in mod_info and 'input_size' in mod_info: + # Handle image-like modalities + max_tokens = mod_info['max_tokens'] + empty_mod['tensor'] = torch.zeros((mod_info['num_channels'], mod_info['input_size'], mod_info['input_size']), dtype=torch.float32) + elif mod_name == 't5_caption': + # Handle T5 embedding + max_tokens = mod_info['max_tokens'] + orig_emb_dim = mod_info['encoder_embedding']().orig_emb_dim + empty_mod['tensor'] = torch.zeros((max_tokens, orig_emb_dim), dtype=torch.float32) + elif mod_info['type'] in ['seq', 'seq_emb', 'seq_token']: + # Handle all other discrete sequence modalities + max_tokens = (mod_info['max_tokens'] + 1) * 2 + empty_mod['tensor'] = torch.zeros((max_tokens), dtype=torch.int32) + else: + max_tokens = mod_info['max_tokens'] + empty_mod['tensor'] = torch.zeros((max_tokens), dtype=torch.int32) + + # Input and target masks + empty_mod['input_mask'] = torch.ones((max_tokens), dtype=torch.bool) + empty_mod['target_mask'] = torch.ones((max_tokens), dtype=torch.bool) + + # Decoder attention mask + empty_mod['decoder_attention_mask'] = torch.zeros((max_tokens), dtype=torch.int32) + + empty_mod_dicts[mod_name] = empty_mod + + return empty_mod_dicts + + +class MixtureDataset(IterableDataset): + def __init__(self, data_iters, weights, modality_info): + self.orig_data_iters = data_iters + self.data_iters = [iter(data_iter) for data_iter in data_iters] # Create initial iterators + self.sampling_probs = np.array(weights) / sum(weights) + self.modality_info = modality_info + + def reset_iterator(self, idx): + """ Reset the iterator when exhausted. """ + self.data_iters[idx] = iter(self.orig_data_iters[idx]) + + def __iter__(self): + while True: + dataset_idx = np.random.choice(len(self.sampling_probs), p=self.sampling_probs) + try: + data = next(self.data_iters[dataset_idx]) + except StopIteration: # If the iterator is exhausted + self.reset_iterator(dataset_idx) # Reset it + data = next(self.data_iters[dataset_idx]) + + mod_dict = make_empty_mod_dict(self.modality_info) + mod_dict.update(data) + yield mod_dict + + +def build_mixture_dataloader(data_iters, weights, modality_info, batch_size, num_workers, epoch_size, num_gpus): + mixture_pipe = wds.DataPipeline( + MixtureDataset(data_iters, weights, modality_info), + wds.batched(batch_size, collation_fn=default_collate, partial=False), + ).with_epoch(epoch_size // (num_gpus * num_workers * batch_size)) # Pre-define iterator length + + mixture_loader = wds.WebLoader(mixture_pipe, num_workers=num_workers, batch_size=None) + + return mixture_loader diff --git a/fourm/demo_4M_sampler.py b/fourm/demo_4M_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..26399744127f501a48e829d18c1d6479cb9645f5 --- /dev/null +++ b/fourm/demo_4M_sampler.py @@ -0,0 +1,540 @@ +from typing import Optional, List +import os +import math +from PIL import Image +import numpy as np +import torch +import torch.nn as nn +import requests +from tokenizers import Tokenizer +import matplotlib.pyplot as plt +from torchvision.transforms.functional import center_crop + +from fourm.models.fm import FM +from fourm.vq.vqvae import VQVAE, DiVAE +from fourm.models.generate import GenerationSampler, build_chained_generation_schedules, init_empty_target_modality, init_full_input_modality, custom_text +from fourm.utils.plotting_utils import decode_dict +from fourm.data.modality_info import MODALITY_INFO +from fourm.data.modality_transforms import RGBTransform +from fourm.utils import load_safetensors +from fourm.utils.plotting_utils import decode_dict, visualize_bboxes, plot_text_in_square, text_to_pil_image + +# The flag below controls whether to allow TF32 on matmul. This flag defaults to False in PyTorch 1.12 and later. +torch.backends.cuda.matmul.allow_tf32 = True +# The flag below controls whether to allow TF32 on cuDNN. This flag defaults to True. +torch.backends.cudnn.allow_tf32 = True + + +# Default chained generation order +DEFAULT_ORDER = [ + 'tok_clip@224', 'tok_dinov2@224', 'tok_imagebind@224', 'tok_depth@224', 'tok_normal@224', + 'tok_semseg@224', 'tok_canny_edge@224', 'tok_sam_edge@224', 'tok_rgb@224', + 'caption', 'det', 'human_poses', 'sam_instance', 'color_palette', 'metadata', +] + +# Default super-resolution chained generation order +DEFAULT_ORDER_SR = [ + 'tok_clip@448', 'tok_depth@448', 'tok_normal@448', + 'tok_semseg@448', 'tok_rgb@448', +] + +# Default generation parameters for the case where the input contains RGB +DEFAULTS_RGB2X = { + 'tok_clip@224/tok_depth@224/tok_normal@224/tok_semseg@224/tok_canny_edge@224/tok_sam_edge@224': { + 'tokens_per_target': 196, 'autoregression_scheme': 'roar', 'decoding_steps': 1, + 'token_decoding_schedule': 'linear', 'temp': 0.01, 'temp_schedule': 'constant', + 'cfg_scale': 2.0, 'cfg_schedule': 'constant', + }, + 'tok_dinov2@224/tok_imagebind@224': { + 'tokens_per_target': 256, 'autoregression_scheme': 'roar', 'decoding_steps': 1, + 'token_decoding_schedule': 'linear', 'temp': 0.01, 'temp_schedule': 'constant', + 'cfg_scale': 2.0, 'cfg_schedule': 'constant', + }, + 'caption/det': { + 'tokens_per_target': 256, 'autoregression_scheme': 'autoregressive', 'decoding_steps': None, + 'token_decoding_schedule': None, 'temp': 0.3, 'temp_schedule': 'constant', + 'cfg_scale': 1.0, 'cfg_schedule': 'constant', + }, + 'human_poses': { + 'tokens_per_target': 275, 'autoregression_scheme': 'autoregressive', 'decoding_steps': None, + 'token_decoding_schedule': None, 'temp': 0.1, 'temp_schedule': 'constant', + 'cfg_scale': 1.0, 'cfg_schedule': 'constant', + }, + 'sam_instance': { + 'tokens_per_target': 256, 'autoregression_scheme': 'autoregressive', 'decoding_steps': None, + 'token_decoding_schedule': None, 'temp': 0.01, 'temp_schedule': 'constant', + 'cfg_scale': 1.0, 'cfg_schedule': 'constant', + }, + 'color_palette': { + 'tokens_per_target': 23, 'autoregression_scheme': 'autoregressive', 'decoding_steps': None, + 'token_decoding_schedule': None, 'temp': 0.1, 'temp_schedule': 'constant', + 'cfg_scale': 1.0, 'cfg_schedule': 'constant', + }, + 'metadata': { + 'tokens_per_target': 40, 'autoregression_scheme': 'autoregressive', 'decoding_steps': None, + 'token_decoding_schedule': None, 'temp': 0.1, 'temp_schedule': 'constant', + 'cfg_scale': 1.0, 'cfg_schedule': 'constant', + }, +} + +# Default generation parameters for the case where the target is RGB +DEFAULTS_X2RGB = { + 'tok_clip@224': { + 'tokens_per_target': 196, 'autoregression_scheme': 'roar', 'decoding_steps': 50, + 'token_decoding_schedule': 'linear', 'temp': 5.0, 'temp_schedule': 'onex:0.5:0.5', + 'cfg_scale': 3.0, 'cfg_schedule': 'constant', + }, + 'tok_dinov2@224/tok_imagebind@224': { + 'tokens_per_target': 256, 'autoregression_scheme': 'roar', 'decoding_steps': 8, + 'token_decoding_schedule': 'linear', 'temp': 0.01, 'temp_schedule': 'constant', + 'cfg_scale': 2.0, 'cfg_schedule': 'constant', + }, + 'tok_depth@224/tok_normal@224/tok_semseg@224/tok_canny_edge@224/tok_sam_edge@224': { + 'tokens_per_target': 196, 'autoregression_scheme': 'roar', 'decoding_steps': 8, + 'token_decoding_schedule': 'linear', 'temp': 3.0, 'temp_schedule': 'onex:0.5:0.5', + 'cfg_scale': 2.0, 'cfg_schedule': 'constant', + }, + 'tok_rgb@224': { + 'tokens_per_target': 196, 'autoregression_scheme': 'roar', 'decoding_steps': 25, + 'token_decoding_schedule': 'linear', 'temp': 3.0, 'temp_schedule': 'onex:0.5:0.5', + 'cfg_scale': 2.0, 'cfg_schedule': 'constant', + }, + 'caption/det': { + 'tokens_per_target': 256, 'autoregression_scheme': 'autoregressive', 'decoding_steps': None, + 'token_decoding_schedule': None, 'temp': 0.3, 'temp_schedule': 'constant', + 'cfg_scale': 1.0, 'cfg_schedule': 'constant', + }, + 'human_poses': { + 'tokens_per_target': 275, 'autoregression_scheme': 'autoregressive', 'decoding_steps': None, + 'token_decoding_schedule': None, 'temp': 0.1, 'temp_schedule': 'constant', + 'cfg_scale': 1.0, 'cfg_schedule': 'constant', + }, + 'sam_instance': { + 'tokens_per_target': 256, 'autoregression_scheme': 'autoregressive', 'decoding_steps': None, + 'token_decoding_schedule': None, 'temp': 0.01, 'temp_schedule': 'constant', + 'cfg_scale': 1.0, 'cfg_schedule': 'constant', + }, + 'color_palette': { + 'tokens_per_target': 23, 'autoregression_scheme': 'autoregressive', 'decoding_steps': None, + 'token_decoding_schedule': None, 'temp': 0.1, 'temp_schedule': 'constant', + 'cfg_scale': 1.0, 'cfg_schedule': 'constant', + }, + 'metadata': { + 'tokens_per_target': 40, 'autoregression_scheme': 'autoregressive', 'decoding_steps': None, + 'token_decoding_schedule': None, 'temp': 0.1, 'temp_schedule': 'constant', + 'cfg_scale': 1.0, 'cfg_schedule': 'constant', + }, +} + +# Default generation parameters for super-resolution +DEFAULTS_SR = { + 'tok_clip@448/tok_depth@448/tok_normal@448/tok_semseg@448/tok_rgb@448': { + 'tokens_per_target': 784, 'autoregression_scheme': 'maskgit', 'decoding_steps': 8, + 'token_decoding_schedule': 'cosine', 'temp': 1.0, 'temp_schedule': 'constant', + 'cfg_scale': 2.0, 'cfg_schedule': 'constant', + }, +} + +# Plotting names for each modality +MODALITY_PLOTTING_NAME_MAP = { + 'caption': 'Caption', + 'det': 'Bounding boxes', + 'human_poses': 'Human poses', + 'sam_instance': 'SAM instances (single pass)', + 'color_palette': 'Color palette', + 'metadata': 'Metadata', + 'rgb@224': 'RGB (224x224)', + 'rgb@448': 'RGB (448x448)', + 'tok_rgb@224': 'RGB (tokenized, 224x224)', + 'tok_rgb@448': 'RGB (tokenized, 448x448)', + 'tok_clip@224': 'CLIP-B/16 (224x224)', + 'tok_clip@448': 'CLIP-B/16 (448x448)', + 'tok_depth@224': 'Depth (224x224)', + 'tok_depth@448': 'Depth (448x448)', + 'tok_normal@224': 'Normals (224x224)', + 'tok_normal@448': 'Normals (448x448)', + 'tok_semseg@224': 'Semantic segmentation (224x224)', + 'tok_semseg@448': 'Semantic segmentation (448x448)', + 'tok_canny_edge@224': 'Canny edges (224x224)', + 'tok_sam_edge@224': 'SAM edges (224x224)', + 'tok_dinov2@224': 'DINOv2-B/14 (224x224)', + 'tok_imagebind@224': 'ImageBind-H/14 (224x224)', +} + +# Optional fixed plotting order (by default, plotting order is determined by generation order) +MODALITY_PLOTTING_ORDER = [ + 'rgb@224', 'rgb@448', 'tok_rgb@224', 'tok_rgb@448', + 'tok_depth@224', 'tok_depth@448', 'tok_normal@224', 'tok_normal@448', + 'tok_semseg@224', 'tok_semseg@448', 'tok_canny_edge@224', 'tok_sam_edge@224', + 'sam_instance', 'human_poses', 'det', 'caption', 'metadata', 'color_palette', + 'tok_clip@224', 'tok_clip@448', 'tok_dinov2@224', 'tok_imagebind@224', +] + + +def get_value(defaults_dict, domain, key): + """Look up a default value belonging to a given domain and key.""" + for domains, defaults in defaults_dict.items(): + if domain in domains: + return defaults[key] + +def load_model(model_id, model_class): + """Load a model from HuggingFace hub or a given .safetensors checkpoint path.""" + if model_id.endswith('.safetensors'): + ckpt, config = load_safetensors(model_id) + model = model_class(config=config) + model.load_state_dict(ckpt) + else: + model = model_class.from_pretrained(model_id) + return model + +def img_from_url(url: str): + rgb_transform = RGBTransform(imagenet_default_mean_and_std=True) + img_data = requests.get(url).content + with open('demo.png', 'wb') as handler: + handler.write(img_data) + img_pil = rgb_transform.load('./demo.png') + img_pil = rgb_transform.preprocess(img_pil) + img_pil = center_crop(img_pil, (min(img_pil.size), min(img_pil.size))).resize((224,224)) + img = rgb_transform.postprocess(img_pil).unsqueeze(0) + return img + + +class Demo4MSampler(nn.Module): + """Convenience wrapper for easy 4M loading and generation. Users can specify HuggingFace Hub + model URLs, or downloaded safetensors checkpoints paths, and the models will be automatically + loaded. The `forward` function can be used for RGB-2-all and {caption,det}-2-all generation. + This wrapper is only intended for quickly trying out 4M models. For more advanced usecases we + recommend looking at the generation notebooks in `./notebooks/`, and `./run_generation.py`. + + Args: + fm: Hub or safetensors path of 4M base model + fm_sr: Hub or safetensors path of 4M super-resolution model + tok_rgb: Hub or safetensors path of RGB tokenizer + tok_depth: Hub or safetensors path of depth tokenizer + tok_normal: Hub or safetensors path of surface normal tokenizer + tok_edge: Hub or safetensors path of canny edge tokenizer (for SAM and RGB edges) + tok_semseg: Hub or safetensors path of COCO semantic segmentation tokenizer + tok_clip: Hub or safetensors path of CLIP-B/16 tokenizer + tok_dinov2: Hub or safetensors path of DINOv2-B/14 tokenizer + tok_imagebind: Hub or safetensors path of ImageBind-H/14 tokenizer + tok_sam_instance: Hub or safetensors path of SAM instance tokenizer + tok_human_poses: Hub or safetensors path of human poses tokenizer + tok_text: Path to text tokenizer JSON file + mods: Optional list of modalities to override default behavior of generating everything + mods_sr: Optional list of super-res modalities to override default behavior of generating everything + """ + def __init__(self, + fm: str = 'EPFL-VILAB/4M-21_XL_CC12M', + fm_sr: Optional[str] = 'EPFL-VILAB/4M-7-SR_L_CC12M', + tok_rgb: Optional[str] = 'EPFL-VILAB/4M_tokenizers_rgb_16k_224-448', + tok_depth: Optional[str] = 'EPFL-VILAB/4M_tokenizers_depth_8k_224-448', + tok_normal: Optional[str] = 'EPFL-VILAB/4M_tokenizers_normal_8k_224-448', + tok_edge: Optional[str] = 'EPFL-VILAB/4M_tokenizers_edge_8k_224-512', + tok_semseg: Optional[str] = 'EPFL-VILAB/4M_tokenizers_semseg_4k_224-448', + tok_clip: Optional[str] = 'EPFL-VILAB/4M_tokenizers_CLIP-B16_8k_224-448', + tok_dinov2: Optional[str] = 'EPFL-VILAB/4M_tokenizers_DINOv2-B14_8k_224-448', + tok_imagebind: Optional[str] = 'EPFL-VILAB/4M_tokenizers_ImageBind-H14_8k_224-448', + tok_sam_instance: Optional[str] = 'EPFL-VILAB/4M_tokenizers_sam-instance_1k_64', + tok_human_poses: Optional[str] = 'EPFL-VILAB/4M_tokenizers_human-poses_1k_8', + tok_text: str = './fourm/utils/tokenizer/trained/text_tokenizer_4m_wordpiece_30k.json', + mods: Optional[List[str]] = None, + mods_sr: Optional[List[str]] = None, + verbose: bool = True): + super().__init__() + + self.verbose = verbose + if self.verbose: + print('Loading 4M models and tokenizers...', end='') + + # Load 4M model and initialize sampler + fm = load_model(fm, FM) + self.sampler_fm = GenerationSampler(fm) + self.mods = mods or list(set(fm.encoder_modalities) | set(fm.decoder_modalities)) + + # Load optional 4M super-res model and initialize sampler + if fm_sr is not None: + fm_sr = load_model(fm_sr, FM) + self.sampler_fm_sr = GenerationSampler(fm_sr) + self.mods_sr = mods_sr or list(set(fm_sr.encoder_modalities) | set(fm_sr.decoder_modalities)) + else: + self.sampler_fm_sr = None + + # Load tokenizers + self.toks = {} + if ('tok_rgb@224' in self.mods or 'tok_rgb@448' in self.mods_sr) and tok_rgb is not None: + self.toks['tok_rgb'] = load_model(tok_rgb, DiVAE) + if ('tok_depth@224' in self.mods or 'tok_depth@448' in self.mods_sr) and tok_depth is not None: + self.toks['tok_depth'] = load_model(tok_depth, DiVAE) + if ('tok_normal@224' in self.mods or 'tok_normal@448' in self.mods_sr) and tok_normal is not None: + self.toks['tok_normal'] = load_model(tok_normal, DiVAE) + if ('tok_canny_edge@224' in self.mods or 'tok_sam_edge@224' in self.mods) and tok_edge is not None: + self.toks['tok_canny_edge'] = load_model(tok_edge, DiVAE) + self.toks['tok_sam_edge'] = self.toks['tok_canny_edge'] # Shared tokenizer + if ('tok_semseg@224' in self.mods or 'tok_semseg@448' in self.mods_sr) and tok_semseg is not None: + self.toks['tok_semseg'] = load_model(tok_semseg, VQVAE) + if ('tok_clip@224' in self.mods or 'tok_clip@448' in self.mods_sr) and tok_clip is not None: + self.toks['tok_clip'] = load_model(tok_clip, VQVAE) + if 'tok_dinov2@224' in self.mods and tok_dinov2 is not None: + self.toks['tok_dinov2'] = load_model(tok_dinov2, VQVAE) + if 'tok_imagebind@224' in self.mods and tok_imagebind is not None: + self.toks['tok_imagebind'] = load_model(tok_imagebind, VQVAE) + if 'sam_instance' in self.mods and tok_sam_instance is not None: + self.toks['sam_instance'] = load_model(tok_sam_instance, VQVAE) + if 'human_poses' in self.mods and tok_human_poses is not None: + self.toks['human_poses'] = load_model(tok_human_poses, VQVAE) + self.toks = nn.ModuleDict(self.toks) + self.tok_text = Tokenizer.from_file(tok_text) + + if self.verbose: + print(' done!') + + @property + def device(self): + return next(self.parameters()).device + + def __setup_conds_and_targets(self, sample): + # Input and output modalities + cond_domains = [domain for domain in list(sample.keys()) if domain in self.mods] + target_domains = [domain for domain in DEFAULT_ORDER if (domain not in cond_domains and domain in self.mods)] + if 'rgb@224' in cond_domains: + # Do not generate tokenized RGB if pixel RGB is given as input + target_domains.remove('tok_rgb@224') + return cond_domains, target_domains + + def __setup_sr_conds_and_targets(self, sample): + cond_domains_sr = [domain for domain in list(sample.keys()) if domain in self.mods_sr] + target_domains_sr = [domain for domain in DEFAULT_ORDER_SR if (domain.replace('448', '224') in cond_domains_sr and domain in self.mods_sr)] + return cond_domains_sr, target_domains_sr + + def __setup_sample_and_schedule(self, sample, cond_domains, target_domains, cfg_grow_conditioning=True): + # 1 - Setup generation schedule + + defaults = DEFAULTS_RGB2X if ('rgb@224' in cond_domains or 'tok_rgb@224' in cond_domains) else DEFAULTS_X2RGB + + tokens_per_target = [get_value(defaults, domain, 'tokens_per_target') for domain in target_domains] + autoregression_schemes = [get_value(defaults, domain, 'autoregression_scheme') for domain in target_domains] + decoding_steps = [get_value(defaults, domain, 'decoding_steps') for domain in target_domains] + token_decoding_schedules = [get_value(defaults, domain, 'token_decoding_schedule') for domain in target_domains] + temps = [get_value(defaults, domain, 'temp') for domain in target_domains] + temp_schedules = [get_value(defaults, domain, 'temp_schedule') for domain in target_domains] + cfg_scales = [get_value(defaults, domain, 'cfg_scale') for domain in target_domains] + cfg_schedules = [get_value(defaults, domain, 'cfg_schedule') for domain in target_domains] + + schedule = build_chained_generation_schedules( + cond_domains=cond_domains, target_domains=target_domains, tokens_per_target=tokens_per_target, + autoregression_schemes=autoregression_schemes, decoding_steps=decoding_steps, + token_decoding_schedules=token_decoding_schedules, temps=temps, temp_schedules=temp_schedules, + cfg_scales=cfg_scales, cfg_schedules=cfg_schedules, cfg_grow_conditioning=cfg_grow_conditioning, + ) + + # 2 - Setup sample + + sample_dict = {} + + # Handle special cases + if 'caption' in sample: + caption = sample.pop('caption') + sample_dict = custom_text( + sample_dict, input_text=caption, eos_token='[EOS]', + key='caption', device=self.device, text_tokenizer=self.tok_text + ) + if 'det' in sample: + caption = sample.pop('det') + sample_dict = custom_text( + sample_dict, input_text=caption, eos_token='[EOS]', + key='det', device=self.device, text_tokenizer=self.tok_text + ) + # Add remaining modalities + sample_dict.update({domain: {'tensor': tensor} for domain, tensor in sample.items()}) + + # Initialize these remaining input modalities (caption and det are already initialized by custom_text) + for cond_mod in sample.keys(): + sample_dict = init_full_input_modality(sample_dict, MODALITY_INFO, cond_mod, self.device, eos_id=self.tok_text.token_to_id("[EOS]")) + + # Initialize target modalities + for target_mod, ntoks in zip(target_domains, tokens_per_target): + sample_dict = init_empty_target_modality(sample_dict, MODALITY_INFO, target_mod, 1, ntoks, self.device) + + return sample_dict, schedule + + def __setup_sr_sample_and_schedule(self, out_dict, cond_domains_sr, target_domains_sr, cfg_grow_conditioning_sr=True): + # 1 - Setup generation schedule + + tokens_per_target_sr = [get_value(DEFAULTS_SR, domain, 'tokens_per_target') for domain in target_domains_sr] + autoregression_schemes_sr = [get_value(DEFAULTS_SR, domain, 'autoregression_scheme') for domain in target_domains_sr] + decoding_steps_sr = [get_value(DEFAULTS_SR, domain, 'decoding_steps') for domain in target_domains_sr] + token_decoding_schedules_sr = [get_value(DEFAULTS_SR, domain, 'token_decoding_schedule') for domain in target_domains_sr] + temps_sr = [get_value(DEFAULTS_SR, domain, 'temp') for domain in target_domains_sr] + temp_schedules_sr = [get_value(DEFAULTS_SR, domain, 'temp_schedule') for domain in target_domains_sr] + cfg_scales_sr = [get_value(DEFAULTS_SR, domain, 'cfg_scale') for domain in target_domains_sr] + cfg_schedules_sr = [get_value(DEFAULTS_SR, domain, 'cfg_schedule') for domain in target_domains_sr] + + schedule_sr = build_chained_generation_schedules( + cond_domains=cond_domains_sr, target_domains=target_domains_sr, tokens_per_target=tokens_per_target_sr, + autoregression_schemes=autoregression_schemes_sr, decoding_steps=decoding_steps_sr, + token_decoding_schedules=token_decoding_schedules_sr, temps=temps_sr, temp_schedules=temp_schedules_sr, + cfg_scales=cfg_scales_sr, cfg_schedules=cfg_schedules_sr, cfg_grow_conditioning=cfg_grow_conditioning_sr, + ) + + # 2 - Setup sample + + sample_sr = out_dict + + # Handle case where generated caption or bounding boxes is just [EOS] + if 'caption' in sample_sr and sample_sr['caption']['tensor'].shape[1] <= 1 and 'caption' in cond_domains_sr: + sample_sr = custom_text( + sample_sr, input_text='[S_1]', eos_token='[EOS]', + key='caption', device=self.device, text_tokenizer=self.tok_text + ) + if 'det' in sample_sr and sample_sr['det']['tensor'].shape[1] <= 1 and 'det' in cond_domains_sr: + sample_sr = custom_text( + sample_sr, input_text='[S_1]', eos_token='[EOS]', + key='det', device=self.device, text_tokenizer=self.tok_text + ) + + # Initialize input modalities + for cond_mod in cond_domains_sr: + sample_sr = init_full_input_modality(sample_sr, MODALITY_INFO, cond_mod, self.device, eos_id=self.tok_text.token_to_id("[EOS]")) + + # Initialize target modalities + for target_mod, ntoks in zip(target_domains_sr, tokens_per_target_sr): + sample_sr = init_empty_target_modality(sample_sr, MODALITY_INFO, target_mod, 1, ntoks, self.device) + + return sample_sr, schedule_sr + + def forward(self, sample, seed: Optional[int] = None, top_p: float = 0.8, top_k: float = 0.0, target_modalities: Optional[List[str]] = None, perform_sr: bool = True): + seed = seed or np.random.randint(np.iinfo(np.int64).max) + + # Prepare the generation parameters and sample + cond_domains, target_domains = self.__setup_conds_and_targets(sample) + target_domains = target_modalities or target_domains + sample, generation_schedule = self.__setup_sample_and_schedule(sample, cond_domains, target_domains) + + # Generation and decoding at the base resolution 224x224 + if self.verbose: + print(f'Generating {cond_domains} -> {target_domains} ...') + out_dict = self.sampler_fm.generate( + sample, generation_schedule, text_tokenizer=self.tok_text, + verbose=self.verbose, seed=seed, top_p=top_p, top_k=top_k, + ) + dec_dict = decode_dict( + out_dict, self.toks, self.tok_text, image_size=224, + patch_size=16, decoding_steps=50 + ) + + # Optional upsampling to 448x448 + if self.sampler_fm_sr is not None and perform_sr: + cond_domains_sr, target_domains_sr = self.__setup_sr_conds_and_targets(out_dict) + sample_sr, generation_schedule_sr = self.__setup_sr_sample_and_schedule(out_dict, cond_domains_sr, target_domains_sr) + + if self.verbose: + print(f'Super-resolving {target_domains_sr} ...') + out_dict_sr = self.sampler_fm_sr.generate( + sample_sr, generation_schedule_sr, text_tokenizer=self.tok_text, + verbose=self.verbose, seed=seed+1, top_p=top_p, top_k=top_k, + ) + dec_dict = decode_dict( + out_dict_sr, self.toks, self.tok_text, image_size=448, + patch_size=16, decoding_steps=50 + ) + + # Remove padding tokens + if 'caption' in dec_dict: + dec_dict['caption'][0].replace('[PAD]', '').strip() + if 'det' in dec_dict: + dec_dict['det'][0].replace('[PAD]', '').strip() + + return dec_dict + + def plot_modalities(self, mod_dict, ncols_max=5, figscale=4.0, save_path=None, use_fixed_plotting_order=False): + nmods = len(mod_dict) + ncols = min(nmods, ncols_max) + nrows = math.ceil(nmods / ncols) + + fig, ax = plt.subplots( + nrows=nrows, ncols=ncols, + figsize=(ncols*figscale, nrows*figscale), + facecolor=(1, 1, 1) + ) + + if use_fixed_plotting_order: + mod_dict = { + k: mod_dict[k] for k in MODALITY_PLOTTING_ORDER + if k in mod_dict + } + + for i, (mod_name, mod) in enumerate(mod_dict.items()): + if nrows == 1: + ax_i = ax[i] + else: + row, col = i // ncols, i % ncols + ax_i = ax[row,col] + + if mod_name == 'det': + # Attempt to get the first available value from mod_dict according to the priority + keys_in_order = ['rgb@448', 'rgb@224', 'tok_rgb@448', 'tok_rgb@224'] + rgb_background = next((mod_dict[key] for key in keys_in_order if key in mod_dict), np.ones((224, 224, 3))) + rgb_background = (255 * rgb_background).astype(np.uint8) + ax_i.imshow(visualize_bboxes(rgb_background, mod[0],).astype(np.uint8)) + elif mod_name == 'caption': + plot_text_in_square(ax_i, mod[0], wrap_width=16, fontsize=14) + elif mod_name == 'metadata': + metadata_pred = ',\n'.join([f'{k}: {v:.2f}' if isinstance(v, float) else f'{k}: {v}' for k, v in mod.items()]) + plot_text_in_square(ax_i, metadata_pred, wrap_width=36, fontsize=13) + else: + ax_i.imshow(mod) + + ax_i.set_title(MODALITY_PLOTTING_NAME_MAP.get(mod_name, mod_name), fontsize=18) + + for i, axis in enumerate(ax.flatten()): + axis.set_xticks([]) + axis.set_yticks([]) + if i >= len(mod_dict): + axis.spines['top'].set_visible(False) + axis.spines['right'].set_visible(False) + axis.spines['bottom'].set_visible(False) + axis.spines['left'].set_visible(False) + + plt.tight_layout() + if save_path is not None: + os.makedirs(os.path.dirname(save_path), exist_ok=True) + plt.savefig(save_path, bbox_inches='tight', dpi=300) + plt.close() + else: + plt.show() + + def modalities_to_pil(self, mod_dict, use_fixed_plotting_order=False, resize=None): + if use_fixed_plotting_order: + mod_dict = { + k: mod_dict[k] for k in MODALITY_PLOTTING_ORDER + if k in mod_dict + } + + plotted_modalities = [] + + for i, (mod_name, mod) in enumerate(mod_dict.items()): + if mod_name == 'det': + # Attempt to get the first available value from mod_dict according to the priority + keys_in_order = ['rgb@448', 'rgb@224', 'tok_rgb@448', 'tok_rgb@224'] + rgb_background = next((mod_dict[key] for key in keys_in_order if key in mod_dict), np.ones((224, 224, 3))) + rgb_background = (255 * rgb_background).astype(np.uint8) + img_pil = Image.fromarray(visualize_bboxes(rgb_background, mod[0],).astype(np.uint8)) + elif mod_name == 'caption': + img_pil = text_to_pil_image(mod[0][:512], wrap_width=40, fontsize=14) + elif mod_name == 'metadata': + metadata_pred = ',\n'.join([f'{k}: {v:.2f}' if isinstance(v, float) else f'{k}: {v}' for k, v in mod.items()]) + img_pil = text_to_pil_image(metadata_pred, wrap_width=36, fontsize=13) + else: + img_pil = Image.fromarray((255*mod).astype(np.uint8)) + + if resize is not None: + if mod_name in ['tok_clip@224', 'tok_dinov2@224', 'tok_imagebind@224', 'tok_clip@448']: + resample_mode = Image.Resampling.NEAREST + else: + resample_mode = Image.Resampling.BILINEAR + img_pil = img_pil.resize((resize, resize), resample=resample_mode) + + plot_name = MODALITY_PLOTTING_NAME_MAP.get(mod_name, mod_name) + plotted_modalities.append((img_pil, plot_name)) + + return plotted_modalities \ No newline at end of file diff --git a/fourm/models/__init__.py b/fourm/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/fourm/models/decoder_embeddings.py b/fourm/models/decoder_embeddings.py new file mode 100644 index 0000000000000000000000000000000000000000..3045793d5675ac5b6564afe3f8ebdf22ea2e038f --- /dev/null +++ b/fourm/models/decoder_embeddings.py @@ -0,0 +1,268 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from functools import partial +from typing import Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +from einops import repeat + +from .fm_utils import build_1d_sincos_posemb, build_2d_sincos_posemb, pair + + +class SequenceDecoderEmbedding(nn.Module): + """Embedding module for sequence inputs, like captions or a sequence of objects. + + Args: + vocab_size: Vocabulary size + max_length: Maximum number of tokens in the sequence + dim_tokens: Dimension of output tokens. Can be set using init method. + sincos_pos_emb: Set to True (default) to use fixed 1D sin-cos positional embeddings + padding_idx: Padding index for word embedding + share_embedding: Set to True to share input and output embedding weights + """ + def __init__(self, + vocab_size: int, + max_length: int, + dim_tokens: Optional[int] = None, + sincos_pos_emb: bool = True, + max_sincos_pos_emb: int = 512, + padding_idx: int = 0, + share_embedding: bool = True, + **kwargs): + super().__init__() + self.vocab_size = vocab_size + self.max_length = max_length + self.dim_tokens = dim_tokens + self.sincos_pos_emb = sincos_pos_emb + self.padding_idx = padding_idx + self.max_sincos_pos_emb = max_sincos_pos_emb + self.share_embedding = share_embedding + + if self.dim_tokens is not None: + self.init(dim_tokens=dim_tokens) + + def init(self, dim_tokens: int = 768, init_std=0.02): + """ + Initialize parts of embedding module that are dependent on dimension of tokens. + Should be called when setting up FourM. + + Args: + dim_tokens: Dimension of tokens + init_std: Standard deviation of init + """ + self.dim_tokens = dim_tokens + + # Task embedding identifying from which task a given token comes from + # Fixed-size positional embeddings. Can be interpolated to different input sizes + + if self.sincos_pos_emb: + if self.max_length > self.max_sincos_pos_emb: + raise ValueError(f"Max length ({self.max_length}) is greater than the number of posembs ({self.max_sincos_pos_emb}") + # Get all posembs, than truncate up to max length + pos_emb = build_1d_sincos_posemb(max_len=self.max_sincos_pos_emb, embed_dim=self.dim_tokens)[:self.max_length] + self.register_buffer("pos_emb", pos_emb) + else: + self.pos_emb = nn.Parameter(torch.zeros(1, self.max_length, self.dim_tokens)) + nn.init.normal_(self.pos_emb, std=init_std) + + self.mod_emb = nn.Parameter(torch.zeros(1, 1, self.dim_tokens)) + nn.init.normal_(self.mod_emb, std=init_std) + + # Token embedding + self.token_emb = nn.Embedding(num_embeddings=self.vocab_size, embedding_dim=self.dim_tokens, padding_idx=self.padding_idx) + + # Output projection layer + self.to_logits = nn.Linear(self.dim_tokens, self.vocab_size, bias=False) + + if self.share_embedding: + # Share input and output embedding weights + self.to_logits.weight = self.token_emb.weight + + + @torch.jit.ignore + def no_weight_decay(self): + return set() + + def forward_embed(self, d: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """ + Forward pass through embedding module, transforming sequence of ids to sequence of embeddings. + Creates corresponding modality and positional embeddings and adds them to the dict. + + Args: + d (Dict[str, torch.Tensor]): Modality dict, with at least the following keys: + - 'tensor' (torch.Tensor): Token sequence for each batch. Shape (B, L) where B is the batch size and L is the sequence length. + - 'target_mask' (torch.Tensor): Mask for valid tokens in the target sequence (set to 0 for valid tokens and 1 otherwise). Shape (B, L). + + Returns: + Dict[str, torch.Tensor]: Modality dict with added keys: + - 'x' (torch.Tensor): Embedded token sequence. Shape (B, L, D) where D is the embedding dimension. + - 'emb' (torch.Tensor): Sum of positional and modality embeddings for the target sequence. Shape (B, L, D). + - 'ids' (torch.Tensor): Original token sequence from input dict. Shape (B, L). + """ + ids = d['tensor'] + B = ids.shape[0] + assert self.dim_tokens is not None, 'Need to call init(dim_tokens) function first' + + # Map to embedding + x = self.token_emb(ids) + + expanded_pos_emb = repeat(self.pos_emb, "() n d -> b n d", b=B) + + # Target pos encoding + target_mask = d['target_mask'] + target_pos_id = (~target_mask).int().cumsum(dim=1) - 1 + target_pos_id[target_mask] = 0 + # Sometimes target sequence is over max length, it will be truncated in decoder + target_pos_id[target_pos_id >= self.max_length] = 0 + target_pos_emb = torch.gather(expanded_pos_emb, dim=1, index=repeat(target_pos_id, "b n -> b n d", d=expanded_pos_emb.shape[2])) + target_pos_emb[target_mask] = 0 + + x_emb = target_pos_emb + self.mod_emb + + + d['x'] = x + d['emb'] = x_emb + d['ids'] = d['tensor'] + + return d + + def forward_logits(self, x: torch.Tensor) -> torch.Tensor: + """ + Forward pass through output projection layer, transforming sequence of embeddings to logits. + + Args: + x (torch.Tensor): Output tokens from the decoder. Shape (B, M, D) + + Returns: + torch.Tensor: Logits for each token in the sequence. Shape (B, M, V) + """ + logits = self.to_logits(x) + return logits + + + +class ImageTokenDecoderEmbedding(nn.Module): + """Embedding module for tokenized spatial inputs. + + Args: + vocab_size: Vocabulary size + patch_size: Int or tuple of the patch size over the full image size. + dim_tokens: Dimension of output tokens. Can be set using init method. + sincos_pos_emb: Set to True (default) to use fixed 2D sin-cos positional embeddings + image_size: Default image size. Used to initialize size of positional embeddings. + share_embedding: Set to True to share input and output embedding weights + """ + def __init__(self, + vocab_size: int, + patch_size: Union[int, Tuple[int,int]] = 16, + dim_tokens: Optional[int] = None, + sincos_pos_emb: bool = True, + image_size: Union[int, Tuple[int]] = 224, + share_embedding: bool = True, + **kwargs): + super().__init__() + self.vocab_size = vocab_size + self.patch_size = pair(patch_size) + self.dim_tokens = dim_tokens + self.sincos_pos_emb = sincos_pos_emb + self.image_size = pair(image_size) + self.num_patches = (self.image_size[0] // self.patch_size[0]) * (self.image_size[1] // self.patch_size[1]) + self.share_embedding = share_embedding + + if self.dim_tokens is not None: + self.init(dim_tokens=dim_tokens) + + def init(self, dim_tokens: int = 768, init_std=0.02): + """ + Initialize parts of module that are dependent on dimension of tokens. + Should be called when setting up FourM. + + Args: + dim_tokens: Dimension of tokens + init_std: Standard deviation of init + """ + self.dim_tokens = dim_tokens + + # Task embedding identifying from which task a given token comes from + # Fixed-size positional embeddings. Can be interpolated to different input sizes + h_posemb = self.image_size[0] // self.patch_size[0] + w_posemb = self.image_size[1] // self.patch_size[1] + if self.sincos_pos_emb: + pos_emb = build_2d_sincos_posemb(h=h_posemb, w=w_posemb, embed_dim=self.dim_tokens) + self.register_buffer("pos_emb", pos_emb) + else: + self.pos_emb = nn.Parameter(torch.zeros(1, (h_posemb * w_posemb), self.dim_tokens)) + nn.init.normal_(self.pos_emb, std=init_std) + + self.mod_emb = nn.Parameter(torch.zeros(1, 1, self.dim_tokens)) + nn.init.normal_(self.mod_emb, std=init_std) + + # Token embedding (not needed if only masked tokens are given as input, but can be useful to train Token Critic) + self.token_emb = nn.Embedding(num_embeddings=self.vocab_size, embedding_dim=self.dim_tokens) + + # Output projection layer + self.to_logits = nn.Linear(self.dim_tokens, self.vocab_size, bias=False) + + if self.share_embedding: + # Share input and output embedding weights + self.to_logits.weight = self.token_emb.weight + + @torch.jit.ignore + def no_weight_decay(self): + return set() + + def forward_embed(self, d: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """ + Forward pass through the embedding module, transforming tokenized spatial inputs to embeddings. + Creates corresponding modality and positional embeddings and adds them to the dict. + + Args: + d (Dict[str, torch.Tensor]): Modality dict, with at least the following key: + - 'tensor' (torch.Tensor): Modality tokens for each batch (e.g. from tokenized images). Shape (B, H, W) where B is the batch size, H and W are height and width after tokenization. + + + Returns: + Dict[str, torch.Tensor]: Modality dict with added keys: + - 'x' (torch.Tensor): Embedded token sequence, which is replaced by mask tokens in the 4M decoder. Shape (B, H*W, D) where D is the embedding dimension. + - 'emb' (torch.Tensor): Sum of positional and modality embeddings for the token sequence. Shape (B, H*W, D). + - 'ids' (torch.Tensor): Reshaped token sequence from input dict, flattened in the spatial dimensions. Shape (B, H*W). + """ + ids = d['tensor'] + B = ids.shape[0] + ids = ids.reshape(B, -1) + + # Map to embedding + x = self.token_emb(ids) + + # Create positional embedding + modality embedding + x_emb = repeat(self.pos_emb + self.mod_emb, '() n d -> b n d', b=B) + + d['x'] = x + d['emb'] = x_emb + d['ids'] = ids + return d + + def forward_logits(self, x: torch.Tensor) -> torch.Tensor: + """ + Forward pass through output projection layer, transforming sequence of embeddings to logits. + + Args: + x (torch.Tensor): Output tokens from the decoder. Shape (B, M, D) + + Returns: + torch.Tensor: Logits for each token in the sequence. Shape (B, M, V) + """ + logits = self.to_logits(x) + return logits diff --git a/fourm/models/encoder_embeddings.py b/fourm/models/encoder_embeddings.py new file mode 100644 index 0000000000000000000000000000000000000000..d1d42a18b8620841d03eee38251a73d5b5796514 --- /dev/null +++ b/fourm/models/encoder_embeddings.py @@ -0,0 +1,422 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +from einops import rearrange, repeat + +from .fm_utils import build_1d_sincos_posemb, build_2d_sincos_posemb, pair + +class SequenceEncoderEmbedding(nn.Module): + """Embedding module for encoding sequence inputs, like captions or a sequence of objects. + + Args: + vocab_size: Vocabulary size + max_length: Maximum number of tokens in the sequence + dim_tokens: Dimension of output tokens. Can be set using init method. + sincos_pos_emb: Set to True (default) to use fixed 1D sin-cos positional embeddings + max_sincos_pos_emb: Maximum allowed length for sin-cos positional embeddings + padding_idx: Padding index for word embedding + """ + + def __init__(self, + vocab_size: int, + max_length: int, + dim_tokens: Optional[int] = None, + sincos_pos_emb: bool = True, + max_sincos_pos_emb: int = 512, + padding_idx: int = 0, + ): + super().__init__() + self.vocab_size = vocab_size + self.max_length = max_length + self.dim_tokens = dim_tokens + self.sincos_pos_emb = sincos_pos_emb + self.padding_idx = padding_idx + self.max_sincos_pos_emb = max_sincos_pos_emb + + if self.dim_tokens is not None: + self.init(dim_tokens=dim_tokens) + + def init(self, dim_tokens: int = 768, init_std=0.02): + """ + Initialize parts of embedding module that are dependent on dimension of tokens. + Should be called when setting up FourM. + + Args: + dim_tokens: Dimension of tokens + init_std: Standard deviation of init + """ + self.dim_tokens = dim_tokens + + # Task embedding identifying from which task a given token comes from + # Fixed-size positional embeddings. Can be interpolated to different input sizes + if self.sincos_pos_emb: + if self.max_length > self.max_sincos_pos_emb: + raise ValueError(f"Max length ({self.max_length}) is greater than the number of posembs ({self.max_sincos_pos_emb}") + pos_emb = build_1d_sincos_posemb(max_len=self.max_sincos_pos_emb, embed_dim=self.dim_tokens)[:self.max_length] + self.register_buffer("pos_emb", pos_emb) # self.pos_emb is now a buffer for FSDP + else: + self.pos_emb = nn.Parameter(torch.zeros(1, self.max_length, self.dim_tokens)) + nn.init.normal_(self.pos_emb, std=init_std) + + self.mod_emb = nn.Parameter(torch.zeros(1, 1, self.dim_tokens)) + nn.init.normal_(self.mod_emb, std=init_std) + + # Token embedding + self.token_emb = nn.Embedding(num_embeddings=self.vocab_size, embedding_dim=self.dim_tokens, + padding_idx=self.padding_idx) + + + @torch.jit.ignore + def no_weight_decay(self): + return set() + + def forward(self, d : Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """ + Forward pass through embedding module, transforming sequence of ids to sequence of embeddings. + Creates corresponding modality and positional embeddings and adds them to the dict. + + Args: + d (Dict[str, torch.Tensor]): Modality dict with at least the following keys: + - 'tensor' (torch.Tensor): Input token sequence for each batch. Shape (B, L) where B is the batch size and L is the sequence length. + - 'input_mask' (torch.Tensor): Mask for valid tokens in the input sequence (set to 0 for valid tokens and 1 otherwise). Shape (B, L). + + Returns: + Dict[str, torch.Tensor]: Modality dict with added keys: + - 'x' (torch.Tensor): Embedded token sequence. Shape (B, L, D) where D is the embedding dimension. + - 'emb' (torch.Tensor): Sum of positional and modality embeddings for the input sequence. Shape (B, L, D). + """ + ids = d['tensor'] + B = ids.shape[0] + assert self.dim_tokens is not None, 'Need to call init(dim_tokens) function first' + + # Map to embedding + x = self.token_emb(ids) + + expanded_pos_emb = repeat(self.pos_emb, "() n d -> b n d", b=B) + # Input pos encoding + input_mask = d['input_mask'] + input_pos_id = (~input_mask).int().cumsum(dim=1) - 1 + input_pos_id[input_mask] = 0 + input_pos_emb = torch.gather(expanded_pos_emb, dim=1, index=repeat(input_pos_id, "b n -> b n d", d=expanded_pos_emb.shape[2])) + input_pos_emb[input_mask] = 0 + + x_emb = input_pos_emb + self.mod_emb + + d['x'] = x + d['emb'] = x_emb + return d + +class ImageTokenEncoderEmbedding(nn.Module): + """Embedding module for tokenized spatial inputs. + + Args: + vocab_size: Vocabulary size + patch_size: Int or tuple of the patch size over the full image size. + dim_tokens: Dimension of output tokens. Can be set using init method. + sincos_pos_emb: Set to True (default) to use fixed 2D sin-cos positional embeddings + image_size: Default image size. Used to initialize size of positional embeddings. + """ + def __init__(self, + vocab_size: int, + patch_size: Union[int, Tuple[int,int]] = 16, + dim_tokens: Optional[int] = None, + sincos_pos_emb: bool = True, + image_size: Union[int, Tuple[int]] = 224, + **kwargs): + + super().__init__() + self.vocab_size = vocab_size + self.patch_size = pair(patch_size) + self.dim_tokens = dim_tokens + self.sincos_pos_emb = sincos_pos_emb + self.image_size = pair(image_size) + self.num_patches = (self.image_size[0] // patch_size) * (self.image_size[1] // patch_size) + + if self.dim_tokens is not None: + self.init(dim_tokens=dim_tokens) + + def init(self, dim_tokens: int = 768, init_std=0.02): + """ + Initialize parts of module that are dependent on dimension of tokens. + Should be called when setting up FourM. + + Args: + dim_tokens: Dimension of tokens + init_std: Standard deviation of init + """ + self.dim_tokens = dim_tokens + + # Task embedding identifying from which task a given token comes from + # Fixed-size positional embeddings. Can be interpolated to different input sizes + h_posemb = self.image_size[0] // self.patch_size[0] + w_posemb = self.image_size[1] // self.patch_size[1] + if self.sincos_pos_emb: + pos_emb = build_2d_sincos_posemb(h=h_posemb, w=w_posemb, embed_dim=self.dim_tokens) + self.register_buffer("pos_emb", pos_emb) # self.pos_emb is now a buffer for FSDP + else: + self.pos_emb = nn.Parameter(torch.zeros(1, (h_posemb * w_posemb), self.dim_tokens)) + nn.init.normal_(self.pos_emb, std=init_std) + + self.mod_emb = nn.Parameter(torch.zeros(1, 1, self.dim_tokens)) + nn.init.normal_(self.mod_emb, std=init_std) + + # Token embedding + self.token_emb = nn.Embedding(num_embeddings=self.vocab_size, embedding_dim=self.dim_tokens) + + @torch.jit.ignore + def no_weight_decay(self): + return set() + + def forward(self, d: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """ + Forward pass through embedding module, transforming image tokens to a sequence of embeddings. + Creates corresponding modality and positional embeddings and adds them to the dict. + + Args: + d (Dict[str, torch.Tensor]): Modality dict with at least the following key: + - 'tensor' (torch.Tensor): Input image tokens for each batch. Shape (B, H, W) where B is the batch size, and H, W are height and width of the tokenized image. - 'input_mask' (torch.Tensor): Mask for valid tokens in the input sequence (set to 0 for valid tokens and 1 otherwise). Shape (B, L). + + Returns: + Dict[str, torch.Tensor]: Modality dictionary with added keys: + - 'x' (torch.Tensor): Embedded token sequence. Shape (B, H*W, D). + - 'emb' (torch.Tensor): Sum of positional and modality embeddings for the input sequence. Shape (B, H*W, D). + """ + ids = d['tensor'] + B = ids.shape[0] + ids = ids.reshape(B, -1) + + # Map to embedding + x = self.token_emb(ids) + + # Create positional embedding + modality embedding + x_emb = repeat(self.pos_emb + self.mod_emb, '() n d -> b n d', b=B) + + d['x'] = x + d['emb'] = x_emb + + return d + + +class ImageEncoderEmbedding(nn.Module): + """Embedding module for spatial inputs, like images or feature maps. + Creates tokens from patches over the image. + + This adapter / embedding differs from the one of MultiMAE by taking as input a dict and + separating positional embeddings and modality embeddings from the input projection + Input projection is 'x', posemb + modemb is 'emb' + + Args: + num_channels: Number of input channels of the image/feature map + patch_size: Int or tuple of the patch size over the full image size. + dim_tokens: Dimension of output tokens. Can be set using init method. + sincos_pos_emb: Set to True (default) to use fixed 2D sin-cos positional embeddings + image_size: Default image size. Used to initialize size of positional embeddings. + """ + def __init__(self, + num_channels: int, + patch_size: Union[int, Tuple[int,int]], + dim_tokens: Optional[int] = None, + sincos_pos_emb: bool = True, + image_size: Union[int, Tuple[int]] = 224): + + super().__init__() + self.num_channels = num_channels + self.patch_size = pair(patch_size) + self.dim_tokens = dim_tokens + self.sincos_pos_emb = sincos_pos_emb + self.image_size = pair(image_size) + self.num_patches = (self.image_size[0] // patch_size) * (self.image_size[1] // patch_size) + + if self.dim_tokens is not None: + self.init(dim_tokens=dim_tokens) + + def init(self, dim_tokens: int = 768, init_std=0.02): + """ + Initialize parts of encoder that are dependent on dimension of tokens. + Should be called when setting up FourM. + + Args: + dim_tokens: Dimension of tokens + init_std: Standard deviation of init + """ + self.dim_tokens = dim_tokens + + # Task embedding identifying from which task a given token comes from + # Fixed-size positional embeddings. Can be interpolated to different input sizes + h_posemb = self.image_size[0] // self.patch_size[0] + w_posemb = self.image_size[1] // self.patch_size[1] + if self.sincos_pos_emb: + pos_emb = build_2d_sincos_posemb(h=h_posemb, w=w_posemb, embed_dim=self.dim_tokens) + self.register_buffer("pos_emb", pos_emb) # self.pos_emb is now a buffer for FSDP + else: + self.pos_emb = nn.Parameter(torch.zeros(1, (h_posemb * w_posemb), self.dim_tokens)) + nn.init.normal_(self.pos_emb, std=init_std) + + self.mod_emb = nn.Parameter(torch.zeros(1, 1, self.dim_tokens)) + nn.init.normal_(self.mod_emb, std=init_std) + + # Image -> tokens projection + # No bias term here, so modality embedding fully comes from self.mod_emb + self.proj = nn.Linear(self.num_channels * self.patch_size[0] * self.patch_size[1], self.dim_tokens, bias=False) + + @torch.jit.ignore + def no_weight_decay(self): + return set() + + def forward(self, d: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """ + Forward pass through embedding module, transforming image to sequence of tokens. + Creates corresponding modality and positional embeddings and adds them to the dict. + + Args: + d (Dict[str, torch.Tensor]): Modality dict with at least the following key: + - 'tensor' (torch.Tensor): Input image for each batch. Shape (B, C, H, W) where B is the batch size, C is the number of channels, and H, W are height and width of the image. + + + Returns: + Dict[str, torch.Tensor]: Modality dict with added keys: + - 'x' (torch.Tensor): Embedded token sequence. Shape (B, (H / PH) * (W / PW), D), where PH and PW are the patch sizes + - 'emb' (torch.Tensor): Sum of positional and modality embeddings for the input sequence. Shape (B, (H / PH) * (W / PW), D) + """ + x = d['tensor'] + B, C, H, W = x.shape + assert self.dim_tokens is not None, 'Need to call init(dim_tokens) function first' + assert (H % self.patch_size[0] == 0) and (W % self.patch_size[1] == 0), f'Image sizes {H}x{W} must be divisible by patch sizes {self.patch_size[0]}x{self.patch_size[1]}' + + # Create patches [B, C, H, W] -> [B, (H*W), C] + x_patch = self.proj(rearrange(x, 'b d (nh ph) (nw pw) -> b (nh nw) (ph pw d)', ph=self.patch_size[0], pw=self.patch_size[1])) + + # Create positional embedding + modality embedding + x_emb = repeat(self.pos_emb + self.mod_emb, '() n d -> b n d', b=B) + + d['x'] = x_patch + d['emb'] = x_emb + + return d + + +class SequenceEmbEncoderEmbedding(nn.Module): + """Adapter for sequence emb inputs, like T5-XXL, CLIP text embeddings. + + Args: + max_length: Maximum number of tokens in the sequence + dim_tokens: Dimension of output tokens. Can be set using init method. + sincos_pos_emb: Set to True (default) to use fixed 1D sin-cos positional embeddings + padding_idx: Padding index for word embedding + orig_emb_dim: Dimension of original embeddings + bottleneck_dim: Dimension of bottleneck layer + use_bottleneck: Set to True to use bottleneck layer + """ + def __init__(self, + max_length: int, + dim_tokens: Optional[int] = None, + sincos_pos_emb: bool = True, + max_sincos_pos_emb: int = 512, + padding_idx: int = 0, + orig_emb_dim: int = 4096, + bottleneck_dim: int = 64, + use_bottleneck: bool = False, + ): + super().__init__() + self.max_length = max_length + self.dim_tokens = dim_tokens + self.sincos_pos_emb = sincos_pos_emb + self.padding_idx = padding_idx + self.max_sincos_pos_emb = max_sincos_pos_emb + self.orig_emb_dim = orig_emb_dim + self.use_bottleneck = use_bottleneck + if self.use_bottleneck: + self.bottleneck_dim = bottleneck_dim + + if self.dim_tokens is not None: + self.init(dim_tokens=dim_tokens) + + def init(self, dim_tokens: int = 768, init_std=0.02): + """ + Initialize parts of embedding module that are dependent on dimension of tokens. + Should be called when setting up FourM. + + Args: + dim_tokens: Dimension of tokens + init_std: Standard deviation of init + """ + self.dim_tokens = dim_tokens + + # Task embedding identifying from which task a given token comes from + # Fixed-size positional embeddings. Can be interpolated to different input sizes + if self.sincos_pos_emb: + if self.max_length > self.max_sincos_pos_emb: + raise ValueError(f"Max length ({self.max_length}) is greater than the number of posembs ({self.max_sincos_pos_emb}") + pos_emb = build_1d_sincos_posemb(max_len=self.max_sincos_pos_emb, embed_dim=self.dim_tokens)[:self.max_length] + self.register_buffer("pos_emb", pos_emb) # self.pos_emb is now a buffer for FSDP + else: + self.pos_emb = nn.Parameter(torch.zeros(1, self.max_length, self.dim_tokens)) + nn.init.normal_(self.pos_emb, std=init_std) + + self.mod_emb = nn.Parameter(torch.zeros(1, 1, self.dim_tokens)) + nn.init.normal_(self.mod_emb, std=init_std) + + # Token embedding projection + if self.use_bottleneck: + self.emb_proj = nn.Sequential( + nn.Linear(self.orig_emb_dim, self.bottleneck_dim), + nn.Linear(self.bottleneck_dim, self.dim_tokens), + ) + else: + self.emb_proj = nn.Linear(self.orig_emb_dim, self.dim_tokens) + + + @torch.jit.ignore + def no_weight_decay(self): + return set() + + def forward(self, d): + """ + Forward pass through embedding module, projecting original embeddings to the Transformer dimension. + Creates corresponding modality and positional embeddings and adds them to the dict. + + Args: + d (Dict[str, torch.Tensor]): Modality dict with at least the following keys: + - 'tensor' (torch.Tensor): Input token sequence for each batch. Shape (B, L, E) where B is the batch size and L is the sequence length, and E is the dimension of the original embeddings. + - 'input_mask' (torch.Tensor): Mask for valid tokens in the input sequence (set to 0 for valid tokens and 1 otherwise). Shape (B, L). + + Returns: + Dict[str, torch.Tensor]: Modality dict with added keys: + - 'x' (torch.Tensor): Embedded token sequence. Shape (B, L, D) where D is the Transformer embedding dimension. + - 'emb' (torch.Tensor): Sum of positional and modality embeddings for the input sequence. Shape (B, L, D). + """ + orig_emb = d['tensor'] + B = orig_emb.shape[0] + assert self.dim_tokens is not None, 'Need to call init(dim_tokens) function first' + + # Map to embedding + x = self.emb_proj(orig_emb) + + expanded_pos_emb = repeat(self.pos_emb, "() n d -> b n d", b=B) + # Input pos encoding + input_mask = d['input_mask'] + input_pos_id = (~input_mask).int().cumsum(dim=1) - 1 + input_pos_id[input_mask] = 0 + input_pos_emb = torch.gather(expanded_pos_emb, dim=1, index=repeat(input_pos_id, "b n -> b n d", d=expanded_pos_emb.shape[2])) + input_pos_emb[input_mask] = 0 + + x_emb = input_pos_emb + self.mod_emb + + d['x'] = x + d['emb'] = x_emb + return d + \ No newline at end of file diff --git a/fourm/models/fm.py b/fourm/models/fm.py new file mode 100644 index 0000000000000000000000000000000000000000..b69c13e256a423a77fd1caf110419e4cbd6cb97e --- /dev/null +++ b/fourm/models/fm.py @@ -0,0 +1,1130 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +import random +import copy +from functools import partial +from typing import Any, Dict, Optional, Tuple, Union + +import torch +from einops import rearrange, repeat +from torch import nn +import torch.nn.functional as F + +from fourm.utils.timm.registry import register_model +from huggingface_hub import PyTorchModelHubMixin + +from .fm_utils import Block, DecoderBlock, LayerNorm +from fourm.data.modality_info import MODALITY_INFO + + +# Model definitions +__all__ = [ + # GELU models + 'fm_tiny_6e_6d_gelu', + 'fm_small_8e_8d_gelu', + 'fm_base_12e_12d_gelu', + 'fm_large_24e_24d_gelu', + 'fm_xlarge_24e_24d_gelu', + # SwiGLU models + 'fm_tiny_6e_6d_swiglu_nobias', + 'fm_small_8e_8d_swiglu_nobias', + 'fm_base_12e_12d_swiglu_nobias', + 'fm_large_24e_24d_swiglu_nobias', + 'fm_xlarge_24e_24d_swiglu_nobias', + # SwiGLU + QKNorm models + 'fm_base_12e_12d_swiglu_qknorm_nobias', + 'fm_large_24e_24d_swiglu_qknorm_nobias', + 'fm_xlarge_24e_24d_swiglu_qknorm_nobias', +] + + + +class FourM(nn.Module): + """4M model. + + Args: + encoder_embeddings: Dict of encoder embedding modules. + decoder_embeddings: Dict of decoder embedding modules. + modality_info: Dict containing modality information. + dim: Embedding dimension. + encoder_depth: Number of encoder blocks. + decoder_depth: Number of decoder blocks. + num_heads: Number of attention heads. + mlp_ratio: Ratio of mlp hidden dim to embedding dim. + qkv_bias: If True, add a learnable bias to query, key, value projections. + proj_bias: If True, add a learnable bias to the last projection of the attention block. + mlp_bias: If True, add a learnable bias to linear layers in the MLP / feed-forward. + drop_path_rate_encoder: Stochastic depth rate for encoder. + drop_path_rate_decoder: Stochastic depth rate for decoder. + shared_drop_path: If True, shares drop path between encoder and decoder. + act_layer: Activation layer to be used. + norm_layer: Normalization layer to be used. + gated_mlp: If True, make the feedforward gated (e.g., SwiGLU). + qk_norm: If True, applies normalization to queries and keys (QKNorm). + decoder_causal_mask: If True, decoder will use a causal mask for all tokens. + decoder_sep_mask: If True, decoder attention is restricted to within each modality only. + num_register_tokens: Number of register tokens. + use_act_checkpoint: If True, use activation checkpoint for each block. + """ + def __init__(self, + encoder_embeddings: Dict[str, nn.Module], + decoder_embeddings: Dict[str, nn.Module], + modality_info: Dict[str, Any], + dim: int = 768, + encoder_depth: int = 12, + decoder_depth: int = 12, + num_heads: int = 12, + mlp_ratio: float = 4.0, + qkv_bias: bool = True, + proj_bias: bool = True, + mlp_bias: bool = True, + drop_path_rate_encoder: float = 0.0, + drop_path_rate_decoder: float = 0.0, + shared_drop_path: bool = False, + act_layer: nn.Module = nn.GELU, + norm_layer: Union[partial, nn.Module] = partial(LayerNorm, eps=1e-6), + gated_mlp: bool = False, # Make the feedforward gated for e.g. SwiGLU + qk_norm: bool = False, + decoder_causal_mask: bool = False, + decoder_sep_mask: bool = True, + num_register_tokens: int = 0, + use_act_checkpoint: bool = False, + share_modality_embeddings: bool = True, + ): + super().__init__() + + self.modality_info = modality_info + self.dim = dim + self.decoder_causal_mask = decoder_causal_mask + self.decoder_sep_mask = decoder_sep_mask + self.init_std = 0.02 + self.use_act_checkpoint = use_act_checkpoint + self.num_register_tokens = num_register_tokens + + + # Encoder embeddings & init + self.encoder_modalities = set(encoder_embeddings.keys()) + for emb in encoder_embeddings.values(): + emb.init(dim_tokens=dim, init_std=self.init_std) + self.encoder_embeddings = nn.ModuleDict(encoder_embeddings) + + # Decoder embeddings & init + self.decoder_modalities = set(decoder_embeddings.keys()) + for emb in decoder_embeddings.values(): + emb.init(dim_tokens=dim, init_std=self.init_std) + self.decoder_embeddings = nn.ModuleDict(decoder_embeddings) + + # Share modality embeddings across the encoder and decoder embedding modules + if share_modality_embeddings: + self.share_modality_embeddings() + + ## Transformer encoder + if shared_drop_path: + dpr_encoder = [x.item() for x in torch.linspace(0, drop_path_rate_encoder, encoder_depth + decoder_depth)][:encoder_depth] + else: + dpr_encoder = [x.item() for x in torch.linspace(0, drop_path_rate_encoder, encoder_depth)] # stochastic depth decay rule + + self.encoder = nn.ModuleList([ + Block(dim=dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, proj_bias=proj_bias, mlp_bias=mlp_bias, + drop_path=dpr_encoder[i], act_layer=act_layer, norm_layer=norm_layer, gated_mlp=gated_mlp, qk_norm=qk_norm) + for i in range(encoder_depth) + ]) + self.encoder_norm = norm_layer(dim) + + + ## Transformer decoder + if shared_drop_path: + dpr_decoder = [x.item() for x in torch.linspace(0, drop_path_rate_decoder, encoder_depth + decoder_depth)][encoder_depth:] + else: + dpr_decoder = [x.item() for x in torch.linspace(0, drop_path_rate_decoder, decoder_depth)] # stochastic depth decay rule + + # Projection of encoder tokens before adding the embeddings again + self.decoder_proj_context = nn.Linear(dim, dim) + + self.decoder = nn.ModuleList([ + DecoderBlock(dim=dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, proj_bias=proj_bias, mlp_bias=mlp_bias, + drop_path=dpr_decoder[i], act_layer=act_layer, norm_layer=norm_layer, gated_mlp=gated_mlp, qk_norm=qk_norm) + for i in range(decoder_depth) + ]) + self.decoder_norm = norm_layer(dim) + + self.mask_token = nn.Parameter(torch.zeros(1, 1, dim)) + nn.init.normal_(self.mask_token, std=self.init_std) + + # Additional register tokens that can be used by the encoder during fine-tuning + if self.num_register_tokens > 0: + self.register_tokens = nn.Parameter(torch.zeros(1, self.num_register_tokens, dim)) + nn.init.normal_(self.register_tokens, std=self.init_std) + else: + self.register_tokens = None + + # Weight init + self.init_weights() + + def share_modality_embeddings(self): + """Share modality embeddings across the encoder and decoder embedding modules.""" + shared_modalities = self.encoder_modalities & self.decoder_modalities + for mod in shared_modalities: + self.decoder_embeddings[mod].mod_emb = self.encoder_embeddings[mod].mod_emb + + def init_weights(self): + """Weight initialization following MAE's initialization scheme""" + + for name, m in self.named_modules(): + # Skipping tokenizers to avoid reinitializing them + if "tokenizer" in name: + continue + # Linear + elif isinstance(m, nn.Linear): + if 'qkv' in name: + # treat the weights of Q, K, V separately + val = math.sqrt(6. / float(m.weight.shape[0] // 3 + m.weight.shape[1])) + nn.init.uniform_(m.weight, -val, val) + elif 'kv' in name: + # treat the weights of K, V separately + val = math.sqrt(6. / float(m.weight.shape[0] // 2 + m.weight.shape[1])) + nn.init.uniform_(m.weight, -val, val) + else: + nn.init.xavier_uniform_(m.weight) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + # LayerNorm + elif isinstance(m, nn.LayerNorm) or isinstance(m, LayerNorm): + nn.init.constant_(m.weight, 1.0) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + # Embedding + elif isinstance(m, nn.Embedding): + nn.init.normal_(m.weight, std=self.init_std) + # Conv2d + elif isinstance(m, nn.Conv2d): + if '.proj' in name: + # From MAE, initialize projection like nn.Linear (instead of nn.Conv2d) + w = m.weight.data + nn.init.xavier_uniform_(w.view([w.shape[0], -1])) + + def get_num_layers_encoder(self): + return len(self.encoder) + + def get_num_layers_decoder(self): + return len(self.decoder) + + def get_num_layers(self): + return self.get_num_layers_encoder() + self.get_num_layers_decoder() + + @torch.jit.ignore + def no_weight_decay(self): + no_wd_set = set() + + for mod, emb_module in self.encoder_embeddings.items(): + if hasattr(emb_module, 'no_weight_decay'): + to_skip = emb_module.no_weight_decay() + to_skip = set([f'encoder_embeddings.{mod}.{name}' for name in to_skip]) + no_wd_set = no_wd_set | to_skip + + for mod, emb_module in self.decoder_embeddings.items(): + if hasattr(emb_module, 'no_weight_decay'): + to_skip = emb_module.no_weight_decay() + to_skip = set([f'decoder_embeddings.{mod}.{name}' for name in to_skip]) + no_wd_set = no_wd_set | to_skip + + return no_wd_set + + def cat_encoder_tensors(self, mod_dict: Dict[str, torch.Tensor]) -> Tuple[torch.Tensor]: + """Concatenate encoder tensors from different modalities. + + Args: + mod_dict (dict): A dictionary containing information for each modality. + Expected keys for each modality are 'x' (input tokens), + 'emb' (embeddings), 'input_mask', etc. + + Returns: + tuple: + - encoder_tokens_all (torch.Tensor): Concatenated encoder tokens from all modalities. Shape (B, O, D) where O is the total number of all encoder tokens. + - emb_all (torch.Tensor): Concatenated encoder embeddings from all modalities. Shape (B, O, D) + - encoder_mask_all (torch.Tensor): Concatenated boolean masks indicating which tokens are part of the encoder input (set to 0 for valid tokens, 1 otherwise). Shape (B, O) + - mod_mask_all (torch.Tensor): Concatenated integer mask marking the modality type for each encoder token. Shape (B, O) + """ + + encoder_tokens_all = [] + emb_all = [] + encoder_mask_all = [] + mod_mask_all = [] + + for mod, d in mod_dict.items(): + encoder_tokens_all.append(d['x']) + emb_all.append(d['emb']) + encoder_mask_all.append(d['input_mask']) + mod_mask_all.append(torch.full_like(d['input_mask'], self.modality_info[mod]['id'], dtype=torch.int16)) + + encoder_tokens_all = torch.cat(encoder_tokens_all, dim=1) + emb_all = torch.cat(emb_all, dim=1) + encoder_mask_all = torch.cat(encoder_mask_all, dim=1) + mod_mask_all = torch.cat(mod_mask_all, dim=1) + + return encoder_tokens_all, emb_all, encoder_mask_all, mod_mask_all + + def cat_decoder_tensors(self, mod_dict: Dict[str, Dict[str, torch.Tensor]]) -> Tuple[torch.Tensor]: + """Concatenate decoder tensors from different modalities. + + Args: + mod_dict (dict): A dictionary containing information for each modality. + Expected keys for each modality include 'x' (input tokens), + 'ids' (target IDs), 'emb' (embeddings), 'target_mask', 'decoder_attention_mask', etc. + + + Returns: + tuple: + - decoder_tokens_all (torch.Tensor): Concatenated decoder tokens from all modalities. Shape (B, P, D) where P is the total number of all decoder tokens. + - emb_all (torch.Tensor): Concatenated decoder embeddings from all modalities. Shape (B, P, D) + - decoder_mask_all (torch.Tensor): Concatenated boolean masks indicating which tokens are part of the decoder input / target (set to 0 for valid tokens, 1 otherwise). Shape (B, P) + - target_ids_all (torch.Tensor): Concatenated target IDs from all modalities. Shape (B, P) + - attention_mask_all (torch.Tensor): Concatenated attention masks in compressed format, needs to be passed to adapt_decoder_attention_mask() to obtain the final attention mask. Shape (B, P) + - mod_mask_all (torch.Tensor): Concatenated integer mask marking the modality type for each decoder token. Shape (B, P) + """ + + decoder_tokens_all = [] + target_ids_all = [] + emb_all = [] + decoder_mask_all = [] + attention_mask_all = [] + mod_mask_all = [] + + # Shuffle order in which modalities are provided (useful for modality causal mask) + mod_dict = {mod: d for mod, d in random.sample(mod_dict.items(), len(mod_dict))} + + for mod, d in mod_dict.items(): + if self.modality_info[mod]['type'] in ['seq', 'seq_emb', 'seq_token']: + # Important: This makes the assumption that the target sequence appears sequentially + # before sorting / gathering + decoder_tokens_all.append(d['x'][:, :-1]) + target_ids_all.append(d['ids'][:, 1:]) # Shifted left + emb_all.append(d['emb'][:, :-1]) + # Logical or with left shifting removes the last unmasked position + decoder_mask_all.append(torch.logical_or(d['target_mask'][:, 1:], d['target_mask'][:, :-1])) + # Add attention mask ids + attention_mask_all.append(d['decoder_attention_mask'][:, :-1]) + mod_mask_all.append(torch.full_like(d['ids'][:, :-1], self.modality_info[mod]['id'], dtype=torch.int16)) + else: + # Important: For 2d / image modalities, the decoder input tokens are replaced by the mask token + decoder_tokens_all.append(torch.zeros_like(d['x']) + self.mask_token) # Replace x by mask token + target_ids_all.append(d['ids']) + emb_all.append(d['emb']) + decoder_mask_all.append(d['target_mask']) + attention_mask_all.append(d['decoder_attention_mask']) + mod_mask_all.append(torch.full_like(d['ids'], self.modality_info[mod]['id'], dtype=torch.int16)) + + decoder_tokens_all = torch.cat(decoder_tokens_all, dim=1) + emb_all = torch.cat(emb_all, dim=1) + decoder_mask_all = torch.cat(decoder_mask_all, dim=1) + target_ids_all = torch.cat(target_ids_all, dim=1) + attention_mask_all = torch.cat(attention_mask_all, dim=1) + mod_mask_all = torch.cat(mod_mask_all, dim=1) + + return decoder_tokens_all, emb_all, decoder_mask_all, target_ids_all, attention_mask_all, mod_mask_all + + def forward_mask_encoder(self, mod_dict: Dict[str, Dict[str, torch.Tensor]], num_encoder_tokens: int) -> Tuple[torch.Tensor]: + """Concatenates and mask encoder tensors based on provided modality information. + + This function consolidates encoder tokens from multiple modalities, then selects a specified number of them based on modality information (i.e. masking). + + Args: + mod_dict (dict): Dictionary containing tensors for different modalities. + It is expected to have keys for each modality and values + containing the modalities' associated tensors. + num_encoder_tokens (int): Number of encoder tokens to retain after masking. + + Returns: + tuple: + - encoder_tokens (torch.Tensor): Selected encoder tokens from all modalities. Shape (B, N, D) where N is the number of selected encoder tokens. + - encoder_emb (torch.Tensor): Corresponding embeddings for encoder tokens. Shape (B, N, D) + - encoder_mask (torch.Tensor): A boolean mask indicating which encoder tokens are valid (set to 0 for valid tokens, 1 otherwise). Shape (B, 1, N) + - mod_mask (torch.Tensor): An integer mask marking the modality type for each encoder token (with -1 indicating unassigned pad tokens). Shape (B, N) + + Notes: + - If `num_register_tokens` is set and greater than 0, register tokens are added at the beginning of the sequence. + """ + B = list(mod_dict.values())[0]['tensor'].shape[0] + + encoder_tokens_all, emb_all, encoder_mask_all, mod_mask_all = self.cat_encoder_tensors(mod_dict) + + # Add arange multiplied by small constant to mask so they get sorted in a deterministic way + mask_arange = torch.arange(encoder_mask_all.shape[1], device=encoder_mask_all.device).unsqueeze(0) * 1e-6 + ids_shuffle = torch.argsort(encoder_mask_all + mask_arange, dim=1) + # ids_restore = torch.argsort(ids_shuffle, dim=1) + ids_keep = ids_shuffle[:, :num_encoder_tokens] + + encoder_tokens = torch.gather(encoder_tokens_all, dim=1, + index=repeat(ids_keep, "b n -> b n d", d=encoder_tokens_all.shape[2])) + encoder_emb = torch.gather(emb_all, dim=1, index=repeat(ids_keep, "b n -> b n d", d=emb_all.shape[2])) + encoder_mask = torch.gather(encoder_mask_all, dim=1, index=ids_keep) + mod_mask = torch.gather(mod_mask_all, dim=1, index=ids_keep) + + if self.num_register_tokens > 0: + register_tokens = repeat(self.register_tokens, '() n d -> b n d', b=B) + # We add register tokens at the beginning of the sequence + encoder_tokens = torch.cat([register_tokens, encoder_tokens], dim=1) + encoder_emb = torch.cat([torch.zeros_like(register_tokens), encoder_emb], dim=1) + encoder_mask = torch.cat([torch.zeros((B, register_tokens.shape[1]), dtype=torch.bool, device=encoder_mask.device), encoder_mask], dim=1) + mod_mask = torch.cat([torch.full((B, register_tokens.shape[1]), -1, dtype=torch.int16, device=mod_mask.device), mod_mask], dim=1) + + encoder_tokens[encoder_mask] = 0. + encoder_emb[encoder_mask] = 0. + mod_mask[encoder_mask] = -1 + # Mask could be of shape 'b n1 n2' but not needed for masked_fill + # This means this mask can then be re-used for decoder cross-attention + encoder_mask = rearrange(encoder_mask, 'b n2 -> b 1 n2') + + return encoder_tokens, encoder_emb, encoder_mask, mod_mask + + def forward_mask_decoder(self, mod_dict: Dict[str, Dict[str, torch.Tensor]], num_decoder_tokens: int) -> Tuple[torch.Tensor]: + """Concatenates and mask decoder tensors based on provided modality information. + + This function consolidates decoder tokens from multiple modalities, selects a specified number of them based on modality information, and applies appropriate masking. + + Args: + mod_dict (dict): Dictionary containing tensors for different modalities. + It is expected to have keys for each modality and values + containing the modalities' associated tensors. + num_decoder_tokens (int): Number of decoder tokens to retain after masking. + + Returns: + tuple: + - decoder_tokens (torch.Tensor): Selected decoder tokens from all modalities. Shape (B, M, D) where M is the number of selected decoder tokens. + - decoder_emb (torch.Tensor): Corresponding embeddings for decoder tokens. Shape (B, M, D) + - decoder_mask (torch.Tensor): A boolean mask indicating which decoder tokens are valid (set to 0 for valid tokens, 1 otherwise). Shape (B, 1, M) + - target_ids (torch.Tensor): IDs of the target tokens corresponding to the decoder tokens. Shape (B, M) + - decoder_attention_mask (torch.Tensor): Mask for the decoder self-attention layers. Shape (B, M, M) + - mod_mask (torch.Tensor): An integer mask marking the modality type for each decoder token (with -1 indicating unassigned pad tokens). Shape (B, M) + """ + # decoder_mask and target_mask are equivalent, we rename it here to harmonize with forward_mask_encoder + decoder_tokens_all, emb_all, decoder_mask_all, target_ids_all, decoder_attention_mask_all, mod_mask_all = self.cat_decoder_tensors(mod_dict) + + # Add arange multiplied by small constant to mask so they get sorted in a deterministic way + mask_arange = torch.arange(decoder_mask_all.shape[1], device=decoder_mask_all.device).unsqueeze(0) * 1e-6 + ids_shuffle = torch.argsort(decoder_mask_all + mask_arange, dim=1) + # ids_restore = torch.argsort(ids_shuffle, dim=1) + ids_keep = ids_shuffle[:, :num_decoder_tokens] + + decoder_tokens = torch.gather(decoder_tokens_all, dim=1, index=repeat(ids_keep, "b n -> b n d", d=decoder_tokens_all.shape[2])) + decoder_emb = torch.gather(emb_all, dim=1, index=repeat(ids_keep, "b n -> b n d", d=emb_all.shape[2])) + decoder_mask = torch.gather(decoder_mask_all, dim=1, index=ids_keep) + target_ids = torch.gather(target_ids_all, dim=1, index=ids_keep) + decoder_attention_mask = torch.gather(decoder_attention_mask_all, dim=1, index=ids_keep) + mod_mask = torch.gather(mod_mask_all, dim=1, index=ids_keep) + + decoder_tokens[decoder_mask] = 0. + decoder_emb[decoder_mask] = 0. + target_ids[decoder_mask] = 0 + decoder_attention_mask = self.adapt_decoder_attention_mask(decoder_attention_mask, mod_mask) + mod_mask[decoder_mask] = -1 + + # This means this mask can then be re-used for decoder cross-attention + decoder_mask = rearrange(decoder_mask, 'b n2 -> b 1 n2') + + + return decoder_tokens, decoder_emb, decoder_mask, target_ids, decoder_attention_mask, mod_mask + + def adapt_decoder_attention_mask(self, decoder_attention_mask: torch.Tensor, mod_mask=Optional[torch.Tensor]) -> torch.Tensor: + """ + Transforms the compressed decoder attention mask to a full attention mask based on the specified constraints. + + Args: + decoder_attention_mask (torch.Tensor): Initial attention mask indicating attention constraints. Shape (B, M) where M is the number of the decoder tokens. + mod_mask (torch.Tensor, optional): Modality mask to separate attention masks per modality. Shape (B, M) + + Returns: + torch.Tensor: Adapted attention mask. Shape (B, M, M) where M is the number of the decoder tokens. + """ + B, N = decoder_attention_mask.shape + + if self.decoder_causal_mask: + # For causal mode, tokens can only attend to preceding tokens and themselves. + causal_mask = torch.ones((N, N), dtype=torch.bool, device=decoder_attention_mask.device).triu(1) + causal_mask = repeat(causal_mask, "n1 n2 -> b n1 n2", b=B) + adapted_attention_mask = causal_mask + else: + # Cumulatively sum the attention mask to determine token-wise attention behavior. + # Examples: + # Mask [4, 0, 0, 0] -> Cumsum: [4, 4, 4, 4] -> All tokens attend to each other. + # Mask [1, 1, 1, 1] -> Cumsum: [1, 2, 3, 4] -> Strict autoregressive behavior. + # Mask [2, 0, 1, 1] -> Cumsum: [2, 2, 3, 4] -> Tokens 1 and 2 attend to each other, token 3 attends to tokens 1-3, and token 4 to all. + attention_arange = torch.arange(N, device=decoder_attention_mask.device) + attention_arange = repeat(attention_arange, "n2 -> b n1 n2", b=B, n1=N) + cumsum_mask = torch.cumsum(decoder_attention_mask, dim=-1) + cumsum_mask = rearrange(cumsum_mask, "b n -> b n 1") + adapted_attention_mask = (attention_arange >= cumsum_mask) + + if self.decoder_sep_mask: + # Separate attention between tokens based on their modality using mod_mask. + sep_mask = repeat(mod_mask, "b n2 -> b n1 n2", n1=N) != repeat(mod_mask, "b n1 -> b n1 n2", n2=N) + adapted_attention_mask = adapted_attention_mask | sep_mask + + return adapted_attention_mask + + def forward_encoder(self, + x: torch.Tensor, + encoder_mask: torch.Tensor) -> torch.Tensor: + """Forward pass for the encoder. + + Args: + x (torch.Tensor): Encoder input tokens. Shape (B, N, D) where N is the number of encoder tokens. + encoder_mask (torch.Tensor): Encoder mask indicating which tokens are valid (set to 0 for valid tokens, 1 otherwise). Shape (B, 1, N) + + Returns: + torch.Tensor: Encoder output. Shape (B, N, D) + """ + + for blk in self.encoder: + x = blk(x, mask=encoder_mask) + + x = self.encoder_norm(x) + + return x + + def forward_decoder(self, + y: torch.Tensor, + context: torch.Tensor, + encoder_mask: torch.Tensor, + decoder_attention_mask: torch.Tensor) -> torch.Tensor: + """Forward pass for the decoder. + + Args: + y (torch.Tensor): Decoder input tokens. Shape (B, M, D). + context (torch.Tensor): Context for the decoder (i.e. encoder output). Shape (B, N, D). + encoder_mask (torch.Tensor): Encoder mask indicating which tokens are valid (set to 0 for valid tokens, 1 otherwise). Shape (B, 1, N). + decoder_attention_mask (torch.Tensor): Decoder attention mask. Shape (B, M, M). + + Returns: + torch.Tensor: Decoder output. Shape (B, M, D). + """ + + for blk in self.decoder: + y = blk(y, context, sa_mask=decoder_attention_mask, xa_mask=encoder_mask) + + y = self.decoder_norm(y) + + return y + + def forward_logits(self, + y: torch.Tensor, + decoder_mod_dict: Dict[str, Dict[str, torch.Tensor]], + decoder_mod_mask: torch.Tensor, + return_all_logits: bool = False) -> Dict[str, torch.Tensor]: + """Forward computation of logits for each modality. + + Args: + y (torch.Tensor): Decoder output. Shape (B, M, D). + decoder_mod_dict (dict): Dictionary containing tensor information for each modality in the decoder. + decoder_mod_mask (torch.Tensor): Integer mask indicating which tokens belong to which modality. Shape (B, M). + + Returns: + Dict[str, torch.Tensor]: Dictionary of logits for each modality. + """ + + mod_logits = {} + for mod, d in decoder_mod_dict.items(): + idx = self.modality_info[mod]["id"] + if return_all_logits: + logits = self.decoder_embeddings[mod].forward_logits(y) + else: + logits = self.decoder_embeddings[mod].forward_logits(y[decoder_mod_mask == idx]) + mod_logits[mod] = logits + return mod_logits + + def forward_loss(self, + y: torch.Tensor, + target_ids: torch.Tensor, + decoder_mod_dict: Dict[str, Any], + decoder_mod_mask: torch.Tensor, loss_type: str) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """Computes the loss based on the specified loss type. + + Args: + y (torch.Tensor): Decoder output. Shape (B, M, D). + target_ids (torch.Tensor): Ground truth token IDs. Shape (B, M). + decoder_mod_dict (dict): Dictionary containing tensor information for each modality in the decoder. + decoder_mod_mask (torch.Tensor): Integer mask indicating which tokens belong to which modality. Shape (B, M). + loss_type (str): The type of loss to compute. Either 'mod' or 'token'. + + Returns: + Tuple[torch.Tensor, Dict[str, torch.Tensor]]: Total loss and dictionary of loss for each modality. + """ + if loss_type in ['mod', 'modality']: + loss, mod_loss = self.forward_mod_loss(y, target_ids, decoder_mod_dict, decoder_mod_mask) + elif loss_type == 'token': + loss, mod_loss = self.forward_token_loss(y, target_ids, decoder_mod_dict, decoder_mod_mask) + else: + raise ValueError("Invalid loss type") + + return loss, mod_loss + + def forward_mod_loss(self, + y: torch.Tensor, + target_ids: torch.Tensor, + decoder_mod_dict: Dict[str, Any], + decoder_mod_mask: torch.Tensor) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """Computes the modality-wise loss. + + Args: + y (torch.Tensor): Decoder tokens. Shape (B, M, D). + target_ids (torch.Tensor): Ground truth token IDs. Shape (B, M). + decoder_mod_dict (dict): Dictionary containing tensor information for each modality in the decoder. + decoder_mod_mask (torch.Tensor): Mask indicating which tokens belong to which modality. Shape (B, M). + + Returns: + Tuple[torch.Tensor, Dict[str, torch.Tensor]]: Total modality loss and dictionary of loss for each modality. + """ + mod_loss = {} + for mod, d in decoder_mod_dict.items(): + idx = self.modality_info[mod]["id"] + logits = self.decoder_embeddings[mod].forward_logits(y[decoder_mod_mask == idx]) + if logits.numel() == 0: + # If there are no logits / targets, set mod_loss to 0 + mod_loss[mod] = torch.zeros(1, device=logits.device) + else: + loss = F.cross_entropy(logits, target_ids[decoder_mod_mask == idx].long(), reduction='mean') + mod_loss[mod] = loss + + loss = sum(mod_loss.values()) / len(mod_loss) + + return loss, mod_loss + + def forward_token_loss(self, + y: torch.Tensor, + target_ids: torch.Tensor, + decoder_mod_dict: Dict[str, Any], + decoder_mod_mask: torch.Tensor) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """Computes the token-wise loss. + + Args: + y (torch.Tensor): Decoder tokens. Shape (B, M, D). + target_ids (torch.Tensor): Ground truth token IDs. Shape (B, M). + decoder_mod_dict (dict): Dictionary containing tensor information for each modality in the decoder. + decoder_mod_mask (torch.Tensor): Mask indicating which tokens belong to which modality. Shape (B, M). + + Returns: + Tuple[torch.Tensor, Dict[str, torch.Tensor]]: Total token loss and dictionary of loss for each modality. + """ + mod_loss = {} + mod_count = {} + + for mod, d in decoder_mod_dict.items(): + idx = self.modality_info[mod]["id"] + logits = self.decoder_embeddings[mod].forward_logits(y[decoder_mod_mask == idx]) + if logits.numel() == 0: + # If there are no logits / targets, set mod_loss to 0 + mod_loss[mod] = torch.zeros(1, device=logits.device) + mod_count[mod] = 0 + else: + loss = F.cross_entropy(logits, target_ids[decoder_mod_mask == idx].long(), reduction='mean') + mod_loss[mod] = loss + mod_count[mod] = logits.numel() + + loss = sum([mod_loss[mod] * mod_count[mod] for mod in mod_loss.keys()]) / sum(mod_count.values()) + + return loss, mod_loss + + + def forward(self, + mod_dict: Dict[str, Dict[str, torch.Tensor]], + num_encoder_tokens: int, + num_decoder_tokens: int, + loss_type: str = 'mod', + return_logits: bool = False) -> Union[Dict[str, torch.Tensor], Tuple[torch.Tensor, Dict[str, torch.Tensor]]]: + """ + Forward pass for the model. + + Args: + mod_dict (Dict[str, Dict[str, torch.Tensor]]): Dictionary containing the tensors, masks, and other info for each modality. + - mod_dict[modality_name]["tensor_name"]: Shape can vary based on tensor_name and modality. + num_encoder_tokens (int): Number of tokens to keep for the encoder. + num_decoder_tokens (int): Number of tokens to keep for the decoder. + loss_type (str, optional): The type of loss to compute. Can be 'mod' (average of loss per modality) or 'token' (average loss per token). Default is 'mod'. + return_logits (bool, optional): If True, return the logits. Default is False. + + Returns: + Union[dict, tuple]: + - If return_logits is True: Dictionary of logits for each modality. + - Otherwise: Tuple containing the total loss and dictionary of loss for each modality. + """ + + # Mod dicts + encoder_mod_dict = {mod: self.encoder_embeddings[mod](d) + for mod, d in mod_dict.items() + if mod in self.encoder_embeddings} + encoder_tokens, encoder_emb, encoder_mask, encoder_mod_mask = self.forward_mask_encoder(encoder_mod_dict, num_encoder_tokens) + + decoder_mod_dict = {mod: self.decoder_embeddings[mod].forward_embed(d) + for mod, d in mod_dict.items() + if mod in self.decoder_embeddings} + decoder_tokens, decoder_emb, decoder_mask, target_ids, decoder_attention_mask, decoder_mod_mask = self.forward_mask_decoder(decoder_mod_dict, num_decoder_tokens) + + # Encoder + x = encoder_tokens + encoder_emb + x = self.forward_encoder(x, encoder_mask=encoder_mask) + + # Decoder + context = self.decoder_proj_context(x) + encoder_emb + y = decoder_tokens + decoder_emb + y = self.forward_decoder(y, context, encoder_mask=encoder_mask, decoder_attention_mask=decoder_attention_mask) + + # Logits + if return_logits: + mod_logits = self.forward_logits(y, decoder_mod_dict, decoder_mod_mask, return_all_logits=True) + return mod_logits + + # Loss + loss, mod_loss = self.forward_loss(y, target_ids, decoder_mod_dict, decoder_mod_mask, loss_type) + + return loss, mod_loss + + + def freeze_encoder(self, freeze_embeddings=True): + for param in self.encoder.parameters(): + param.requires_grad = False + + for param in self.encoder_norm.parameters(): + param.requires_grad = False + + if freeze_embeddings: + for param in self.encoder_embeddings.parameters(): + param.requires_grad = False + + def freeze_encoder_except_specific_embeddings(self, frozen_embedding_domain): + frozen_embedding_domain = frozen_embedding_domain.split('-') + for param in self.encoder.parameters(): + param.requires_grad = False + + for param in self.encoder_norm.parameters(): + param.requires_grad = False + + for name, param in self.encoder_embeddings.named_parameters(): + if name.split('.')[0] in frozen_embedding_domain: + param.requires_grad = False + + def unfreeze_encoder(self, unfreeze_embeddings=True): + for param in self.encoder.parameters(): + param.requires_grad = True + + for param in self.encoder_norm.parameters(): + param.requires_grad = True + + if unfreeze_embeddings: + for param in self.encoder_embeddings.parameters(): + param.requires_grad = True + + def freeze_decoder(self, freeze_embeddings=True): + for param in self.decoder.parameters(): + param.requires_grad = False + + for param in self.decoder_norm.parameters(): + param.requires_grad = False + + if freeze_embeddings: + for param in self.decoder_embeddings.parameters(): + param.requires_grad = False + + def freeze_decoder_except_specific_embeddings(self, frozen_embedding_domain): + frozen_embedding_domain = frozen_embedding_domain.split('-') + for param in self.decoder.parameters(): + param.requires_grad = False + + for param in self.decoder_norm.parameters(): + param.requires_grad = False + + for name, param in self.decoder_embeddings.named_parameters(): + if name.split('.')[0] in frozen_embedding_domain: + param.requires_grad = False + + def unfreeze_decoder(self, unfreeze_embeddings=True): + for param in self.decoder.parameters(): + param.requires_grad = True + + for param in self.decoder_norm.parameters(): + param.requires_grad = True + + if unfreeze_embeddings: + for param in self.decoder_embeddings.parameters(): + param.requires_grad = True + + def freeze_shared_params(self): + self.freeze_encoder(freeze_embeddings=False) + self.freeze_decoder(freeze_embeddings=False) + + def freeze_params_except_specific_embeddings(self, frozen_embedding_domain): + self.freeze_encoder_except_specific_embeddings(frozen_embedding_domain=frozen_embedding_domain) + self.freeze_decoder_except_specific_embeddings(frozen_embedding_domain=frozen_embedding_domain) + + def unfreeze_shared_params(self): + self.unfreeze_encoder(unfreeze_embeddings=False) + self.unfreeze_decoder(unfreeze_embeddings=False) + + def unfreeze_all(self): + self.unfreeze_encoder(unfreeze_embeddings=True) + self.unfreeze_decoder(unfreeze_embeddings=True) + + +################################################ + +# Wrapper for easy loading with Huggingface Hub + +class FM(FourM, PyTorchModelHubMixin): + """Wrapper around FourM for easy loading with Huggingface Hub. + + Args: + config (dict): Dictionary containing the model and modality configuration, + used for loading from Huggingface Hub. + """ + def __init__(self, config: dict): + + config = copy.deepcopy(config) + + all_domains = sorted(list(set(config['domains_in']) | set(config['domains_out']))) + modality_info = {mod: MODALITY_INFO[mod] for mod in all_domains} + + encoder_embeddings = {} + for mod in config['domains_in']: + info = modality_info[mod] + if info.get("encoder_embedding", None) is not None: + if info["type"] == "img": + image_size, patch_size = info.get('input_size', config['image_size']), info.get('patch_size', config['patch_size']) + encoder_embeddings[mod] = info["encoder_embedding"](patch_size=patch_size, image_size=image_size) + else: + encoder_embeddings[mod] = info["encoder_embedding"]() + + decoder_embeddings = {} + for mod in config['domains_out']: + info = modality_info[mod] + if info.get("decoder_embedding", None) is not None: + if info["type"] == "img": + image_size, patch_size = info.get('input_size', config['image_size']), info.get('patch_size', config['patch_size']) + decoder_embeddings[mod] = info["decoder_embedding"](patch_size=patch_size, image_size=image_size, share_embedding=False) + else: + decoder_embeddings[mod] = info["decoder_embedding"](share_embedding=False) + + config['norm_layer'] = partial(LayerNorm, eps=1e-6, bias=config['norm_bias']) + config['act_layer'] = getattr(torch.nn, config['act_layer']) + + del config['norm_bias'] + del config['domains_in'] + del config['domains_out'] + del config['image_size'] + del config['patch_size'] + + super().__init__( + encoder_embeddings=encoder_embeddings, + decoder_embeddings=decoder_embeddings, + modality_info=modality_info, + **config + ) + + +################################################ + +# Model definitions + +# GELU variants +@register_model +def fm_tiny_6e_6d_gelu( + encoder_embeddings: Dict[str, nn.Module], + decoder_embeddings: Dict[str, nn.Module], + **kwargs): + model = FourM( + encoder_embeddings=encoder_embeddings, + decoder_embeddings=decoder_embeddings, + encoder_depth=6, + decoder_depth=6, + dim=384, + num_heads=6, + mlp_ratio=4, + qkv_bias=True, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + **kwargs + ) + return model + + +@register_model +def fm_small_8e_8d_gelu( + encoder_embeddings: Dict[str, nn.Module], + decoder_embeddings: Dict[str, nn.Module], + **kwargs): + model = FourM( + encoder_embeddings=encoder_embeddings, + decoder_embeddings=decoder_embeddings, + encoder_depth=8, + decoder_depth=8, + dim=512, + num_heads=8, + mlp_ratio=4, + qkv_bias=True, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + **kwargs + ) + return model + + +@register_model +def fm_base_12e_12d_gelu( + encoder_embeddings: Dict[str, nn.Module], + decoder_embeddings: Dict[str, nn.Module], + **kwargs): + model = FourM( + encoder_embeddings=encoder_embeddings, + decoder_embeddings=decoder_embeddings, + encoder_depth=12, + decoder_depth=12, + dim=768, + num_heads=12, + mlp_ratio=4, + qkv_bias=True, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + **kwargs + ) + return model + + +@register_model +def fm_large_24e_24d_gelu( + encoder_embeddings: Dict[str, nn.Module], + decoder_embeddings: Dict[str, nn.Module], + **kwargs): + model = FourM( + encoder_embeddings=encoder_embeddings, + decoder_embeddings=decoder_embeddings, + encoder_depth=24, + decoder_depth=24, + dim=1024, + num_heads=16, + mlp_ratio=4, + qkv_bias=True, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + **kwargs + ) + return model + +@register_model +def fm_xlarge_24e_24d_gelu( + encoder_embeddings: Dict[str, nn.Module], + decoder_embeddings: Dict[str, nn.Module], + **kwargs): + model = FourM( + encoder_embeddings=encoder_embeddings, + decoder_embeddings=decoder_embeddings, + encoder_depth=24, + decoder_depth=24, + dim=2048, + num_heads=32, + mlp_ratio=4, + qkv_bias=True, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + **kwargs + ) + return model + + +# SwiGLU variants +@register_model +def fm_tiny_6e_6d_swiglu_nobias( + encoder_embeddings: Dict[str, nn.Module], + decoder_embeddings: Dict[str, nn.Module], + **kwargs): + model = FourM( + encoder_embeddings=encoder_embeddings, + decoder_embeddings=decoder_embeddings, + encoder_depth=6, + decoder_depth=6, + dim=384, + num_heads=6, + mlp_ratio=4, + qkv_bias=False, + proj_bias=False, + mlp_bias=False, + norm_layer=partial(LayerNorm, eps=1e-6, bias=False), + act_layer=nn.SiLU, + gated_mlp=True, + **kwargs + ) + return model + + +@register_model +def fm_small_8e_8d_swiglu_nobias( + encoder_embeddings: Dict[str, nn.Module], + decoder_embeddings: Dict[str, nn.Module], + **kwargs): + model = FourM( + encoder_embeddings=encoder_embeddings, + decoder_embeddings=decoder_embeddings, + encoder_depth=8, + decoder_depth=8, + dim=512, + num_heads=8, + mlp_ratio=4, + qkv_bias=False, + proj_bias=False, + mlp_bias=False, + norm_layer=partial(LayerNorm, eps=1e-6, bias=False), + act_layer=nn.SiLU, + gated_mlp=True, + **kwargs + ) + return model + + +@register_model +def fm_base_12e_12d_swiglu_nobias( + encoder_embeddings: Dict[str, nn.Module], + decoder_embeddings: Dict[str, nn.Module], + **kwargs): + model = FourM( + encoder_embeddings=encoder_embeddings, + decoder_embeddings=decoder_embeddings, + encoder_depth=12, + decoder_depth=12, + dim=768, + num_heads=12, + mlp_ratio=4, + qkv_bias=False, + proj_bias=False, + mlp_bias=False, + norm_layer=partial(LayerNorm, eps=1e-6, bias=False), + act_layer=nn.SiLU, + gated_mlp=True, + **kwargs + ) + return model + +@register_model +def fm_large_24e_24d_swiglu_nobias( + encoder_embeddings: Dict[str, nn.Module], + decoder_embeddings: Dict[str, nn.Module], + **kwargs): + model = FourM( + encoder_embeddings=encoder_embeddings, + decoder_embeddings=decoder_embeddings, + encoder_depth=24, + decoder_depth=24, + dim=1024, + num_heads=16, + mlp_ratio=4, + qkv_bias=False, + proj_bias=False, + mlp_bias=False, + norm_layer=partial(LayerNorm, eps=1e-6, bias=False), + act_layer=nn.SiLU, + gated_mlp=True, + **kwargs + ) + return model + +@register_model +def fm_xlarge_24e_24d_swiglu_nobias( + encoder_embeddings: Dict[str, nn.Module], + decoder_embeddings: Dict[str, nn.Module], + **kwargs): + model = FourM( + encoder_embeddings=encoder_embeddings, + decoder_embeddings=decoder_embeddings, + encoder_depth=24, + decoder_depth=24, + dim=2048, + num_heads=32, + mlp_ratio=4, + qkv_bias=False, + proj_bias=False, + mlp_bias=False, + norm_layer=partial(LayerNorm, eps=1e-6, bias=False), + act_layer=nn.SiLU, + gated_mlp=True, + **kwargs + ) + return model + +# SwiGLU + QKNorm variants + + +@register_model +def fm_base_12e_12d_swiglu_qknorm_nobias( + encoder_embeddings: Dict[str, nn.Module], + decoder_embeddings: Dict[str, nn.Module], + **kwargs): + model = FourM( + encoder_embeddings=encoder_embeddings, + decoder_embeddings=decoder_embeddings, + encoder_depth=12, + decoder_depth=12, + dim=768, + num_heads=12, + mlp_ratio=4, + qkv_bias=False, + proj_bias=False, + mlp_bias=False, + norm_layer=partial(LayerNorm, eps=1e-6, bias=False), + act_layer=nn.SiLU, + gated_mlp=True, + qk_norm=True, + **kwargs + ) + return model + + +@register_model +def fm_large_24e_24d_swiglu_qknorm_nobias( + encoder_embeddings: Dict[str, nn.Module], + decoder_embeddings: Dict[str, nn.Module], + **kwargs): + model = FourM( + encoder_embeddings=encoder_embeddings, + decoder_embeddings=decoder_embeddings, + encoder_depth=24, + decoder_depth=24, + dim=1024, + num_heads=16, + mlp_ratio=4, + qkv_bias=False, + proj_bias=False, + mlp_bias=False, + norm_layer=partial(LayerNorm, eps=1e-6, bias=False), + act_layer=nn.SiLU, + gated_mlp=True, + qk_norm=True, + **kwargs + ) + return model + +@register_model +def fm_xlarge_24e_24d_swiglu_qknorm_nobias( + encoder_embeddings: Dict[str, nn.Module], + decoder_embeddings: Dict[str, nn.Module], + **kwargs): + model = FourM( + encoder_embeddings=encoder_embeddings, + decoder_embeddings=decoder_embeddings, + encoder_depth=24, + decoder_depth=24, + dim=2048, + num_heads=32, + mlp_ratio=4, + qkv_bias=False, + proj_bias=False, + mlp_bias=False, + norm_layer=partial(LayerNorm, eps=1e-6, bias=False), + act_layer=nn.SiLU, + gated_mlp=True, + qk_norm=True, + **kwargs + ) + return model \ No newline at end of file diff --git a/fourm/models/fm_utils.py b/fourm/models/fm_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..072acdf32c1bc1104eed9ada9cf92e370d135c0e --- /dev/null +++ b/fourm/models/fm_utils.py @@ -0,0 +1,387 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -------------------------------------------------------- +# Some functions are based on the timm code base +# https://github.com/huggingface/pytorch-image-models +# -------------------------------------------------------- + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange + + +def pair(t): + return t if isinstance(t, tuple) else (t, t) + +def softmax1(tensor): + # See https://www.evanmiller.org/attention-is-off-by-one.html + return F.pad(tensor, (0,1)).softmax(dim=-1)[...,:-1] + +def build_1d_sincos_posemb(max_len, embed_dim=1024, temperature=10000.): + """Sine-cosine positional embeddings from MoCo-v3, adapted back to 1d + + Returns positional embedding of shape (1, N, D) + """ + arange = torch.arange(max_len, dtype=torch.float32) # Shape (N,) + assert embed_dim % 2 == 0, 'Embed dimension must be divisible by 2 for 1D sin-cos position embedding' + pos_dim = embed_dim // 2 + omega = torch.arange(pos_dim, dtype=torch.float32) / pos_dim # Shape (D/2,) + omega = 1. / (temperature ** omega) + out = torch.einsum('n,d->nd', [arange, omega]) # Outer product, shape (N, D/2) + pos_emb = torch.cat([torch.sin(out), torch.cos(out)], dim=1).unsqueeze(0) # Shape (1, N, D) + return pos_emb + +def build_2d_sincos_posemb(h, w, embed_dim=1024, temperature=10000.0): + """Sine-cosine positional embeddings as used in MoCo-v3 + + Returns positional embedding of shape (1, N, D) where N = W*H + """ + grid_w = torch.arange(w, dtype=torch.float32) # Shape (W,) + grid_h = torch.arange(h, dtype=torch.float32) # Shape (H, ) + grid_w, grid_h = torch.meshgrid(grid_w, grid_h, indexing='ij') # Shapes (W, H) + assert embed_dim % 4 == 0, 'Embed dimension must be divisible by 4 for 2D sin-cos position embedding' + pos_dim = embed_dim // 4 + omega = torch.arange(pos_dim, dtype=torch.float32) / pos_dim # Shape (D/4,) + omega = 1. / (temperature ** omega) + out_w = torch.einsum('n,d->nd', [grid_w.reshape(-1), omega]) # Outer product, shape (W*H, D/4) + out_h = torch.einsum('n,d->nd', [grid_h.reshape(-1), omega]) # Outer product, shape (W*H, D/4) + pos_emb = torch.cat([torch.sin(out_w), torch.cos(out_w), torch.sin(out_h), torch.cos(out_h)], dim=1).unsqueeze(0) # Shape (1, W*H, D) + return pos_emb + + +def drop_path(x, drop_prob: float = 0., training: bool = False): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + Implementation from timm: https://github.com/huggingface/pytorch-image-models/blob/main/timm/layers/drop.py + """ + if drop_prob == 0. or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = keep_prob + torch.rand(shape, dtype=x.dtype, device=x.device) + random_tensor.floor_() # binarize + output = x.div(keep_prob) * random_tensor + return output + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + def extra_repr(self) -> str: + return 'p={}'.format(self.drop_prob) + + +class LayerNorm(nn.Module): + """Custom implementation of LayerNorm with the option to disable the bias term""" + def __init__(self, normalized_shape: int, eps=1e-5, bias=True): + super().__init__() + self.eps = eps + self.weight = nn.Parameter(torch.ones(normalized_shape)) + if bias: + self.bias = nn.Parameter(torch.zeros(normalized_shape)) + else: + self.register_buffer("bias", torch.zeros(normalized_shape)) + + # Normalized shape must be a tuple for F.layer_norm + self.normalized_shape = (normalized_shape,) + + def forward(self, x): + return nn.functional.layer_norm(x, self.normalized_shape, self.weight, self.bias, eps=self.eps) + + +class Mlp(nn.Module): + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0., bias=True): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features, bias=bias) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features, bias=bias) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class GatedMlp(nn.Module): + """Implements SwiGLU and other gated feed-forward layers from Noam Shazeer's paper: https://arxiv.org/abs/2002.05202 + """ + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.SiLU, bias=True): + super().__init__() + out_features = out_features or in_features + # If gated, multiply hidden_dim by 2/3 to account for extra matmul + hidden_features = int(2 * (hidden_features or in_features) / 3) + self.fc1 = nn.Linear(in_features, hidden_features, bias=bias) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features, bias=bias) + self.fc3 = nn.Linear(in_features, hidden_features, bias=bias) + + def forward(self, x): + x = self.fc2(self.act(self.fc1(x)) * self.fc3(x)) + return x + + +class Attention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, proj_bias=True, attn_drop=0., proj_drop=0., allow_zero_attn=False): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim ** -0.5 + self.allow_zero_attn = allow_zero_attn + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim, bias=proj_bias) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, mask=None): + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv.unbind(0) # make torchscript happy (cannot use tensor as tuple) + + attn = (q @ k.transpose(-2, -1)) * self.scale + + if mask is not None: + mask = mask.unsqueeze(1) # Unsqueeze attention mask for multi-head + attn = attn.masked_fill(mask, -torch.finfo(attn.dtype).max) + + if self.allow_zero_attn: + attn = softmax1(attn) + else: + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + +class CrossAttention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, proj_bias=True, attn_drop=0., proj_drop=0., allow_zero_attn=False): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim ** -0.5 + self.allow_zero_attn = allow_zero_attn + + self.q = nn.Linear(dim, dim, bias=qkv_bias) + self.kv = nn.Linear(dim, dim * 2, bias=qkv_bias) + + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim, bias=proj_bias) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, context, mask=None): + B, N, C = x.shape + _, M, _ = context.shape + + q = self.q(x).reshape(B, N, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3) + kv = self.kv(context).reshape(B, M, 2, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + k, v = kv[0], kv[1] + + attn = (q @ k.transpose(-2, -1)) * self.scale + if mask is not None: + mask = rearrange(mask, "b n m -> b 1 n m") # Unsqueeze / reshape for multi-head + attn = attn.masked_fill(mask, -torch.finfo(attn.dtype).max) + + if self.allow_zero_attn: + attn = softmax1(attn) + else: + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, -1) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class NormAttention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, proj_bias=True, norm_layer=nn.LayerNorm, attn_drop=0., proj_drop=0., allow_zero_attn=False): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim ** -0.5 + self.allow_zero_attn = allow_zero_attn + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim, bias=proj_bias) + self.proj_drop = nn.Dropout(proj_drop) + + self.q_norm = norm_layer(head_dim) + self.k_norm = norm_layer(head_dim) + + def forward(self, x, mask=None): + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv.unbind(0) # make torchscript happy (cannot use tensor as tuple) + + q = self.q_norm(q) + k = self.k_norm(k) + + attn = (q @ k.transpose(-2, -1)) * self.scale + + if mask is not None: + mask = mask.unsqueeze(1) # Unsqueeze for multi-head + attn = attn.masked_fill(mask, -torch.finfo(attn.dtype).max) + + if self.allow_zero_attn: + attn = softmax1(attn) + else: + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class NormCrossAttention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, proj_bias=True, norm_layer=nn.LayerNorm, attn_drop=0., proj_drop=0., allow_zero_attn=False): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim ** -0.5 + self.allow_zero_attn = allow_zero_attn + + self.q = nn.Linear(dim, dim, bias=qkv_bias) + self.kv = nn.Linear(dim, dim * 2, bias=qkv_bias) + + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim, bias=proj_bias) + self.proj_drop = nn.Dropout(proj_drop) + + self.q_norm = norm_layer(head_dim) + self.k_norm = norm_layer(head_dim) + + def forward(self, x, context, mask=None): + B, N, C = x.shape + _, M, _ = context.shape + + q = self.q(x).reshape(B, N, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3) + kv = self.kv(context).reshape(B, M, 2, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + k, v = kv[0], kv[1] + + q = self.q_norm(q) + k = self.k_norm(k) + + attn = (q @ k.transpose(-2, -1)) * self.scale + if mask is not None: + mask = rearrange(mask, "b n m -> b 1 n m") # Unsqueeze / reshape for multi-head + attn = attn.masked_fill(mask, -torch.finfo(attn.dtype).max) + + if self.allow_zero_attn: + attn = softmax1(attn) + else: + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, -1) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class Block(nn.Module): + + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=True, proj_bias=True, mlp_bias=True, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, gated_mlp=False, qk_norm=False, allow_zero_attn=False): + super().__init__() + self.norm1 = norm_layer(dim) + + if not qk_norm: + self.attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, proj_bias=proj_bias, attn_drop=attn_drop, proj_drop=drop, allow_zero_attn=allow_zero_attn) + else: + self.attn = NormAttention(dim, num_heads=num_heads, qkv_bias=qkv_bias, proj_bias=proj_bias, norm_layer=norm_layer, attn_drop=attn_drop, proj_drop=drop, allow_zero_attn=allow_zero_attn) + + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + + if not gated_mlp: + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, bias=mlp_bias, drop=drop) + else: + self.mlp = GatedMlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, bias=mlp_bias) + + def forward(self, x, mask=None): + x = x + self.drop_path(self.attn(self.norm1(x), mask)) + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + + +class DecoderBlock(nn.Module): + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=True, proj_bias=True, mlp_bias=True, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, gated_mlp=False, qk_norm=False, allow_zero_attn=False): + super().__init__() + self.norm1 = norm_layer(dim) + + if not qk_norm: + self.self_attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, proj_bias=proj_bias, attn_drop=attn_drop, proj_drop=drop, allow_zero_attn=allow_zero_attn) + self.cross_attn = CrossAttention(dim, num_heads=num_heads, qkv_bias=qkv_bias, proj_bias=proj_bias, attn_drop=attn_drop, proj_drop=drop, allow_zero_attn=allow_zero_attn) + else: + self.self_attn = NormAttention(dim, num_heads=num_heads, qkv_bias=qkv_bias, proj_bias=proj_bias, norm_layer=norm_layer, attn_drop=attn_drop, proj_drop=drop, allow_zero_attn=allow_zero_attn) + self.cross_attn = NormCrossAttention(dim, num_heads=num_heads, qkv_bias=qkv_bias, proj_bias=proj_bias, norm_layer=norm_layer, attn_drop=attn_drop, proj_drop=drop, allow_zero_attn=allow_zero_attn) + + + self.query_norm = norm_layer(dim) + self.context_norm = norm_layer(dim) + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + + if not gated_mlp: + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, bias=mlp_bias, drop=drop) + else: + self.mlp = GatedMlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, bias=mlp_bias) + + def forward(self, x, context, sa_mask=None, xa_mask=None): + x = x + self.drop_path(self.self_attn(self.norm1(x), sa_mask)) + x = x + self.drop_path(self.cross_attn(self.query_norm(x), self.context_norm(context), xa_mask)) + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + + +class CrossAttentionBlock(nn.Module): + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, gated_mlp=False, allow_zero_attn=False): + super().__init__() + self.cross_attn = CrossAttention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop, allow_zero_attn=allow_zero_attn) + self.query_norm = norm_layer(dim) + self.context_norm = norm_layer(dim) + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + if not gated_mlp: + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + else: + self.mlp = GatedMlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer) + + def forward(self, x, context, xa_mask=None, **kwargs): + x = x + self.drop_path(self.cross_attn(self.query_norm(x), self.context_norm(context), xa_mask)) + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x diff --git a/fourm/models/fm_vit.py b/fourm/models/fm_vit.py new file mode 100644 index 0000000000000000000000000000000000000000..ba466cba535f62176da4db9df7a1d53e8da06255 --- /dev/null +++ b/fourm/models/fm_vit.py @@ -0,0 +1,485 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +import copy +from functools import partial +from typing import Optional, Union + +import torch +from torch import nn + +from fourm.utils.timm.registry import register_model +from huggingface_hub import PyTorchModelHubMixin + +from .encoder_embeddings import ImageEncoderEmbedding +from .fm_utils import Block, LayerNorm +from fourm.data.modality_info import MODALITY_INFO + + +__all__ = [ + # GELU models + 'fm_vit_tiny_6e_gelu', + 'fm_vit_small_8e_gelu', + 'fm_vit_base_12e_gelu', + 'fm_vit_large_24e_gelu', + 'fm_vit_xlarge_24e_gelu', + # SwiGLU models + 'fm_vit_tiny_6e_swiglu_nobias', + 'fm_vit_small_8e_swiglu_nobias', + 'fm_vit_base_12e_swiglu_nobias', + 'fm_vit_large_24e_swiglu_nobias', + 'fm_vit_xlarge_24e_swiglu_nobias', + # SwiGLU + QKNorm models + 'fm_vit_base_12e_swiglu_qknorm_nobias', + 'fm_vit_large_24e_swiglu_qknorm_nobias', + 'fm_vit_xlarge_24e_swiglu_qknorm_nobias', +] + +class FourMViT(nn.Module): + """Modified 4M model, adapted to behave as a simple RGB-only ViT. + + Args: + img_size (int): Input image size. + patch_size (int): Patch size. + in_chans (int): Number of input image channels. + dim (int): Patch embedding dimension. + encoder_depth (int): Depth of ViT / number of encoder blocks. + num_heads (int): Number of attention heads in each ViT block. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool): If True, add a learnable bias to query, key, value. + proj_bias (bool): If True, adds a bias to the attention out proj layer. + mlp_bias (bool): If True, adds a learnable bias for the feedforward. + drop_path_rate (float): Stochastic depth rate. + drop_rate (float): Dropout rate. + attn_drop_rate (float): Attention dropout rate. + act_layer (nn.Module): Activation layer. + norm_layer (nn.Module): Normalization layer. + gated_mlp (bool): If True, makes the feedforward gated (e.g., for SwiGLU) + qk_norm (bool): If True, normalizes the query and keys (as in ViT-22B) + use_act_checkpoint (bool): If True, use activation checkpointing. + encoder_norm (bool): If True, adds a norm layer after the last encoder block. + output_head (Optional[nn.Module]): Optional output head after the encoder + """ + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + dim=768, + encoder_depth=12, + num_heads=12, + mlp_ratio=4.0, + qkv_bias: bool = True, + proj_bias: bool = True, + mlp_bias: bool = True, + drop_path_rate: float =0.0, + drop_rate: float = 0.0, + attn_drop_rate: float =0.0, + act_layer: torch.Tensor =nn.GELU, + norm_layer: Union[partial, nn.Module] = partial(LayerNorm, eps=1e-6), + gated_mlp: bool = False, # Make the feedforward gated for e.g. SwiGLU + qk_norm: bool = False, + encoder_norm = True, + output_head: Optional[nn.Module] = None, + ): + super().__init__() + self.img_size = img_size + self.init_std = 0.02 + rgb_embedding = ImageEncoderEmbedding(num_channels=in_chans, patch_size=patch_size, + dim_tokens=dim, sincos_pos_emb=True, image_size=img_size) + self.num_patches = rgb_embedding.num_patches + self.encoder_embeddings = nn.ModuleDict({f"rgb@{img_size}": rgb_embedding}) + + # stochastic depth decay rule + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, encoder_depth)] + + self.encoder = nn.ModuleList([ + Block(dim=dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, proj_bias=proj_bias, mlp_bias=mlp_bias, + drop_path=dpr[i], drop=drop_rate, attn_drop=attn_drop_rate, act_layer=act_layer, norm_layer=norm_layer, + gated_mlp=gated_mlp, qk_norm=qk_norm) + for i in range(encoder_depth) + ]) + + self.encoder_norm = norm_layer(dim) if encoder_norm else nn.Identity() + + # Weight init + self.init_weights() + + # Classification head is initialized after init_weights() to allow for special init scale + if output_head is not None: + self.output_head = output_head + if hasattr(self.output_head, 'init'): + self.output_head.init(dim) + else: + self.output_head = nn.Identity() + + def init_weights(self): + """Weight initialization following MAE's initialization scheme""" + + for name, m in self.named_modules(): + # Skipping tokenizers to avoid reinitializing them + if "tokenizer" in name: + continue + # Linear + elif isinstance(m, nn.Linear): + if 'qkv' in name: + # treat the weights of Q, K, V separately + val = math.sqrt(6. / float(m.weight.shape[0] // 3 + m.weight.shape[1])) + nn.init.uniform_(m.weight, -val, val) + elif 'kv' in name: + # treat the weights of K, V separately + val = math.sqrt(6. / float(m.weight.shape[0] // 2 + m.weight.shape[1])) + nn.init.uniform_(m.weight, -val, val) + else: + nn.init.xavier_uniform_(m.weight) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + # LayerNorm + elif isinstance(m, nn.LayerNorm) or isinstance(m, LayerNorm): + nn.init.constant_(m.weight, 1.0) + nn.init.constant_(m.bias, 0) + + # Embedding + elif isinstance(m, nn.Embedding): + nn.init.normal_(m.weight, std=self.init_std) + # Conv2d + elif isinstance(m, nn.Conv2d): + if '.proj' in name: + # From MAE, initialize projection like nn.Linear (instead of nn.Conv2d) + w = m.weight.data + nn.init.xavier_uniform_(w.view([w.shape[0], -1])) + + def get_num_layers_encoder(self): + return len(self.encoder) + + def get_num_layers(self): + return self.get_num_layers_encoder() + + @torch.jit.ignore + def no_weight_decay(self): + no_wd_set = set() + + for mod, emb_module in self.encoder_embeddings.items(): + if hasattr(emb_module, 'no_weight_decay'): + to_skip = emb_module.no_weight_decay() + to_skip = set([f'encoder_embeddings.{mod}.{name}' for name in to_skip]) + no_wd_set = no_wd_set | to_skip + + return no_wd_set + + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Forward pass of the model. + + Args: + x (torch.Tensor): Input tensor. Shape (B, C, H, W) + + Returns: + torch.Tensor: Output tensor. Shape (B, num_classes). + """ + rgb_dict = {'tensor': x} + rgb_dict = self.encoder_embeddings[f'rgb@{self.img_size}'](rgb_dict) + + # Add embeddings to patchified RGB image + x = rgb_dict['x'] + rgb_dict['emb'] # Shape: (B, N, D) with N = num_patches + + for blk in self.encoder: + x = blk(x) + + x = self.encoder_norm(x) # Shape: (B, N, D) + + out = self.output_head(x) + + return out + + + def freeze_encoder(self, freeze_embeddings=True): + for param in self.encoder.parameters(): + param.requires_grad = False + + for param in self.encoder_norm.parameters(): + param.requires_grad = False + + if freeze_embeddings: + for param in self.encoder_embeddings.parameters(): + param.requires_grad = False + + def unfreeze_encoder(self, unfreeze_embeddings=True): + for param in self.encoder.parameters(): + param.requires_grad = True + + for param in self.encoder_norm.parameters(): + param.requires_grad = True + + if unfreeze_embeddings: + for param in self.encoder_embeddings.parameters(): + param.requires_grad = True + + +################################################ + +# Wrapper for easy loading with Huggingface Hub + +class FMViT(FourMViT, PyTorchModelHubMixin): + """Wrapper around FourMViT for easy loading with Huggingface Hub. + + Args: + config (dict): Dictionary containing the model and modality configuration, + used for loading from Huggingface Hub. + output_head (nn.Module): Optional output head. + """ + def __init__(self, config: dict, output_head: Optional[nn.Module] = None): + + config = copy.deepcopy(config) + + + + config['norm_layer'] = partial(LayerNorm, eps=1e-6, bias=config['norm_bias']) + config['act_layer'] = getattr(torch.nn, config['act_layer']) + + img_size = config['image_size'] + config['img_size'] = img_size + config['patch_size'] = MODALITY_INFO[f'rgb@{img_size}'].get('patch_size', config['patch_size']) + config['in_chans'] = MODALITY_INFO[f'rgb@{img_size}'].get('num_channels', 3) + + for key in ['image_size', 'norm_bias', 'domains_in', 'domains_out', 'decoder_depth', 'share_modality_embeddings']: + if key in config: + del config[key] + + super().__init__( + output_head=output_head, + **config + ) + + +################################################ + +# Model definitions + +# GELU variants +@register_model +def fm_vit_tiny_6e_gelu(**kwargs): + model = FourMViT( + encoder_depth=6, + dim=384, + num_heads=6, + mlp_ratio=4, + qkv_bias=True, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + **kwargs + ) + return model + + +@register_model +def fm_vit_small_8e_gelu(**kwargs): + model = FourMViT( + encoder_depth=8, + dim=512, + num_heads=8, + mlp_ratio=4, + qkv_bias=True, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + **kwargs + ) + return model + + +@register_model +def fm_vit_base_12e_gelu(**kwargs): + model = FourMViT( + encoder_depth=12, + dim=768, + num_heads=12, + mlp_ratio=4, + qkv_bias=True, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + **kwargs + ) + return model + + +@register_model +def fm_vit_large_24e_gelu(**kwargs): + model = FourMViT( + encoder_depth=24, + dim=1024, + num_heads=16, + mlp_ratio=4, + qkv_bias=True, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + **kwargs + ) + return model + +@register_model +def fm_vit_xlarge_24e_gelu(**kwargs): + model = FourMViT( + encoder_depth=24, + dim=2048, + num_heads=32, + mlp_ratio=4, + qkv_bias=True, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + **kwargs + ) + return model + + +# SwiGLU variants +@register_model +def fm_vit_tiny_6e_swiglu_nobias(**kwargs): + model = FourMViT( + encoder_depth=6, + dim=384, + num_heads=6, + mlp_ratio=4, + qkv_bias=False, + proj_bias=False, + mlp_bias=False, + norm_layer=partial(LayerNorm, eps=1e-6, bias=False), + act_layer=nn.SiLU, + gated_mlp=True, + **kwargs + ) + return model + + +@register_model +def fm_vit_small_8e_swiglu_nobias(**kwargs): + model = FourMViT( + encoder_depth=8, + dim=512, + num_heads=8, + mlp_ratio=4, + qkv_bias=False, + proj_bias=False, + mlp_bias=False, + norm_layer=partial(LayerNorm, eps=1e-6, bias=False), + act_layer=nn.SiLU, + gated_mlp=True, + **kwargs + ) + return model + + +@register_model +def fm_vit_base_12e_swiglu_nobias(**kwargs): + model = FourMViT( + encoder_depth=12, + dim=768, + num_heads=12, + mlp_ratio=4, + qkv_bias=False, + proj_bias=False, + mlp_bias=False, + norm_layer=partial(LayerNorm, eps=1e-6, bias=False), + act_layer=nn.SiLU, + gated_mlp=True, + **kwargs + ) + return model + + +@register_model +def fm_vit_large_24e_swiglu_nobias(**kwargs): + model = FourMViT( + encoder_depth=24, + dim=1024, + num_heads=16, + mlp_ratio=4, + qkv_bias=False, + proj_bias=False, + mlp_bias=False, + norm_layer=partial(LayerNorm, eps=1e-6, bias=False), + act_layer=nn.SiLU, + gated_mlp=True, + **kwargs + ) + return model + +@register_model +def fm_vit_xlarge_24e_swiglu_nobias(**kwargs): + model = FourMViT( + encoder_depth=24, + dim=2048, + num_heads=32, + mlp_ratio=4, + qkv_bias=False, + proj_bias=False, + mlp_bias=False, + norm_layer=partial(LayerNorm, eps=1e-6, bias=False), + act_layer=nn.SiLU, + gated_mlp=True, + **kwargs + ) + return model + +# SwiGLU + QKNorm variants + +@register_model +def fm_vit_base_12e_swiglu_qknorm_nobias(**kwargs): + model = FourMViT( + encoder_depth=12, + dim=768, + num_heads=12, + mlp_ratio=4, + qkv_bias=False, + proj_bias=False, + mlp_bias=False, + norm_layer=partial(LayerNorm, eps=1e-6, bias=False), + act_layer=nn.SiLU, + gated_mlp=True, + qk_norm=True, + **kwargs + ) + return model + + +@register_model +def fm_vit_large_24e_swiglu_qknorm_nobias(**kwargs): + model = FourMViT( + encoder_depth=24, + dim=1024, + num_heads=16, + mlp_ratio=4, + qkv_bias=False, + proj_bias=False, + mlp_bias=False, + norm_layer=partial(LayerNorm, eps=1e-6, bias=False), + act_layer=nn.SiLU, + gated_mlp=True, + qk_norm=True, + **kwargs + ) + return model + +@register_model +def fm_vit_xlarge_24e_swiglu_qknorm_nobias(**kwargs): + model = FourMViT( + encoder_depth=24, + dim=2048, + num_heads=32, + mlp_ratio=4, + qkv_bias=False, + proj_bias=False, + mlp_bias=False, + norm_layer=partial(LayerNorm, eps=1e-6, bias=False), + act_layer=nn.SiLU, + gated_mlp=True, + qk_norm=True, + **kwargs + ) + return model \ No newline at end of file diff --git a/fourm/models/generate.py b/fourm/models/generate.py new file mode 100644 index 0000000000000000000000000000000000000000..4b5a951636696f38be067e8757d2d002dc2aac6f --- /dev/null +++ b/fourm/models/generate.py @@ -0,0 +1,1273 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import defaultdict +from typing import Union, List, Optional + +import numpy as np +import torch +from einops import rearrange, repeat +from torch import nn +import torch.nn.functional as F + +from fourm.utils import get_sentinel_to_id_mapping, merge_span_masking +from fourm.utils.generation import cosine_schedule, linear_schedule, onex_temp_schedule, linear_temp_schedule, continue_schedule +from tqdm import tqdm +import copy + + + +def empty_img_modality(mod_dict, key): + # Input mask + mod_dict[key]['input_mask'][:] = True + + # Target Mask + mod_dict[key]['target_mask'][:] = False + + return mod_dict + +def empty_seq_modality(mod_dict, key, s1_id=5): + # To create an empty sequence, we suppose an input budget of 1, and the rest assigned to targets + + # Input tensor + # Input is [S_1], target is [S_1] ...... [S_2] + # (so [S_1] [S_1] ..... [S_2] when combined) + mod_dict[key]['tensor'][:] = 0 + mod_dict[key]['tensor'][:,[0,1]] = s1_id # s1_id is id of the first sentinel token ([S_1]) + mod_dict[key]['tensor'][:,-1] = s1_id + 1 + + # Input mask + # Set first token to input (i.e. 0), rest to target (i.e. 1) + mod_dict[key]['input_mask'][:] = True + mod_dict[key]['input_mask'][:,0] = False + + # Target Mask + mod_dict[key]['target_mask'] = ~mod_dict[key]['input_mask'] + + # Decoder attn mask + # WARNING: Not needed / used in GenerationSampler, where causal mask is enforced + # First token is input, not part of target + mod_dict[key]['decoder_attention_mask'][:] = 1 + mod_dict[key]['decoder_attention_mask'][:, 0] = 0 + + return mod_dict + +def empty_seq_emb_modality(mod_dict, key): + # Tensor + mod_dict[key]['tensor'] = torch.zeros_like(mod_dict[key]['tensor']) + + # Input mask + mod_dict[key]['input_mask'] = torch.ones_like(mod_dict[key]['input_mask']) + # It is crucial to specify the input mask as such, CFG won't work otherwise! + mod_dict[key]['input_mask'][:, 0] = False + + # Target Mask + mod_dict[key]['target_mask'] = torch.ones_like(mod_dict[key]['target_mask']) + + # Decoder attn mask + mod_dict[key]['decoder_attention_mask'][:] = False + + return mod_dict + + +def init_empty_target_modality(mod_dict, modality_info, domain, batch_size, num_tokens, device): + """ + Initializes an empty target modality dictionary for a given domain. + Used to initialize target modality dictionaries for generation. + """ + if modality_info[domain]['type'] == 'img': + # Initialize mod dict + mod_dict[domain] = { + 'tensor': torch.zeros((batch_size, num_tokens), dtype=torch.int64, device=device), + 'input_mask': torch.ones((batch_size, num_tokens), dtype=torch.bool, device=device), + 'target_mask': torch.zeros((batch_size, num_tokens), dtype=torch.bool, device=device), + } + # Set it to the correct values + mod_dict = empty_img_modality(mod_dict, domain) + + elif modality_info[domain]['type'] in ['seq', 'seq_token', 'seq_emb']: + # Initialize mod dict + num_tokens = max(num_tokens, 2) + mod_dict[domain] = { + 'tensor': torch.zeros((batch_size, num_tokens), dtype=torch.int32, device=device), + 'input_mask': torch.ones((batch_size, num_tokens), dtype=torch.bool, device=device), + 'target_mask': torch.zeros((batch_size, num_tokens), dtype=torch.bool, device=device), + 'decoder_attention_mask': torch.zeros((batch_size, num_tokens), dtype=torch.bool, device=device), + } + # Set it to the correct values + if modality_info[domain]['type'] in ['seq', 'seq_token']: + mod_dict = empty_seq_modality(mod_dict, domain) + elif modality_info[domain]['type'] == 'seq_emb': + mod_dict = empty_seq_emb_modality(mod_dict, domain) + else: + raise ValueError() + + return mod_dict + +def init_full_input_modality(mod_dict, modality_info, domain, device, eos_id=3): + if domain.startswith('rgb'): + batch_size, _, H, W = mod_dict[domain]['tensor'].shape + patch_size = modality_info[domain]['patch_size'] + num_tokens = (H // patch_size) * (W // patch_size) + shape = (batch_size, num_tokens) + else: + shape = mod_dict[domain]['tensor'].shape + if 'input_mask' not in mod_dict[domain]: + mod_dict[domain]['input_mask'] = torch.zeros(shape, dtype=torch.bool, device=device) + if 'target_mask' not in mod_dict[domain]: + mod_dict[domain]['target_mask'] = torch.ones(shape, dtype=torch.bool, device=device) + if 'decoder_attention_mask' not in mod_dict[domain]: + mod_dict[domain]['decoder_attention_mask'] = torch.zeros(shape, dtype=torch.bool, device=device) + + if modality_info[domain]['type'] == 'img': + mod_dict[domain]['input_mask'][:] = False + mod_dict[domain]['target_mask'][:] = True + + elif modality_info[domain]['type'] in ['seq', 'seq_token']: + if eos_id in mod_dict[domain]['tensor']: + eos_idx = torch.where(mod_dict[domain]['tensor'] == eos_id)[1][0].item() + else: + mod_dict[domain]['tensor'][:,0] = eos_id + eos_idx = 0 + mod_dict[domain]['input_mask'][:,:eos_idx+1] = False + mod_dict[domain]['input_mask'][:,eos_idx+1:] = True + mod_dict[domain]['target_mask'][:] = True + + elif modality_info[domain]['type'] in ['seq_emb']: + # T5 caption has the valid mask saved alongside the embeddings + mod_dict[domain]['input_mask'] = ~mod_dict[domain]['mask_valid'] + mod_dict[domain]['target_mask'] = torch.ones_like(mod_dict[domain]['mask_valid']) + mod_dict[domain]['decoder_attention_mask'] = torch.zeros_like(mod_dict[domain]['mask_valid']) + + return mod_dict + +def custom_text(sample, input_text, eos_token, key, device, text_tokenizer, target_max_len=50, start_token="[S_1]"): + input_ids = text_tokenizer.encode(input_text).ids + input_ids = torch.tensor(input_ids).unsqueeze(0) + + target_text = [start_token] + target_text.extend(["[PAD]"] * (target_max_len - 2)) + target_text.append(eos_token) + target_text = " ".join(target_text) + target_ids = text_tokenizer.encode(target_text).ids + target_ids = torch.tensor(target_ids).unsqueeze(0) + + all_ids = torch.cat([input_ids, target_ids], dim=1) + + input_mask = torch.cat([ + torch.zeros_like(input_ids, dtype=torch.bool), + torch.ones_like(target_ids, dtype=torch.bool), + ], dim=1) + + target_mask = torch.cat([ + torch.ones_like(input_ids, dtype=torch.bool), + torch.zeros_like(target_ids, dtype=torch.bool), + ], dim=1) + + sample[key] = {} + sample[key]['tensor'] = all_ids.to(device) + sample[key]['input_mask'] = input_mask.to(device) + sample[key]['target_mask'] = target_mask.to(device) + sample[key]['decoder_attention_mask'] = torch.zeros(all_ids.shape, dtype=torch.bool, device=device) + + return sample + +def expand_to_batch(mod_dict, batch_size): + for mod, d in mod_dict.items(): + for k, v in d.items(): + if k in ['tensor', 'input_mask', 'target_mask', 'decoder_attention_mask', 'mask_valid']: + B = v.shape[0] + if B == 1: + mod_dict[mod][k] = repeat(v, "1 ... -> b ...", b=batch_size) + elif B != batch_size: + raise ValueError(f"Invalid batch size: {B} instead of {batch_size}") + + return mod_dict + +def build_chained_generation_schedules( + cond_domains: List[str], + target_domains: List[str], + tokens_per_target: List[int], + autoregression_schemes: List[str], + decoding_steps: List[int], + token_decoding_schedules: List[str], + temps: List[float], + temp_schedules: List[float], + cfg_scales: List[float], + cfg_schedules: List[str], + cfg_grow_conditioning: bool = False, + modality_info: Optional[dict] = None, + ): + """ + Builds a list of chained generation schedules, where each schedule is a tuple of the form: + (target_modality, schema, number of decoded tokens, temperature, guidance_scale, cfg_cond_domains) + + Args: + cond_domains: List of conditioning domains + target_domains: List of target domains + tokens_per_target: List of number of tokens to decode for each target domain + autoregression_schemes: List of autoregression schemes for each target domain. maskgit, roar, or autoregressive + decoding_steps: List of number of maskgit steps for each target domain (if applicable) + token_decoding_schedules: List of maskgit token schedules for each target domain (if applicable). cosine or linear + temps: List of starting temperatures for each target domain + temp_schedules: List of temperature schedules for each target domain. linear, constant, or onex:{min_t}:{power} + cfg_scales: List of classifier-free guidance scales for each target domain + cfg_schedules: List of classifier-free guidance schedules for each target domain. constant or cosine + cfg_grow_conditioning: After every completed modality, add them to classifier-free guidance conditioning + modality_info: Dictionary with metadata for each modality, optionally used to verify that the schedule is compatible with the modality + """ + + # List of {target_modality, schema, number of decoded tokens, temperature, guidance_scale, cfg_cond_domains} dicts + chained_schedules = [] + + cond_domains = cond_domains.copy() + + for target_idx in range(len(target_domains)): + + scheme = autoregression_schemes[target_idx] + target_domain = target_domains[target_idx] + ntoks = tokens_per_target[target_idx] + maskgit_token_schedule_name = token_decoding_schedules[target_idx] + temp = temps[target_idx] + temp_schedule_name = temp_schedules[target_idx] + cfg_scale = cfg_scales[target_idx] + cfg_schedule_name = cfg_schedules[target_idx] + + # Auto-regressive (caption, detection, ...) + if scheme == 'autoregressive': + chained_schedules.append({ + 'target_domain': target_domain, + 'scheme': scheme, + 'num_tokens': None, + 'temperature': temp, + 'cfg_scale': cfg_scale, + 'cfg_cond_domains': cond_domains.copy() + }) + continue + + # Use modality info for (optional) assert if provided + if modality_info is not None: + assert modality_info[target_domain]['type'] not in ['seq', 'seq_token'], f'Illegal autoregressive scheme {scheme} for target domain {target_domain}' + + # Token schedule + if scheme == 'maskgit': + # MaskGIT token schedule setup + num_steps = decoding_steps[target_idx] + if maskgit_token_schedule_name == 'cosine': + token_schedule = cosine_schedule(num_steps, (ntoks)) + elif maskgit_token_schedule_name == 'linear': + token_schedule = linear_schedule(num_steps, (ntoks)) + else: + raise ValueError(f'Illegal MaskGIT token schedule {maskgit_token_schedule_name}') + elif scheme == 'roar': + # ROAR token schedule setup (one-by-one, but random order) + num_steps = decoding_steps[target_idx] + token_schedule = linear_schedule(num_steps, ntoks) + else: + raise ValueError(f'Illegal decoding scheme {scheme}') + + # Temperature schedule + if temp_schedule_name == 'linear': + temp_schedule = linear_temp_schedule(temp, token_schedule) + elif temp_schedule_name == 'constant': + temp_schedule = temp * np.ones(num_steps) + elif 'onex' in temp_schedule_name: + # onex temperature schedule has to be formatted like onex:{min_t}:{power} + min_t, power = [float(f) for f in temp_schedule_name.split(':')[1:]] + temp_schedule = onex_temp_schedule(max_t=temp, min_t=min_t, token_schedule=token_schedule, power=power) + else: + raise ValueError(f'Illegal temperature schedule {temp_schedule_name}') + + # Classifier-free guidance scale schedule + if cfg_schedule_name == 'constant': + if isinstance(cfg_scale, float): + cfg_schedule = cfg_scale * np.ones(num_steps) + elif isinstance(cfg_scale, list): + cfg_schedule = np.array(cfg_scale) * np.ones(num_steps).reshape(-1, 1) + elif cfg_schedule_name == 'cosine': + raise NotImplementedError() + else: + raise ValueError(f'Illegal guidance schedule {cfg_schedule_name}') + + # Concatenate schedule for this modality with previous ones + schedule = [ + { + 'target_domain': target_domain, + 'scheme': scheme, + 'num_tokens': tok, + 'temperature': temp, + 'cfg_scale': cfg, + 'cfg_cond_domains': cond_domains.copy() + } + for tok, temp, cfg in zip(token_schedule, temp_schedule, cfg_schedule) + ] + chained_schedules.extend(schedule) + + # Optionally add this new modality to the ones affected by classifier-free guidance + if cfg_grow_conditioning: + cond_domains.append(target_domain) + + return chained_schedules + + +class GenerationSampler(nn.Module): + """Sampler that wraps a trained 4M model for generation use cases. + Implements standard autoregressive, MaskGIT, and ROAR generation schemes with chaining and weighted guidance.""" + + def __init__(self, model): + super().__init__() + self.model = model + + + def top_k_top_p_filtering(self, logits, top_k=0.0, top_p=0.0): + # Compatible with batching + # From https://gist.github.com/thomwolf/1a5a29f6962089e871b94cbd09daf317 + if top_k > 0.0: + if isinstance(top_k, int): + k = min(top_k, logits.shape[-1]) + elif isinstance(top_k, float): + k = min(int(top_k * logits.shape[-1]), logits.shape[-1]) + else: + raise ValueError(f"Invalid value for top_k: {top_k}") + + # Remove all tokens with a probability less than the last token of the top-k + indices_to_remove = logits < torch.topk(logits, k)[0][..., -1, None] + logits[indices_to_remove] = float("-inf") + + if top_p > 0.0: + sorted_logits, sorted_indices = torch.sort(logits, dim=1, descending=True) + cum_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1) + sorted_indices_to_remove = cum_probs > top_p + # Shift the indices to the right to keep also the first token above the threshold + sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone() + sorted_indices_to_remove[..., 0] = 0 + + restore_indices = torch.argsort(sorted_indices, dim=-1) + indices_to_remove = torch.gather(sorted_indices_to_remove, dim=-1, index=restore_indices) + logits[indices_to_remove] = float("-inf") + + return logits + + def sample_tokens(self, logits, temperature=1.0, top_k=0.0, top_p=0.0): + if np.isclose(temperature, 0, atol=1e-10): + samples = torch.argmax(logits, dim=-1) + # Since argmax is used, all sampled_probs will be 1 as we're selecting the max probability + sampled_probs = torch.ones_like(samples, dtype=torch.float32) + else: + filtered_logits = self.top_k_top_p_filtering(logits, top_k, top_p) + probs = F.softmax(filtered_logits / temperature, dim=-1) + samples = torch.multinomial(probs, 1)[:, 0] + sampled_probs = probs[torch.arange(len(samples)), samples] + return samples, sampled_probs + + def sample_tokens_batched(self, logits, temperature=1.0, top_k=0.0, top_p=0.0): + if logits.ndim > 2: + B, N = logits.shape[0], logits.shape[1] + logits = rearrange(logits, 'b n v -> (b n) v') + samples, sampled_probs = self.sample_tokens(logits, temperature, top_k, top_p) + samples = rearrange(samples, '(b n) -> b n', b=B, n=N) + sampled_probs = rearrange(sampled_probs, '(b n) -> b n', b=B, n=N) + return samples, sampled_probs + else: + return self.sample_tokens(logits, temperature, top_k, top_p) + + def select_tokens(self, logits, num_select, temperature=1.0, top_k=0.0, top_p=0.0, return_all_samples=False): + samples, sampled_probs = self.sample_tokens(logits, temperature, top_k, top_p) + top_indices = torch.topk(sampled_probs, num_select)[1] + top_samples = samples[top_indices] + if return_all_samples: + return top_samples, top_indices, samples + else: + return top_samples, top_indices + + def select_tokens_batched(self, logits, num_select, temperature=1.0, top_k=0.0, top_p=0.0, return_all_samples=False): + if logits.ndim > 2: + samples, sampled_probs = self.sample_tokens_batched(logits, temperature, top_k, top_p) # both of shape (B, N) + top_indices = torch.topk(sampled_probs, num_select, dim=-1)[1] + # Need to switch to gather instead of indexing here + top_samples = torch.gather(samples, dim=-1, index=top_indices) + if return_all_samples: + return top_samples, top_indices, samples + else: + return top_samples, top_indices + else: + return self.sample_tokens(logits, num_select, temperature, top_k, top_p, return_all_samples) + + + def forward_mask_encoder_generation(self, encoder_mod_dict): + """Modification of forward_mask_encoder adapted for generation, with support for batching + """ + # Form input + B = list(encoder_mod_dict.values())[0]['tensor'].shape[0] + + encoder_tokens_all, emb_all, encoder_mask_all, mod_mask_all = self.model.cat_encoder_tensors(encoder_mod_dict) + # Take max num encoder of tokens (although assuming it's the same everywhere would be better) + num_encoder_tokens = (~encoder_mask_all.reshape(B, -1)).sum(dim=1).max() + + # Add arange multiplied by small constant to mask so they get sorted in a deterministic way + mask_arange = torch.arange(encoder_mask_all.shape[1], device=encoder_mask_all.device).unsqueeze(0) * 1e-6 + ids_shuffle = torch.argsort(encoder_mask_all + mask_arange, dim=1) + # ids_restore = torch.argsort(ids_shuffle, dim=1) + ids_keep = ids_shuffle[:, :num_encoder_tokens] + + encoder_tokens = torch.gather(encoder_tokens_all, dim=1, + index=repeat(ids_keep, "b n -> b n d", d=encoder_tokens_all.shape[2])) + encoder_emb = torch.gather(emb_all, dim=1, index=repeat(ids_keep, "b n -> b n d", d=emb_all.shape[2])) + encoder_mask = torch.gather(encoder_mask_all, dim=1, index=ids_keep) + mod_mask = torch.gather(mod_mask_all, dim=1, index=ids_keep) + + if self.model.num_register_tokens > 0: + prompt_tokens = repeat(self.prompt_tokens, '() n d -> b n d', b=B) + # We add prompt tokens at the beginning of the sequence + encoder_tokens = torch.cat([prompt_tokens, encoder_tokens], dim=1) + encoder_emb = torch.cat([torch.zeros_like(prompt_tokens), encoder_emb], dim=1) + encoder_mask = torch.cat([torch.zeros((B, prompt_tokens.shape[1]), dtype=torch.bool, device=encoder_mask.device), encoder_mask], dim=1) + mod_mask = torch.cat([torch.full((B, prompt_tokens.shape[1]), -1, dtype=torch.int16, device=mod_mask.device), mod_mask], dim=1) + + encoder_tokens[encoder_mask] = 0. + encoder_emb[encoder_mask] = 0. + mod_mask[encoder_mask] = -1 + # Mask could be of shape 'b n1 n2' but not needed for masked_fill + # This means this mask can then be re-used for decoder cross-attention + encoder_mask = rearrange(encoder_mask, 'b n2 -> b 1 n2') + + return encoder_tokens, encoder_emb, encoder_mask, mod_mask + + + def forward_mask_decoder_maskgit(self, mod_dict, target_mod, seed=None): + """Modification of forward_mask_decoder for MaskGIT generation, with support for batching + """ + if seed is not None: + torch.manual_seed(seed) + d = mod_dict[target_mod] + decoder_tokens_all = torch.zeros_like(d['x']) + self.model.mask_token + emb_all = d['emb'] + decoder_mask_all = d['target_mask'] + B = decoder_tokens_all.shape[0] # Get batch size + mod_mask_all = torch.full_like(d['ids'], self.model.modality_info[target_mod]['id'], dtype=torch.int16) + mod_pos_all = torch.arange(d['x'].shape[1], device=d['x'].device).unsqueeze(0) + mod_pos_all = repeat(mod_pos_all, '1 n -> b n', b=B) # Added: Expansion for batching + num_decoder_tokens = (~decoder_mask_all[0]).sum() # Adapted for batching / Assumes num_decoder_tokens is the same across the batch + + + # Add arange multiplied by small constant to mask so they get sorted in a deterministic way + mask_arange = torch.arange(decoder_mask_all.shape[1], device=decoder_mask_all.device).unsqueeze(0) * 1e-6 + ids_shuffle = torch.argsort(decoder_mask_all + mask_arange, dim=1) + # ids_restore = torch.argsort(ids_shuffle, dim=1) + ids_keep = ids_shuffle[:, :num_decoder_tokens] + + decoder_tokens = torch.gather(decoder_tokens_all, dim=1, index=repeat(ids_keep, "b n -> b n d", d=decoder_tokens_all.shape[2])) + decoder_emb = torch.gather(emb_all, dim=1, index=repeat(ids_keep, "b n -> b n d", d=emb_all.shape[2])) + decoder_mask = torch.gather(decoder_mask_all, dim=1, index=ids_keep) + mod_mask = torch.gather(mod_mask_all, dim=1, index=ids_keep) + mod_pos = torch.gather(mod_pos_all, dim=1, index=ids_keep) + + decoder_tokens[decoder_mask] = 0. + decoder_emb[decoder_mask] = 0. + mod_mask[decoder_mask] = -1 + + return decoder_tokens, decoder_emb, decoder_mask, mod_mask, mod_pos + + def forward_mask_decoder_roar(self, mod_dict, target_mod, num_select, seed=None): + """Modification of forward_mask_decoder for ROAR generation, with support for batching + """ + if seed is not None: + torch.manual_seed(seed) + d = mod_dict[target_mod] + decoder_tokens_all = torch.zeros_like(d['x']) + self.model.mask_token + emb_all = d['emb'] + decoder_mask_all = d['target_mask'] + B = decoder_tokens_all.shape[0] # Get batch size + mod_mask_all = torch.full_like(d['ids'], self.model.modality_info[target_mod]['id'], dtype=torch.int16) + mod_pos_all = torch.arange(d['x'].shape[1], device=d['x'].device).unsqueeze(0) + mod_pos_all = repeat(mod_pos_all, '1 n -> b n', b=B) # Added: Expansion for batching + # Only keep the first num_select tokens + num_decoder_tokens = min(num_select, (~decoder_mask_all[0]).sum()) # Adapted for batching / Assumes num_decoder_tokens is the same across the batch + + # Add a small random number to the mask so they get sorted in a random way, but keeping the masked tokens first + mask_rand = torch.rand(decoder_mask_all.shape[1], device=decoder_mask_all.device).unsqueeze(0) * 1e-6 + ids_shuffle = torch.argsort(decoder_mask_all + mask_rand, dim=1) + # ids_restore = torch.argsort(ids_shuffle, dim=1) + # Only keep the first num_select_tokens + ids_keep = ids_shuffle[:, :num_decoder_tokens] + + decoder_tokens = torch.gather(decoder_tokens_all, dim=1, index=repeat(ids_keep, "b n -> b n d", d=decoder_tokens_all.shape[2])) + decoder_emb = torch.gather(emb_all, dim=1, index=repeat(ids_keep, "b n -> b n d", d=emb_all.shape[2])) + decoder_mask = torch.gather(decoder_mask_all, dim=1, index=ids_keep) + mod_mask = torch.gather(mod_mask_all, dim=1, index=ids_keep) + mod_pos = torch.gather(mod_pos_all, dim=1, index=ids_keep) + + decoder_tokens[decoder_mask] = 0. + decoder_emb[decoder_mask] = 0. + mod_mask[decoder_mask] = -1 + + return decoder_tokens, decoder_emb, decoder_mask, mod_mask, mod_pos + + def forward_mask_decoder_autoregressive(self, mod_dict, target_mod, seed=None): + # Adapted for batching + if seed is not None: + torch.manual_seed(seed) + # This is the concatenation part + d = mod_dict[target_mod] + decoder_ids_all = d['ids'] + emb_all = d['emb'] + decoder_mask_all = d['target_mask'] + B = decoder_ids_all.shape[0] # Get batch size + mod_mask_all = torch.full_like(d['ids'], self.model.modality_info[target_mod]['id'], dtype=torch.int16) + mod_pos_all = torch.arange(d['x'].shape[1], device=d['x'].device).unsqueeze(0) + mod_pos_all = repeat(mod_pos_all, '1 n -> b n', b=B) + num_decoder_tokens = (~decoder_mask_all[0]).sum() # Adapted for batching, but assumes num_decoder_tokens is the same across the batch + + # Add arange multiplied by small constant to mask so they get sorted in a deterministic way + mask_arange = torch.arange(decoder_mask_all.shape[1], device=decoder_mask_all.device).unsqueeze(0) * 1e-6 + ids_shuffle = torch.argsort(decoder_mask_all + mask_arange, dim=1) + # ids_restore = torch.argsort(ids_shuffle, dim=1) + ids_keep = ids_shuffle[:, :num_decoder_tokens] + + # Same as in forward_mask_decoder + decoder_ids = torch.gather(decoder_ids_all, dim=1, index=ids_keep) + decoder_emb = torch.gather(emb_all, dim=1, index=repeat(ids_keep, "b n -> b n d", d=emb_all.shape[2])) + decoder_mask = torch.gather(decoder_mask_all, dim=1, index=ids_keep) + mod_mask = torch.gather(mod_mask_all, dim=1, index=ids_keep) + mod_pos = torch.gather(mod_pos_all, dim=1, index=ids_keep) + + decoder_ids[decoder_mask] = 0 + decoder_emb[decoder_mask] = 0. + mod_mask[decoder_mask] = -1 + + return decoder_ids, decoder_emb, decoder_mask, mod_mask, mod_pos + + def merge_sequences(self, mod_dict, pred_ids, target_mod, text_tokenizer, default_sentinel="[S_1]"): + device = mod_dict[target_mod]['tensor'].device + # Get input ids + input_ids = mod_dict[target_mod]['tensor'].squeeze().detach().cpu() + input_ids = input_ids[mod_dict[target_mod]['input_mask'].squeeze().detach().cpu() == 0] + input_ids = input_ids.tolist() + + if len(input_ids) == 0: + input_ids = [text_tokenizer.get_vocab()[default_sentinel]] + + # Get predicted ids + pred_ids = pred_ids.squeeze().detach().cpu().tolist() + if isinstance(pred_ids, int): + pred_ids = [pred_ids] + + # Get sentinel ids using the tokenizer + sentinel_ids = set(get_sentinel_to_id_mapping(text_tokenizer).values()) + # Perform merging + merged_ids = merge_span_masking(input_ids, pred_ids, sentinel_ids) + merged_ids = torch.tensor(merged_ids).unsqueeze(0) + # Create new dict + new_input_mask = torch.zeros_like(merged_ids, dtype=torch.bool) + new_target_mask = torch.ones_like(merged_ids, dtype=torch.bool) + new_dict = {'tensor': merged_ids.to(device), + 'input_mask': new_input_mask.to(device), + 'target_mask': new_target_mask.to(device)} + new_dict['decoder_attention_mask'] = torch.zeros_like(new_target_mask, dtype=torch.bool) + + mod_dict[target_mod] = new_dict + return mod_dict + + def merge_sequences_batched(self, mod_dict, pred_ids, target_mod, text_tokenizer, default_sentinel="[S_1]"): + # Unbatches and calls merge sequence per batch, then regroups it into a batch + + pad_id = text_tokenizer.token_to_id("[PAD]") + + B = mod_dict[target_mod]['tensor'].shape[0] + device = mod_dict[target_mod]['tensor'].device + + tensors = torch.split(mod_dict[target_mod]['tensor'], 1) + input_masks = torch.split(mod_dict[target_mod]['input_mask'], 1) + pred_ids = torch.split(pred_ids, 1) + + input_dicts = [] + for t, im in zip(tensors, input_masks): + d = {target_mod: {'tensor': t, 'input_mask': im}} + input_dicts.append(d) + + merged_tensors = [] + merged_input_masks = [] + merged_target_masks = [] + merged_seq_lens = [] + for input_d, pi in zip(input_dicts, pred_ids): + # Output of merge_sequences is mod_dict with modified target mod + merged_d = self.merge_sequences(input_d, pi, target_mod, text_tokenizer, default_sentinel)[target_mod] + merged_tensors.append(merged_d['tensor']) + merged_input_masks.append(merged_d['input_mask']) + merged_target_masks.append(merged_d['input_mask']) + merged_seq_lens.append(merged_d['tensor'].shape[1]) + + + max_seq_len = max(merged_seq_lens) + + for i in range(len(merged_tensors)): + # Right pad all tensors + p1d = (0, max_seq_len - merged_seq_lens[i]) + merged_tensors[i] = F.pad(merged_tensors[i], p1d, "constant",pad_id) + merged_input_masks[i] = F.pad(merged_input_masks[i], p1d, "constant", True) + merged_target_masks[i] = F.pad(merged_target_masks[i], p1d, "constant", True) + + new_dict = {'tensor': torch.cat(merged_tensors, dim=0).to(device), + 'input_mask': torch.cat(merged_input_masks, dim=0).to(device), + 'target_mask': torch.cat(merged_target_masks, dim=0).to(device)} + new_dict['decoder_attention_mask'] = torch.zeros_like(new_dict['target_mask'], dtype=torch.bool) + + mod_dict[target_mod] = new_dict + return mod_dict + + def forward_enc_dec_maskgit_batched(self, mod_dict, target_mod, seed=None): + # Encoder + encoder_mod_dict = {mod: self.model.encoder_embeddings[mod](d) + for mod, d in mod_dict.items() + if mod in self.model.encoder_embeddings} + encoder_tokens, encoder_emb, encoder_mask, encoder_mod_mask = self.forward_mask_encoder_generation(encoder_mod_dict) + x = encoder_tokens + encoder_emb + x = self.model.forward_encoder(x, encoder_mask) + + # Decoder + context = self.model.decoder_proj_context(x) + encoder_emb + decoder_mod_dict = {target_mod: self.model.decoder_embeddings[target_mod].forward_embed(mod_dict[target_mod])} + decoder_tokens, decoder_emb, decoder_mask, decoder_mod_mask, mod_pos = self.forward_mask_decoder_maskgit(decoder_mod_dict, target_mod, seed=seed) + y = decoder_tokens + decoder_emb + y = self.model.forward_decoder(y, context, encoder_mask, None) + B, N, D = y.shape + logits = self.model.forward_logits(y, decoder_mod_dict, decoder_mod_mask)[target_mod] + logits = logits.reshape(B, N, -1) + + + return logits, mod_pos + + def maskgit_step_batched(self, mod_dict, target_mod, num_select, temperature, top_k, top_p, seed=None): + logits, mod_pos = self.forward_enc_dec_maskgit_batched(mod_dict, target_mod, seed=seed) + + # MaskGIT sampling + top_samples, top_indices = self.select_tokens_batched(logits, num_select, + temperature=temperature, top_k=top_k, top_p=top_p) + # Update mod dict + # We rely on gather / scatter for batched operations + top_pos = torch.gather(mod_pos, -1, top_indices) # (B, num_select) + mod_dict[target_mod]['tensor'] = torch.scatter(mod_dict[target_mod]['tensor'], -1, top_pos, top_samples) + mod_dict[target_mod]['input_mask'] = torch.scatter(mod_dict[target_mod]['input_mask'], -1, top_pos, torch.zeros_like(top_samples, dtype=torch.bool)) + mod_dict[target_mod]['target_mask'] = torch.scatter(mod_dict[target_mod]['target_mask'], -1, top_pos, torch.ones_like(top_samples, dtype=torch.bool)) + + return mod_dict + + def guided_maskgit_step_batched(self, mod_dict, target_mod, num_select, temperature, top_k, top_p, + conditioning=[], guidance_scale=1.0, seed=None, write_all_predictions=False): + + ### 1 - First pass, with conditioning + logits_cond, _ = self.forward_enc_dec_maskgit_batched(mod_dict, target_mod, seed=seed) + + ### 2 - Second pass, without conditioning + mod_dict_uncond = copy.deepcopy(mod_dict) + for mod in conditioning: + if self.model.modality_info[mod]['type'] in ['seq', 'seq_token']: + mod_dict_uncond = empty_seq_modality(mod_dict_uncond, mod) + elif self.model.modality_info[mod]['type'] in ['seq_emb']: + mod_dict_uncond = empty_seq_emb_modality(mod_dict_uncond, mod) + else: + mod_dict_uncond = empty_img_modality(mod_dict_uncond, mod) + + logits_uncond, mod_pos = self.forward_enc_dec_maskgit_batched(mod_dict_uncond, target_mod, seed=seed) + + ### 3 - Classifier-free guidance + logits = logits_uncond + (logits_cond - logits_uncond) * guidance_scale + + ### 4 - MaskGIT sampling + top_samples, top_indices, all_samples = self.select_tokens_batched( + logits, num_select, + temperature=temperature, top_k=top_k, top_p=top_p, + return_all_samples=True + ) + + ### 5 - Update mod dict + # We rely on gather / scatter for batched operations + top_pos = torch.gather(mod_pos, -1, top_indices) # (B, num_select) + if write_all_predictions: + mod_dict[target_mod]['tensor'][:, mod_pos] = all_samples + else: + mod_dict[target_mod]['tensor'] = torch.scatter(mod_dict[target_mod]['tensor'], -1, top_pos, top_samples) + mod_dict[target_mod]['input_mask'] = torch.scatter(mod_dict[target_mod]['input_mask'], -1, top_pos, torch.zeros_like(top_samples, dtype=torch.bool)) + mod_dict[target_mod]['target_mask'] = torch.scatter(mod_dict[target_mod]['target_mask'], -1, top_pos, torch.ones_like(top_samples, dtype=torch.bool)) + + return mod_dict + + def multi_guided_maskgit_step_batched(self, uncond_dict, cond_dicts, cond_weights, target_mod, num_select, + temperature, top_k, top_p, seed=None, write_all_predictions=False): + + ### 1 - Conditional forward passes (one for each guided condition) + logits_cond_all = [] + for cond_dict in cond_dicts: + logits_cond_i, _ = self.forward_enc_dec_maskgit_batched(cond_dict, target_mod, seed=seed) + logits_cond_all.append(logits_cond_i) + + ### 2 - Unconditional forward pass + logits_uncond, mod_pos = self.forward_enc_dec_maskgit_batched(uncond_dict, target_mod, seed=seed) + + ### 3 Conjunction of multiple conditions: l_uncond + sum_i{w_i * (l_cond_i - l_uncond)} + # See https://arxiv.org/abs/2206.01714 + logits = logits_uncond + torch.stack([w * (logits_cond - logits_uncond) for w, logits_cond in zip(cond_weights, logits_cond_all)]).sum(dim=0) + + ### 4 - MaskGIT sampling + top_samples, top_indices, all_samples = self.select_tokens_batched( + logits, num_select, + temperature=temperature, top_k=top_k, top_p=top_p, + return_all_samples=True + ) + + ### 5 - Update mod dict with newly generated tokens + # We rely on gather / scatter for batched operations + top_pos = torch.gather(mod_pos, -1, top_indices) # (B, num_select) + if write_all_predictions: + uncond_dict[target_mod]['tensor'][:, mod_pos] = all_samples + else: + uncond_dict[target_mod]['tensor'] = torch.scatter(uncond_dict[target_mod]['tensor'], -1, top_pos, top_samples) + uncond_dict[target_mod]['input_mask'] = torch.scatter(uncond_dict[target_mod]['input_mask'], -1, top_pos, torch.zeros_like(top_samples, dtype=torch.bool)) + uncond_dict[target_mod]['target_mask'] = torch.scatter(uncond_dict[target_mod]['target_mask'], -1, top_pos, torch.ones_like(top_samples, dtype=torch.bool)) + # Update conditioning dicts + for i in range(len(cond_dicts)): + cond_dicts[i][target_mod]['tensor'] = torch.scatter(cond_dicts[i][target_mod]['tensor'], -1, top_pos, top_samples) + cond_dicts[i][target_mod]['input_mask'] = torch.scatter(cond_dicts[i][target_mod]['input_mask'], -1, top_pos, torch.zeros_like(top_samples, dtype=torch.bool)) + cond_dicts[i][target_mod]['target_mask'] = torch.scatter(cond_dicts[i][target_mod]['target_mask'], -1, top_pos, torch.ones_like(top_samples, dtype=torch.bool)) + + return uncond_dict, cond_dicts + + def forward_enc_dec_roar_batched(self, mod_dict, target_mod, num_select, seed=None): + # Encoder + encoder_mod_dict = {mod: self.model.encoder_embeddings[mod](d) + for mod, d in mod_dict.items() + if mod in self.model.encoder_embeddings} + encoder_tokens, encoder_emb, encoder_mask, encoder_mod_mask = self.forward_mask_encoder_generation(encoder_mod_dict) + x = encoder_tokens + encoder_emb + x = self.model.forward_encoder(x, encoder_mask) + + # Decoder + context = self.model.decoder_proj_context(x) + encoder_emb + decoder_mod_dict = {target_mod: self.model.decoder_embeddings[target_mod].forward_embed(mod_dict[target_mod])} + decoder_tokens, decoder_emb, decoder_mask, decoder_mod_mask, mod_pos = self.forward_mask_decoder_roar(decoder_mod_dict, target_mod, num_select, seed=seed) + y = decoder_tokens + decoder_emb + y = self.model.forward_decoder(y, context, encoder_mask, None) + B, N, D = y.shape + logits = self.model.forward_logits(y, decoder_mod_dict, decoder_mod_mask)[target_mod] + logits = logits.reshape(B, N, -1) + + return logits, mod_pos + + def roar_step_batched(self, mod_dict, target_mod, num_select, temperature, top_k, top_p, seed=None): + """ROAR = Random Order Autoregression""" + + logits, mod_pos = self.forward_enc_dec_roar_batched(mod_dict, target_mod, num_select, seed=seed) + + # Simple sampling + samples, sampled_probs = self.sample_tokens_batched(logits, temperature, top_k=top_k, top_p=top_p) + + # Update mod dict + # We rely on scatter for batched operations + select_pos = mod_pos + mod_dict[target_mod]['tensor'] = torch.scatter(mod_dict[target_mod]['tensor'], -1, select_pos, samples) + mod_dict[target_mod]['input_mask'] = torch.scatter(mod_dict[target_mod]['input_mask'], -1, select_pos, torch.zeros_like(samples, dtype=torch.bool)) + mod_dict[target_mod]['target_mask'] = torch.scatter(mod_dict[target_mod]['target_mask'], -1, select_pos, torch.ones_like(samples, dtype=torch.bool)) + + return mod_dict + + def guided_roar_step_batched(self, mod_dict, target_mod, num_select, temperature, top_k, top_p, + conditioning=[], guidance_scale=1.0, seed=None): + """ROAR = Random Order Autoregression""" + + ### 1 - First pass, with conditioning + logits_cond, _ = self.forward_enc_dec_roar_batched(mod_dict, target_mod, num_select, seed=seed) + + ### 2 - Second pass, without conditioning + mod_dict_uncond = copy.deepcopy(mod_dict) + for mod in conditioning: + if self.model.modality_info[mod]['type'] in ['seq', 'seq_token']: + mod_dict_uncond = empty_seq_modality(mod_dict_uncond, mod) + elif self.model.modality_info[mod]['type'] in ['seq_emb']: + mod_dict_uncond = empty_seq_emb_modality(mod_dict_uncond, mod) + else: + mod_dict_uncond = empty_img_modality(mod_dict_uncond, mod) + + logits_uncond, mod_pos = self.forward_enc_dec_roar_batched(mod_dict_uncond, target_mod, num_select, seed=seed) + + ### 3 - Classifier-free guidance + logits = logits_uncond + (logits_cond - logits_uncond) * guidance_scale + + ### 4 - Simple sampling + samples, sampled_probs = self.sample_tokens_batched(logits, temperature, top_k=top_k, top_p=top_p) + + ### 5 - Update mod dict + # We rely on gather / scatter for batched operations + select_pos = mod_pos + mod_dict[target_mod]['tensor'] = torch.scatter(mod_dict[target_mod]['tensor'], -1, select_pos, samples) + mod_dict[target_mod]['input_mask'] = torch.scatter(mod_dict[target_mod]['input_mask'], -1, select_pos, torch.zeros_like(samples, dtype=torch.bool)) + mod_dict[target_mod]['target_mask'] = torch.scatter(mod_dict[target_mod]['target_mask'], -1, select_pos, torch.ones_like(samples, dtype=torch.bool)) + + return mod_dict + + def multi_guided_roar_step_batched(self, uncond_dict, cond_dicts, cond_weights, target_mod, + num_select, temperature, top_k, top_p, seed=None): + + ### 1 - Conditional forward passes (one for each guided condition) + logits_cond_all = [] + for cond_dict in cond_dicts: + logits_cond_i, _ = self.forward_enc_dec_roar_batched(cond_dict, target_mod, num_select, seed=seed) + logits_cond_all.append(logits_cond_i) + + ### 2 - Unconditional forward pass + logits_uncond, mod_pos = self.forward_enc_dec_roar_batched(uncond_dict, target_mod, num_select, seed=seed) + + ### 3 Conjunction of multiple conditions: l_uncond + sum_i{w_i * (l_cond_i - l_uncond)} + # See https://arxiv.org/abs/2206.01714 + logits = logits_uncond + torch.stack([w * (logits_cond - logits_uncond) for w, logits_cond in zip(cond_weights, logits_cond_all)]).sum(dim=0) + + ### 4 - Simple sampling + samples, sampled_probs = self.sample_tokens_batched(logits, temperature, top_k=top_k, top_p=top_p) + + ### 5 - Update mod dict + # We rely on gather / scatter for batched operations + select_pos = mod_pos + uncond_dict[target_mod]['tensor'] = torch.scatter(uncond_dict[target_mod]['tensor'], -1, select_pos, samples) + uncond_dict[target_mod]['input_mask'] = torch.scatter(uncond_dict[target_mod]['input_mask'], -1, select_pos, torch.zeros_like(samples, dtype=torch.bool)) + uncond_dict[target_mod]['target_mask'] = torch.scatter(uncond_dict[target_mod]['target_mask'], -1, select_pos, torch.ones_like(samples, dtype=torch.bool)) + # Update conditioning dicts + for i in range(len(cond_dicts)): + cond_dicts[i][target_mod]['tensor'] = torch.scatter(cond_dicts[i][target_mod]['tensor'], -1, select_pos, samples) + cond_dicts[i][target_mod]['input_mask'] = torch.scatter(cond_dicts[i][target_mod]['input_mask'], -1, select_pos, torch.ones_like(samples, dtype=torch.bool)) + cond_dicts[i][target_mod]['target_mask'] = torch.scatter(cond_dicts[i][target_mod]['target_mask'], -1, select_pos, torch.zeros_like(samples, dtype=torch.bool)) + + return uncond_dict, cond_dicts + + def autoregressive_step_batched(self, mod_dict, target_mod, temperature, top_k: Union[float, int], top_p: float, + use_eos=True, eos_token=None, start_tokens=None, text_tokenizer=None, seed=None): + + # Encoder + encoder_mod_dict = {mod: self.model.encoder_embeddings[mod](d) + for mod, d in mod_dict.items() + if mod in self.model.encoder_embeddings} + encoder_tokens, encoder_emb, encoder_mask, encoder_mod_mask = self.forward_mask_encoder_generation(encoder_mod_dict) + x = encoder_tokens + encoder_emb + x = self.model.forward_encoder(x, encoder_mask) # B, N, D + + # Get batch size + B = x.shape[0] + + # Decoder + context = self.model.decoder_proj_context(x) + encoder_emb + decoder_mod_dict = {target_mod: self.model.decoder_embeddings[target_mod].forward_embed(mod_dict[target_mod])} + decoder_ids, decoder_emb, decoder_mask, decoder_mod_mask, mod_pos = self.forward_mask_decoder_autoregressive(decoder_mod_dict, target_mod, seed=seed) + device = decoder_ids.device + seq_len = self.model.modality_info[target_mod]['max_tokens'] + + if use_eos and eos_token is None: + # The eos_token is the final sentinel token provided + eos_token = decoder_ids[0][decoder_mask[0] == 0][-1] # Assumes the EOS token is the same for all + if use_eos: + eos_token = eos_token.to(device) + + # If no start_tokens, just use the beginning of the actual target (i.e., a sentinel token) + out = decoder_ids[:, :1] if start_tokens is None else start_tokens.to(device) + # Set decoder_tokens to None, we do not use them for decoding + decoder_ids = None + + # If all samples of the batch have eos, return early + if use_eos and (out == eos_token).any(dim=-1).all(): + return out + + y_emb = decoder_emb[:, :seq_len] + seq_len = y_emb.shape[1] + + # Auto-regressive decoding and sampling + for i in range(seq_len): + cur_len = out.shape[1] + # Convert ids into word embeddings and add corresponding posembs + modemb + y = self.model.decoder_embeddings[target_mod].token_emb(out) + y_emb[:, :cur_len] + # Build causal mask + causal_mask = torch.ones((cur_len, cur_len), dtype=torch.bool, device=y.device).triu(1) + causal_mask = repeat(causal_mask, "n1 n2 -> b n1 n2", b=B) + + y = self.model.forward_decoder(y, context, encoder_mask, causal_mask) + logits = self.model.forward_logits(y, decoder_mod_dict, decoder_mod_mask[:, :cur_len])[target_mod] + logits = rearrange(logits, "(b n) d -> b n d", b=B, n=cur_len) + last_logits = logits[:, -1] + + # Sample token for the newly generated logit + if np.isclose(temperature, 0, atol=1e-10): + sample = torch.argmax(last_logits, dim=-1, keepdim=True) + else: + filtered_logits = self.top_k_top_p_filtering(last_logits, top_k, top_p) + probs = F.softmax(filtered_logits / temperature, dim=-1) + sample = torch.multinomial(probs, 1) + out = torch.cat((out, sample), dim=-1) + + if use_eos and (out == eos_token).any(dim=-1).all(): + break + + mod_dict = self.merge_sequences_batched(mod_dict, out, target_mod, text_tokenizer) + + return mod_dict + + def guided_autoregressive_step_batched(self, mod_dict, target_mod, temperature, top_k: Union[float, int], top_p: float, + use_eos=True, eos_token=None, start_tokens=None, text_tokenizer=None, + conditioning=[], guidance_scale=1.0, seed=None): + + ### 1 - Encoder forward pass, with conditioning + + # Encoder + encoder_mod_dict = {mod: self.model.encoder_embeddings[mod](d) + for mod, d in mod_dict.items() + if mod in self.model.encoder_embeddings} + encoder_tokens, encoder_emb, encoder_mask_cond, encoder_mod_mask = self.forward_mask_encoder_generation(encoder_mod_dict) + x = encoder_tokens + encoder_emb + x = self.model.forward_encoder(x, encoder_mask_cond) # B, N, D + + # Get batch size + B = x.shape[0] + + # Decoder + context_cond = self.model.decoder_proj_context(x) + encoder_emb + decoder_mod_dict_cond = {target_mod: self.model.decoder_embeddings[target_mod].forward_embed(mod_dict[target_mod])} + decoder_ids, decoder_emb, decoder_mask, decoder_mod_mask_cond, mod_pos = self.forward_mask_decoder_autoregressive(decoder_mod_dict_cond, target_mod, seed=seed) + device = decoder_ids.device + seq_len = self.model.modality_info[target_mod]['max_tokens'] + + + ### 2 - Encoder forward pass, without conditioning + + mod_dict_uncond = copy.deepcopy(mod_dict) + for mod in conditioning: + if self.model.modality_info[mod]['type'] in ['seq', 'seq_token']: + mod_dict_uncond = empty_seq_modality(mod_dict_uncond, mod) + elif self.model.modality_info[mod]['type'] in ['seq_emb']: + mod_dict_uncond = empty_seq_emb_modality(mod_dict_uncond, mod) + else: + mod_dict_uncond = empty_img_modality(mod_dict_uncond, mod) + + # Encoder + encoder_mod_dict = {mod: self.model.encoder_embeddings[mod](d) + for mod, d in mod_dict_uncond.items() + if mod in self.model.encoder_embeddings} + encoder_tokens, encoder_emb, encoder_mask_uncond, encoder_mod_mask = self.forward_mask_encoder_generation(encoder_mod_dict) + x = encoder_tokens + encoder_emb + x = self.model.forward_encoder(x, encoder_mask_uncond) # B, N, D + + # Decoder + context_uncond = self.model.decoder_proj_context(x) + encoder_emb + decoder_mod_dict_uncond = {target_mod: self.model.decoder_embeddings[target_mod].forward_embed(mod_dict[target_mod])} + decoder_ids, decoder_emb, decoder_mask, decoder_mod_mask_uncond, mod_pos = self.forward_mask_decoder_autoregressive(decoder_mod_dict_uncond, target_mod, seed=seed) + + + if use_eos and eos_token is None: + # The eos_token is the final sentinel token provided + eos_token = decoder_ids[0][decoder_mask[0] == 0][-1] # Assumes the EOS token is the same for all + if use_eos: + eos_token = eos_token.to(device) + + # If no start_tokens, just use the beginning of the actual target (i.e., a sentinel token) + out = decoder_ids[:, :1] if start_tokens is None else start_tokens.to(device) + # Set decoder_tokens to None, we do not use them for decoding + decoder_ids = None + + # If all samples of the batch have eos, return early + if use_eos and (out == eos_token).any(dim=-1).all(): + return out + + y_emb = decoder_emb[:, :seq_len] + seq_len = y_emb.shape[1] + + ### 3 - Auto-regressive decoding and sampling + for i in range(seq_len): + cur_len = out.shape[1] + # Convert ids into word embeddings and add corresponding posembs + modemb + y = self.model.decoder_embeddings[target_mod].token_emb(out) + y_emb[:, :cur_len] + # Build causal mask + causal_mask = torch.ones((cur_len, cur_len), dtype=torch.bool, device=y.device).triu(1) + causal_mask = repeat(causal_mask, "n1 n2 -> b n1 n2", b=B) + + ### 3a - Decoder forward pass, with conditioning + y_cond = self.model.forward_decoder(y, context_cond, encoder_mask_cond, causal_mask) + logits_cond = self.model.forward_logits(y_cond, decoder_mod_dict_cond, decoder_mod_mask_cond[:, :cur_len])[target_mod] + logits_cond = rearrange(logits_cond, "(b n) d -> b n d", b=B, n=cur_len) + last_logits_cond = logits_cond[:, -1] + + ### 3b - Decoder forward pass, without conditioning + y_uncond = self.model.forward_decoder(y, context_uncond, encoder_mask_uncond, causal_mask) + logits_uncond = self.model.forward_logits(y_uncond, decoder_mod_dict_uncond, decoder_mod_mask_uncond[:, :cur_len])[target_mod] + logits_uncond = rearrange(logits_uncond, "(b n) d -> b n d", b=B, n=cur_len) + last_logits_uncond = logits_uncond[:, -1] + + ### 3c - Classifier-free guidance + last_logits = last_logits_uncond + (last_logits_cond - last_logits_uncond) * guidance_scale + + # Sample token for the newly generated logit + if np.isclose(temperature, 0, atol=1e-10): + sample = torch.argmax(last_logits, dim=-1, keepdim=True) + else: + filtered_logits = self.top_k_top_p_filtering(last_logits, top_k, top_p) + probs = F.softmax(filtered_logits / temperature, dim=-1) + sample = torch.multinomial(probs, 1) + out = torch.cat((out, sample), dim=-1) + + if use_eos and (out == eos_token).any(dim=-1).all(): + break + + mod_dict = self.merge_sequences_batched(mod_dict, out, target_mod, text_tokenizer) + + return mod_dict + + + @torch.no_grad() + def generate(self, mod_dict, schedule, top_k=0.0, top_p=0.0, text_tokenizer=None, verbose=False, seed=None): + """ Generates a sequence of tokens from the input modalities. + :param mod_dict: Dictionary of modalities. + :param schedule: Schedule of modalities to use. + List of dictionaries containing {target_domain, scheme, num_tokens, temperature, cfg_scale, cfg_cond_domains}. + :param top_k: top_k > 0: Keep only top k tokens with highest probability (a.k.a. top-k filtering). + :param top_p: top_p > 0.0: Keep the top tokens with cumulative probability >= top_p (a.k.a. nucleus filtering). + :param text_tokenizer: Text tokenizer. + :param verbose: Whether to print progress. + :param seed: Random seed. + :return: Generated mod dict. + """ + + # Input embedding -> tokenizes the modalities - Many are placeholder for now + mod_dict = copy.deepcopy(mod_dict) + + for step, schedule_step_info in tqdm(enumerate(schedule), disable=not verbose): + target_mod = schedule_step_info['target_domain'] + temp = schedule_step_info['temperature'] + cfg_scale = schedule_step_info.get('cfg_scale', 1.0) + cfg_conditioning = schedule_step_info.get('cfg_cond_domains', []) + seed_i = seed + step if seed is not None else None + + if self.model.modality_info[target_mod]['type'] == 'img': + scheme = schedule_step_info['scheme'] + num_select = schedule_step_info['num_tokens'] + + if scheme.lower() == 'maskgit': + if cfg_scale == 1.0 or len(cfg_conditioning) == 0: + mod_dict = self.maskgit_step_batched( + mod_dict, target_mod, num_select, temperature=temp, + top_k=top_k, top_p=top_p, seed=seed_i + ) + else: + mod_dict = self.guided_maskgit_step_batched( + mod_dict, target_mod, num_select, temperature=temp, top_k=top_k, top_p=top_p, + conditioning=cfg_conditioning, guidance_scale=cfg_scale, seed=seed_i + ) + elif scheme.lower() == 'roar': + if cfg_scale == 1.0 or len(cfg_conditioning) == 0: + mod_dict = self.roar_step_batched( + mod_dict, target_mod, num_select, temperature=temp, + top_k=top_k, top_p=top_p, seed=seed_i + ) + else: + mod_dict = self.guided_roar_step_batched( + mod_dict, target_mod, num_select, temperature=temp, top_k=top_k, top_p=top_p, + conditioning=cfg_conditioning, guidance_scale=cfg_scale, seed=seed_i + ) + else: + raise ValueError("Invalid sampling scheme") + elif self.model.modality_info[target_mod]['type'] in ['seq', 'seq_token']: + if cfg_scale == 1.0 or len(cfg_conditioning) == 0: + mod_dict = self.autoregressive_step_batched( + mod_dict, target_mod, temperature=temp, top_k=top_k, top_p=top_p, + text_tokenizer=text_tokenizer, seed=seed_i + ) + else: + mod_dict = self.guided_autoregressive_step_batched( + mod_dict, target_mod, temperature=temp, top_k=top_k, top_p=top_p, + text_tokenizer=text_tokenizer, conditioning=cfg_conditioning, + guidance_scale=cfg_scale, seed=seed_i + ) + else: + raise ValueError("Invalid schedule") + + return mod_dict + + + @torch.no_grad() + def generate_iter(self, mod_dict, schedule, top_k=0.0, top_p=0.0, text_tokenizer=None, verbose=False, seed=None): + """ Iterator that generates a sequence of tokens from the input modalities step by step. + :param mod_dict: Dictionary of modalities. + :param schedule: Schedule of modalities to use. + List of dictionaries containing {target_domain, scheme, num_tokens, temperature, cfg_scale, cfg_cond_domains}. + :param top_k: top_k > 0: Keep only top k tokens with highest probability (a.k.a. top-k filtering). + :param top_p: top_p > 0.0: Keep the top tokens with cumulative probability >= top_p (a.k.a. nucleus filtering). + :param text_tokenizer: Text tokenizer. + :param verbose: Whether to print progress. + :param seed: Random seed. + :return: Iterator of generated mod dict. + """ + + # Input embedding -> tokenizes the modalities - Many are placeholder for now + mod_dict = copy.deepcopy(mod_dict) + + for step, schedule_step_info in tqdm(enumerate(schedule), disable=not verbose): + target_mod = schedule_step_info['target_domain'] + temp = schedule_step_info['temperature'] + cfg_scale = schedule_step_info.get('cfg_scale', 1.0) + cfg_conditioning = schedule_step_info.get('cfg_cond_domains', []) + seed_i = seed + step if seed is not None else None + + if self.model.modality_info[target_mod]['type'] == 'img': + scheme = schedule_step_info['scheme'] + num_select = schedule_step_info['num_tokens'] + + if scheme.lower() == 'maskgit': + if cfg_scale == 1.0 or len(cfg_conditioning) == 0: + mod_dict = self.maskgit_step_batched( + mod_dict, target_mod, num_select, temperature=temp, + top_k=top_k, top_p=top_p, seed=seed_i + ) + else: + mod_dict = self.guided_maskgit_step_batched( + mod_dict, target_mod, num_select, temperature=temp, top_k=top_k, top_p=top_p, + conditioning=cfg_conditioning, guidance_scale=cfg_scale, seed=seed_i, + write_all_predictions=True + ) + elif scheme.lower() == 'roar': + if cfg_scale == 1.0 or len(cfg_conditioning) == 0: + mod_dict = self.roar_step_batched( + mod_dict, target_mod, num_select, temperature=temp, + top_k=top_k, top_p=top_p, seed=seed_i + ) + else: + mod_dict = self.guided_roar_step_batched( + mod_dict, target_mod, num_select, temperature=temp, top_k=top_k, top_p=top_p, + conditioning=cfg_conditioning, guidance_scale=cfg_scale, seed=seed_i + ) + else: + raise ValueError("Invalid sampling scheme") + elif self.model.modality_info[target_mod]['type'] in ['seq', 'seq_token']: + if cfg_scale == 1.0 or len(cfg_conditioning) == 0: + mod_dict = self.autoregressive_step_batched( + mod_dict, target_mod, temperature=temp, top_k=top_k, top_p=top_p, + text_tokenizer=text_tokenizer, seed=seed_i + ) + else: + mod_dict = self.guided_autoregressive_step_batched( + mod_dict, target_mod, temperature=temp, top_k=top_k, top_p=top_p, + text_tokenizer=text_tokenizer, conditioning=cfg_conditioning, + guidance_scale=cfg_scale, seed=seed_i + ) + else: + raise ValueError("Invalid schedule") + + yield mod_dict + + @torch.no_grad() + def generate_multi_guided(self, uncond_dict, cond_dicts, schedule, top_k=0.0, top_p=0.0, + text_tokenizer=None, verbose=False, seed=None): + # Generation function for multiple weighted conditions + + # To detect when a modality has finished generating, we keep track of the current target modality + cur_target_mod = schedule[0]['target_domain'] + + uncond_dict = copy.deepcopy(uncond_dict) + cond_dicts = copy.deepcopy(cond_dicts) + + # Add the to-be-generated modality to the conditional dicts + for i in range(len(cond_dicts)): + cond_dicts[i][cur_target_mod] = copy.deepcopy(uncond_dict[cur_target_mod]) + + for step, schedule_step_info in tqdm(enumerate(schedule), disable=not verbose): + target_mod = schedule_step_info['target_domain'] + temp = schedule_step_info['temperature'] + num_select = schedule_step_info['num_tokens'] + cond_weights = schedule_step_info['cfg_scale'] + + # Once a modality is fully generated, add it as a new condition + if cur_target_mod != target_mod: + for i in range(len(cond_dicts)): + # Remove the previously generated modality from the conditionings + del cond_dicts[i][cur_target_mod] + # Add the next modality to be generated to the conditionings + cond_dicts[i][target_mod] = copy.deepcopy(uncond_dict[target_mod]) + + # Remove the fully generated modality from the unconditional dict inputs + uncond_dict[cur_target_mod]['input_mask'][:] = True + + # Add the previously generated modality as an additional condition + new_cond = {} + new_cond[cur_target_mod] = copy.deepcopy(uncond_dict[cur_target_mod]) + new_cond[cur_target_mod]['input_mask'][:] = False + new_cond[cur_target_mod]['target_mask'][:] = True + new_cond[target_mod] = copy.deepcopy(uncond_dict[target_mod]) + cond_dicts.append(new_cond) + + cur_target_mod = target_mod + + if self.model.modality_info[target_mod]['type'] == 'img': + scheme = schedule_step_info['scheme'] + + if scheme.lower() == 'maskgit': + uncond_dict, cond_dicts = self.multi_guided_maskgit_step_batched( + uncond_dict, cond_dicts, cond_weights, target_mod, num_select, temp, top_k, top_p, seed=seed + ) + elif scheme.lower() == 'roar': + uncond_dict, cond_dicts = self.multi_guided_roar_step_batched( + uncond_dict, cond_dicts, cond_weights, target_mod, num_select, temp, top_k, top_p, seed=seed + ) + else: + raise ValueError("Invalid sampling scheme") + + else: + raise NotImplementedError("Only image modalities are supported for now") + + return uncond_dict + + @torch.no_grad() + def generate_sam_dense(self, mod_dict, schedule, text_tokenizer, batch_size=16, + key='sam_instance', top_k=0.0, top_p=0.0, seed=None, verbose=False): + # Generation function for dense SAM instance prediction + + device = mod_dict[list(mod_dict.keys())[0]]['tensor'].device + mod_dict = copy.deepcopy(mod_dict) + # Repeat the input batch to match the batch size + expanded_batch = expand_to_batch(copy.deepcopy(mod_dict), batch_size=batch_size) + + # Filter the schedule to only include the key domain + schedule = [s for s in schedule if s['target_domain'] == key] + + out_dict = self.generate( + expanded_batch, schedule, text_tokenizer=text_tokenizer, + verbose=verbose, seed=seed, + top_p=top_p, top_k=top_k, + ) + + # Merge the batch generated sequences into one sequence + sentinel_ids = set(get_sentinel_to_id_mapping(text_tokenizer).values()) + merged_seq = [] + + for i in range(batch_size): + + input_seq = out_dict[key]['tensor'][i] + input_seq = input_seq[out_dict[key]['input_mask'][i] == 0] + input_seq = input_seq.tolist() + + target_seq = out_dict[key]['tensor'][i] + target_seq = target_seq[out_dict[key]['target_mask'][i] == 0] + target_seq = target_seq.tolist() + + merged_seq.extend(merge_span_masking(input_seq, target_seq, sentinel_ids=sentinel_ids)) + + merged_seq = torch.tensor(merged_seq, device=device).unsqueeze(0) + + mod_dict[key] = { + 'tensor': merged_seq, + 'input_mask': torch.zeros(merged_seq.shape, dtype=torch.bool, device=device), + 'target_mask': torch.ones(merged_seq.shape, dtype=torch.bool, device=device), + 'decoder_attention_mask': torch.zeros(merged_seq.shape, dtype=torch.bool, device=device), + } + + return mod_dict \ No newline at end of file diff --git a/fourm/models/lora_utils.py b/fourm/models/lora_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7f259efa9ce57f4079399793050388a38bd753d8 --- /dev/null +++ b/fourm/models/lora_utils.py @@ -0,0 +1,177 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List, Set, Optional, Type + +import torch +import torch.nn as nn + + +SELF_ATTENTION_MODULES = {'Attention', 'NormAttention'} +CROSS_ATTENTION_MODULES = {'CrossAttention', 'NormCrossAttention'} +ATTENTION_MODULES = SELF_ATTENTION_MODULES | CROSS_ATTENTION_MODULES +MLP_MODULES = {'Mlp', 'GatedMlp', 'SwiGLUFFNFused'} # SwiGLUFFNFused is from DINOv2 +TRANSFORMER_MODULES = ATTENTION_MODULES | MLP_MODULES + + +def get_LoRA_module_names(id: str) -> Set[str]: + """ Returns a list of module names that are LoRA-adapted for the given id. """ + id = id.lower() + if id in ['selfattn', 'selfattention', 'self_attn', 'self_attention']: + return SELF_ATTENTION_MODULES + elif id in ['crossattn', 'crossattention', 'cross_attn', 'cross_attention']: + return CROSS_ATTENTION_MODULES + elif id in ['attn', 'attention']: + return ATTENTION_MODULES + elif id in ['mlp']: + return MLP_MODULES + elif id in ['all', 'transformer']: + return TRANSFORMER_MODULES + else: + raise ValueError(f'Unknown LoRA module id {id}.') + + +class LoRAWrapper(nn.Module): + """Low-Rank Adaptation Wrapper for linear layers. + See https://arxiv.org/abs/2106.09685 + + Args: + linear: nn.Linear layer to wrap + rank: Rank of adaptation matrix B@A + scale: x = W_0@x + scale * B@A@x + num_packed_linear: Set to > 1 when wrapping e.g. packed kv, or qkv attention weights. + Weights will be initialized as if num_packed_linear = 1, but the LoRA bottleneck will + be num_packed_linear times larger. + """ + def __init__(self, linear: nn.Module, rank: int = 4, scale: float = 1.0, num_packed_linear: int = 1): + super().__init__() + self.rank = rank + self.scale = scale + self.in_features, self.out_features = linear.in_features, linear.out_features + assert num_packed_linear * rank <= min(self.in_features, self.out_features), \ + f'LoRA rank {num_packed_linear} * {rank} must be less or equal than {min(self.in_features, self.out_features)}' + + self.linear = linear + self.lora_down = nn.Linear(self.in_features, num_packed_linear*rank, bias=False) + self.lora_up = nn.Linear(num_packed_linear*rank, self.out_features, bias=False) + + nn.init.normal_(self.lora_down.weight, std=1/rank) + nn.init.zeros_(self.lora_up.weight) + + def fuse_LoRA_into_linear(self) -> nn.Linear: + """ Returns a single nn.Linear layer with the LoRA matrix fused into the original one. """ + fused_linear = nn.Linear(self.in_features, self.out_features, bias=self.linear.bias is not None) + fused_linear.weight.data = self.linear.weight + self.scale * (self.lora_up.weight @ self.lora_down.weight) + if self.linear.bias is not None: + fused_linear.bias.data = self.linear.bias + return fused_linear + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ LoRA adapted linear layer forward pass. """ + return self.linear(x) + self.lora_up(self.lora_down(x)) * self.scale + + +def _find_modules( + model, + ancestor_class: Optional[Set[str]] = None, + search_class: List[Type[nn.Module]] = [nn.Linear], + exclude_children_of: Optional[List[Type[nn.Module]]] = [LoRAWrapper], +): + """ + Find all modules of a certain class (or union of classes) that are direct or + indirect descendants of other modules of a certain class (or union of classes). + + Returns all matching modules, along with the parent of those moduless and the + names they are referenced by. + + Adapted from https://github.com/cloneofsimo/lora/blob/master/lora_diffusion/lora.py + """ + # Get the targets we should replace all linears under + if ancestor_class is not None: + ancestors = ( + module + for module in model.modules() + if module.__class__.__name__ in ancestor_class + ) + else: + # this, incase you want to naively iterate over all modules. + ancestors = [module for module in model.modules()] + + # For each target find every linear_class module that isn't a child of a LoRA layer + for ancestor in ancestors: + for fullname, module in ancestor.named_modules(): + if any([isinstance(module, _class) for _class in search_class]): + # Find the direct parent if this is a descendant, not a child, of target + *path, name = fullname.split(".") + parent = ancestor + while path: + parent = parent.get_submodule(path.pop(0)) + # Skip this linear if it's a child of a LoRA layer + if exclude_children_of and any( + [isinstance(parent, _class) for _class in exclude_children_of] + ): + continue + # Otherwise, yield it + yield parent, name, module + + +def inject_trainable_LoRA( + model: nn.Module, + rank: int = 4, + scale: float = 1.0, + target_replace_modules: Set[str] = ATTENTION_MODULES +) -> None: + """Replaces all linear layers of the specified modules with LoRA-adapted linear layers. + Modifies the model in-place. + + Args: + model: nn.Module to modify + rank: Rank of adaptation matrix B@A + scale: x = W_0@x + scale * B@A@x + target_replace_modules: Set of module names to replace linear layers in. + """ + for _module, name, _child_module in _find_modules( + model, target_replace_modules, search_class=[nn.Linear] + ): + if sorted(name) == sorted('qkv'): + num_packed_linear = 3 + elif sorted(name) in [sorted('kv'), sorted('qk'), sorted('qv')]: + num_packed_linear = 2 + else: + num_packed_linear = 1 + + _module._modules[name] = LoRAWrapper(_child_module, rank=rank, scale=scale, num_packed_linear=num_packed_linear) + + +def fuse_LoRA_into_linear( + model: nn.Module, + target_replace_modules: Set[str] = ATTENTION_MODULES +) -> None: + """Fuses all LoRA-adapted linear layers back into single linear layers. + Modifies the model in-place. + + Args: + model: nn.Module to modify + target_replace_modules: Set of module names to replace linear layers in. + """ + for _module, name, _child_module in _find_modules( + model, target_replace_modules, search_class=[LoRAWrapper] + ): + _module._modules[name] = _module._modules[name].fuse_LoRA_into_linear() + + +def unfreeze_all_LoRA_layers(model: nn.Module) -> None: + """ Unfreezes all LoRA-adapted linear layers. """ + for name, param in model.named_parameters(): + if 'lora' in name: + param.requires_grad = True diff --git a/fourm/utils/__init__.py b/fourm/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..75b8e3d75a6da6e37ff5d9ffff6f92751c1588de --- /dev/null +++ b/fourm/utils/__init__.py @@ -0,0 +1,22 @@ +from .misc import * +from .checkpoint import * +from .timm.cross_entropy import LabelSmoothingCrossEntropy, SoftTargetCrossEntropy +from .data_constants import * +from .dist import * +from .logger import * +from .timm.metrics import AverageMeter, accuracy +from .timm.mixup import FastCollateMixup, Mixup +from .timm.model import freeze, get_state_dict, unfreeze, unwrap_model +from .timm.model_builder import create_model +from .timm.model_ema import ModelEma, ModelEmaV2 +from .native_scaler import NativeScalerWithGradNormCount +from .scheduler import cosine_scheduler, constant_scheduler, inverse_sqrt_scheduler +from .optim_factory import create_optimizer +from .timm.registry import model_entrypoint, register_model +from .timm.transforms import * +from .timm.transforms_factory import create_transform +from .tokenizer.text_tokenizer import * +from .s3_utils import * +from .run_name import * +from .generation_datasets import * +from .seeds import * \ No newline at end of file diff --git a/fourm/utils/checkpoint.py b/fourm/utils/checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..d62e52323c5f0ef9b8efa26bc35651e649f9ec17 --- /dev/null +++ b/fourm/utils/checkpoint.py @@ -0,0 +1,185 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -------------------------------------------------------- +# Based on the timm code base +# https://github.com/huggingface/pytorch-image-models +# -------------------------------------------------------- +import io +import os +import ast +import json +from pathlib import Path +from safetensors.torch import load as load_st + +import torch + +from .dist import save_on_main, is_main_process +from .timm.model import get_state_dict +from .s3_utils import save_on_s3 + + +def _load_checkpoint_for_ema(model_ema, checkpoint): + """ + Workaround for ModelEma._load_checkpoint to accept an already-loaded object + """ + mem_file = io.BytesIO() + torch.save(checkpoint, mem_file) + mem_file.seek(0) + model_ema._load_checkpoint(mem_file) + + +def load_state_dict(model, state_dict, prefix='', ignore_missing=''): + missing_keys = [] + unexpected_keys = [] + error_msgs = [] + # copy state_dict so _load_from_state_dict can modify it + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + def load(module, prefix=''): + local_metadata = {} if metadata is None else metadata.get( + prefix[:-1], {}) + module._load_from_state_dict( + state_dict, prefix, local_metadata, True, missing_keys, unexpected_keys, error_msgs) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + load(model, prefix=prefix) + + warn_missing_keys = [] + ignore_missing_keys = [] + for key in missing_keys: + keep_flag = True + for ignore_key in ignore_missing.split('|'): + if ignore_key in key: + keep_flag = False + break + if keep_flag: + warn_missing_keys.append(key) + else: + ignore_missing_keys.append(key) + + missing_keys = warn_missing_keys + + if len(missing_keys) > 0: + print("Weights of {} not initialized from pretrained model: {}".format( + model.__class__.__name__, missing_keys)) + if len(unexpected_keys) > 0: + print("Weights from pretrained model not used in {}: {}".format( + model.__class__.__name__, unexpected_keys)) + if len(ignore_missing_keys) > 0: + print("Ignored weights of {} not initialized from pretrained model: {}".format( + model.__class__.__name__, ignore_missing_keys)) + if len(error_msgs) > 0: + print('\n'.join(error_msgs)) + + +def save_model(args, epoch, model, model_without_ddp, optimizer, loss_scaler, loss_balancer=None, model_ema=None, ckpt_name=None, use_s3=False, all_nodes=False): + output_dir = Path(args.output_dir) + epoch_name = str(epoch) + ckpt_name = ckpt_name or epoch_name + + # Only create the save_dict on the main process, unless all_nodes is set to True + if is_main_process() or (all_nodes and args.gpu == 0): + checkpoint_path = os.path.join(output_dir, f'checkpoint-{ckpt_name}.pth') + + to_save = { + 'model': model_without_ddp.state_dict(), + 'epoch': epoch, + 'args': args, + 'scaler': loss_scaler.state_dict(), + } + + if optimizer is not None: + to_save['optimizer'] = optimizer.state_dict() + + if loss_balancer is not None: + to_save['loss_balancer'] = loss_balancer.state_dict() + + if model_ema is not None: + to_save['model_ema'] = get_state_dict(model_ema) + + save_on_main(to_save, checkpoint_path) + + if use_s3: + s3_path = os.path.join(args.s3_save_dir, f'checkpoint-{ckpt_name}.pth') + save_on_s3(checkpoint_path, s3_path, args.s3_endpoint) + + +def auto_load_model(args, model, model_without_ddp, optimizer, loss_scaler, model_ema=None): + output_dir = Path(args.output_dir) + # torch.amp + if args.auto_resume and len(args.resume) == 0: + import glob + all_checkpoints = glob.glob(os.path.join(output_dir, 'checkpoint-*.pth')) + latest_ckpt = -1 + for ckpt in all_checkpoints: + t = ckpt.split('-')[-1].split('.')[0] + if t.isdigit(): + latest_ckpt = max(int(t), latest_ckpt) + if latest_ckpt >= 0: + args.resume = os.path.join(output_dir, 'checkpoint-%d.pth' % latest_ckpt) + print("Auto resume checkpoint: %s" % args.resume) + + if args.resume: + if args.resume.startswith('https'): + checkpoint = torch.hub.load_state_dict_from_url( + args.resume, map_location='cpu') + else: + checkpoint = torch.load(args.resume, map_location='cpu') + model_without_ddp.load_state_dict(checkpoint['model']) + print("Resume checkpoint %s" % args.resume) + + if 'optimizer' in checkpoint and 'epoch' in checkpoint: + optimizer.load_state_dict(checkpoint['optimizer']) + args.start_epoch = checkpoint['epoch'] + 1 + + if 'scaler' in checkpoint: + loss_scaler.load_state_dict(checkpoint['scaler']) + print("With optim & sched!") + + if hasattr(args, 'model_ema') and args.model_ema: + _load_checkpoint_for_ema(model_ema, {'state_dict_ema': checkpoint['model_ema']}) + print("With EMA!") + +def parse_metadata(metadata_str): + metadata = {} + for k, v in metadata_str.items(): + try: + v_parsed = ast.literal_eval(v) + except: + v_parsed = v + metadata[k] = v_parsed + return metadata + +def load_safetensors(safetensors_path, return_metadata=True): + with open(safetensors_path, 'rb') as f: + data = f.read() + + tensors = load_st(data) + + if not return_metadata: + return tensors + + n_header = data[:8] + n = int.from_bytes(n_header, "little") + metadata_bytes = data[8 : 8 + n] + header = json.loads(metadata_bytes) + metadata = header.get("__metadata__", {}) + metadata = parse_metadata(metadata) + + return tensors, metadata \ No newline at end of file diff --git a/fourm/utils/clip/__init__.py b/fourm/utils/clip/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8ba4daf5f48dd91454429b4aa0c58dc752241c0b --- /dev/null +++ b/fourm/utils/clip/__init__.py @@ -0,0 +1,2 @@ +from .clip import * +from .model import * diff --git a/fourm/utils/clip/clip.py b/fourm/utils/clip/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..79614335e87a21fe5be3c7716e8ddffd8e790947 --- /dev/null +++ b/fourm/utils/clip/clip.py @@ -0,0 +1,236 @@ +# -------------------------------------------------------- +# Based on the CLIP code base +# https://github.com/openai/CLIP +# -------------------------------------------------------- + +import hashlib +import os +import urllib +import warnings +from typing import Any, Union, List +from pkg_resources import packaging + +import torch +from PIL import Image +from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize +from tqdm import tqdm + +from .model import build_model +from .simple_tokenizer import SimpleTokenizer as _Tokenizer + +try: + from torchvision.transforms import InterpolationMode + BICUBIC = InterpolationMode.BICUBIC +except ImportError: + BICUBIC = Image.BICUBIC + + +if packaging.version.parse(torch.__version__) < packaging.version.parse("1.7.1"): + warnings.warn("PyTorch version 1.7.1 or higher is recommended") + + +__all__ = ["available_models", "load", "tokenize"] +_tokenizer = _Tokenizer() + +_MODELS = { + "RN50": "https://openaipublic.azureedge.net/clip/models/afeb0e10f9e5a86da6080e35cf09123aca3b358a0c3e3b6c78a7b63bc04b6762/RN50.pt", + "RN101": "https://openaipublic.azureedge.net/clip/models/8fa8567bab74a42d41c5915025a8e4538c3bdbe8804a470a72f30b0d94fab599/RN101.pt", + "RN50x4": "https://openaipublic.azureedge.net/clip/models/7e526bd135e493cef0776de27d5f42653e6b4c8bf9e0f653bb11773263205fdd/RN50x4.pt", + "RN50x16": "https://openaipublic.azureedge.net/clip/models/52378b407f34354e150460fe41077663dd5b39c54cd0bfd2b27167a4a06ec9aa/RN50x16.pt", + "ViT-B/32": "https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt", + "ViT-B/16": "https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt", + "ViT-L/14": "https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt", + "ViT-L/14@336px": "https://openaipublic.azureedge.net/clip/models/3035c92b350959924f9f00213499208652fc7ea050643e8b385c2dac08641f02/ViT-L-14-336px.pt", +} + + +def _download(url: str, root: str): + os.makedirs(root, exist_ok=True) + filename = os.path.basename(url) + + expected_sha256 = url.split("/")[-2] + download_target = os.path.join(root, filename) + + if os.path.exists(download_target) and not os.path.isfile(download_target): + raise RuntimeError(f"{download_target} exists and is not a regular file") + + if os.path.isfile(download_target): + if hashlib.sha256(open(download_target, "rb").read()).hexdigest() == expected_sha256: + return download_target + else: + warnings.warn(f"{download_target} exists, but the SHA256 checksum does not match; re-downloading the file") + + with urllib.request.urlopen(url) as source, open(download_target, "wb") as output: + with tqdm(total=int(source.info().get("Content-Length")), ncols=80, unit='iB', unit_scale=True, unit_divisor=1024) as loop: + while True: + buffer = source.read(8192) + if not buffer: + break + + output.write(buffer) + loop.update(len(buffer)) + + if hashlib.sha256(open(download_target, "rb").read()).hexdigest() != expected_sha256: + raise RuntimeError(f"Model has been downloaded but the SHA256 checksum does not not match") + + return download_target + + +def _convert_image_to_rgb(image): + return image.convert("RGB") + + +def _transform(n_px): + return Compose([ + Resize(n_px, interpolation=BICUBIC), + CenterCrop(n_px), + _convert_image_to_rgb, + ToTensor(), + Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)), + ]) + + +def available_models() -> List[str]: + """Returns the names of available CLIP models""" + return list(_MODELS.keys()) + + +def load(name: str, device: Union[str, torch.device] = "cuda" if torch.cuda.is_available() else "cpu", jit: bool = False, download_root: str = None): + """Load a CLIP model + + Parameters + ---------- + name : str + A model name listed by `clip.available_models()`, or the path to a model checkpoint containing the state_dict + + device : Union[str, torch.device] + The device to put the loaded model + + jit : bool + Whether to load the optimized JIT model or more hackable non-JIT model (default). + + download_root: str + path to download the model files; by default, it uses "~/.cache/clip" + + Returns + ------- + model : torch.nn.Module + The CLIP model + + preprocess : Callable[[PIL.Image], torch.Tensor] + A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input + """ + if name in _MODELS: + model_path = _download(_MODELS[name], download_root or os.path.expanduser("~/.cache/clip")) + elif os.path.isfile(name): + model_path = name + else: + raise RuntimeError(f"Model {name} not found; available models = {available_models()}") + + try: + # loading JIT archive + model = torch.jit.load(model_path, map_location=device if jit else "cpu").eval() + state_dict = None + except RuntimeError: + # loading saved state dict + if jit: + warnings.warn(f"File {model_path} is not a JIT archive. Loading as a state dict instead") + jit = False + state_dict = torch.load(model_path, map_location="cpu") + + if not jit: + model = build_model(state_dict or model.state_dict()).to(device) + if str(device) == "cpu": + model.float() + return model, _transform(model.visual.input_resolution) + + # patch the device names + device_holder = torch.jit.trace(lambda: torch.ones([]).to(torch.device(device)), example_inputs=[]) + device_node = [n for n in device_holder.graph.findAllNodes("prim::Constant") if "Device" in repr(n)][-1] + + def patch_device(module): + try: + graphs = [module.graph] if hasattr(module, "graph") else [] + except RuntimeError: + graphs = [] + + if hasattr(module, "forward1"): + graphs.append(module.forward1.graph) + + for graph in graphs: + for node in graph.findAllNodes("prim::Constant"): + if "value" in node.attributeNames() and str(node["value"]).startswith("cuda"): + node.copyAttributes(device_node) + + model.apply(patch_device) + patch_device(model.encode_image) + patch_device(model.encode_text) + + # patch dtype to float32 on CPU + if str(device) == "cpu": + float_holder = torch.jit.trace(lambda: torch.ones([]).float(), example_inputs=[]) + float_input = list(float_holder.graph.findNode("aten::to").inputs())[1] + float_node = float_input.node() + + def patch_float(module): + try: + graphs = [module.graph] if hasattr(module, "graph") else [] + except RuntimeError: + graphs = [] + + if hasattr(module, "forward1"): + graphs.append(module.forward1.graph) + + for graph in graphs: + for node in graph.findAllNodes("aten::to"): + inputs = list(node.inputs()) + for i in [1, 2]: # dtype can be the second or third argument to aten::to() + if inputs[i].node()["value"] == 5: + inputs[i].node().copyAttributes(float_node) + + model.apply(patch_float) + patch_float(model.encode_image) + patch_float(model.encode_text) + + model.float() + + return model, _transform(model.input_resolution.item()) + + +def tokenize(texts: Union[str, List[str]], context_length: int = 77, truncate: bool = False) -> torch.LongTensor: + """ + Returns the tokenized representation of given input string(s) + + Parameters + ---------- + texts : Union[str, List[str]] + An input string or a list of input strings to tokenize + + context_length : int + The context length to use; all CLIP models use 77 as the context length + + truncate: bool + Whether to truncate the text in case its encoding is longer than the context length + + Returns + ------- + A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length] + """ + if isinstance(texts, str): + texts = [texts] + + sot_token = _tokenizer.encoder["<|startoftext|>"] + eot_token = _tokenizer.encoder["<|endoftext|>"] + all_tokens = [[sot_token] + _tokenizer.encode(text) + [eot_token] for text in texts] + result = torch.zeros(len(all_tokens), context_length, dtype=torch.long) + + for i, tokens in enumerate(all_tokens): + if len(tokens) > context_length: + if truncate: + tokens = tokens[:context_length] + tokens[-1] = eot_token + else: + raise RuntimeError(f"Input {texts[i]} is too long for context length {context_length}") + result[i, :len(tokens)] = torch.tensor(tokens) + + return result diff --git a/fourm/utils/clip/model.py b/fourm/utils/clip/model.py new file mode 100644 index 0000000000000000000000000000000000000000..f6f73a9572bd215c7c78d2be27785918775bfdc0 --- /dev/null +++ b/fourm/utils/clip/model.py @@ -0,0 +1,504 @@ +# -------------------------------------------------------- +# Based on the CLIP code base +# https://github.com/openai/CLIP +# -------------------------------------------------------- + +from collections import OrderedDict +from typing import Tuple, Union +import math +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1): + super().__init__() + + # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1 + self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + + self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity() + + self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + + self.relu = nn.ReLU(inplace=True) + self.downsample = None + self.stride = stride + + if stride > 1 or inplanes != planes * Bottleneck.expansion: + # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1 + self.downsample = nn.Sequential(OrderedDict([ + ("-1", nn.AvgPool2d(stride)), + ("0", nn.Conv2d(inplanes, planes * self.expansion, 1, stride=1, bias=False)), + ("1", nn.BatchNorm2d(planes * self.expansion)) + ])) + + def forward(self, x: torch.Tensor): + identity = x + + out = self.relu(self.bn1(self.conv1(x))) + out = self.relu(self.bn2(self.conv2(out))) + out = self.avgpool(out) + out = self.bn3(self.conv3(out)) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + return out + + +class AttentionPool2d(nn.Module): + def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None): + super().__init__() + self.positional_embedding = nn.Parameter(torch.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5) + self.k_proj = nn.Linear(embed_dim, embed_dim) + self.q_proj = nn.Linear(embed_dim, embed_dim) + self.v_proj = nn.Linear(embed_dim, embed_dim) + self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) + self.num_heads = num_heads + + def forward(self, x, return_all_tokens=False): + x = x.reshape(x.shape[0], x.shape[1], x.shape[2] * x.shape[3]).permute(2, 0, 1) # NCHW -> (HW)NC + x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC + x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC + x, _ = F.multi_head_attention_forward( + query=x, key=x, value=x, + embed_dim_to_check=x.shape[-1], + num_heads=self.num_heads, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + in_proj_weight=None, + in_proj_bias=torch.cat([self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]), + bias_k=None, + bias_v=None, + add_zero_attn=False, + dropout_p=0, + out_proj_weight=self.c_proj.weight, + out_proj_bias=self.c_proj.bias, + use_separate_proj_weight=True, + training=self.training, + need_weights=False + ) + if return_all_tokens: + return x + else: + return x[0] + + +class ModifiedResNet(nn.Module): + """ + A ResNet class that is similar to torchvision's but contains the following changes: + - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool. + - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1 + - The final pooling layer is a QKV attention instead of an average pool + """ + + def __init__(self, layers, output_dim, heads, input_resolution=224, width=64): + super().__init__() + self.output_dim = output_dim + self.input_resolution = input_resolution + + # the 3-layer stem + self.conv1 = nn.Conv2d(3, width // 2, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(width // 2) + self.conv2 = nn.Conv2d(width // 2, width // 2, kernel_size=3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(width // 2) + self.conv3 = nn.Conv2d(width // 2, width, kernel_size=3, padding=1, bias=False) + self.bn3 = nn.BatchNorm2d(width) + self.avgpool = nn.AvgPool2d(2) + self.relu = nn.ReLU(inplace=True) + + # residual layers + self._inplanes = width # this is a *mutable* variable used during construction + self.layer1 = self._make_layer(width, layers[0]) + self.layer2 = self._make_layer(width * 2, layers[1], stride=2) + self.layer3 = self._make_layer(width * 4, layers[2], stride=2) + self.layer4 = self._make_layer(width * 8, layers[3], stride=2) + + embed_dim = width * 32 # the ResNet feature dimension + self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, heads, output_dim) + + def _make_layer(self, planes, blocks, stride=1): + layers = [Bottleneck(self._inplanes, planes, stride)] + + self._inplanes = planes * Bottleneck.expansion + for _ in range(1, blocks): + layers.append(Bottleneck(self._inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x, return_side_out=False, return_all_tokens=False): + def stem(x): + for conv, bn in [(self.conv1, self.bn1), (self.conv2, self.bn2), (self.conv3, self.bn3)]: + x = self.relu(bn(conv(x))) + x = self.avgpool(x) + return x + out = [] + x = x.type(self.conv1.weight.dtype) + x = stem(x) + x = self.layer1(x) + if return_side_out: + out.append(x) + x = self.layer2(x) + if return_side_out: + out.append(x) + x = self.layer3(x) + if return_side_out: + out.append(x) + x = self.layer4(x) + if return_side_out: + out.append(x) + x = self.attnpool(x, return_all_tokens) + out.append(x) + if len(out) == 1: + return x + else: + return out + + +class LayerNorm(nn.LayerNorm): + """Subclass torch's LayerNorm to handle fp16.""" + + def forward(self, x: torch.Tensor): + orig_type = x.dtype + ret = super().forward(x.type(torch.float32)) + return ret.type(orig_type) + + +class QuickGELU(nn.Module): + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + def __init__(self, d_model: int, n_head: int, attn_mask: torch.Tensor = None): + super().__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential(OrderedDict([ + ("c_fc", nn.Linear(d_model, d_model * 4)), + ("gelu", QuickGELU()), + ("c_proj", nn.Linear(d_model * 4, d_model)) + ])) + self.ln_2 = LayerNorm(d_model) + self.attn_mask = attn_mask + + def attention(self, x: torch.Tensor): + self.attn_mask = self.attn_mask.to(dtype=x.dtype, device=x.device) if self.attn_mask is not None else None + return self.attn(x, x, x, need_weights=False, attn_mask=self.attn_mask)[0] + + def forward(self, x: torch.Tensor): + x = x + self.attention(self.ln_1(x)) + x = x + self.mlp(self.ln_2(x)) + return x + + +class Transformer(nn.Module): + def __init__(self, width: int, layers: int, heads: int, attn_mask: torch.Tensor = None): + super().__init__() + self.width = width + self.layers = layers + self.resblocks = nn.Sequential(*[ResidualAttentionBlock(width, heads, attn_mask) for _ in range(layers)]) + + def forward(self, x: torch.Tensor, return_intermediate_out: bool = False): + if return_intermediate_out: + output = [] + for block in self.resblocks: + x = block(x) + output.append(x) + return output + + return self.resblocks(x) + + +class VisionTransformer(nn.Module): + def __init__(self, input_resolution: int, patch_size: int, width: int, layers: int, heads: int, output_dim: int): + super().__init__() + self.input_resolution = input_resolution + self.patch_size = patch_size + self.output_dim = output_dim + self.width = width + self.heads = heads + self.conv1 = nn.Conv2d(in_channels=3, out_channels=width, kernel_size=patch_size, stride=patch_size, bias=False) + + scale = width ** -0.5 + self.class_embedding = nn.Parameter(scale * torch.randn(width)) + self.positional_embedding = nn.Parameter(scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width)) + self.ln_pre = LayerNorm(width) + + self.transformer = Transformer(width, layers, heads) + + self.ln_post = LayerNorm(width) + self.proj = nn.Parameter(scale * torch.randn(width, output_dim)) + + def forward(self, x: torch.Tensor, return_all_tokens=False, return_all_final_tokens=False, return_final_tokens_no_cls=False, **kwargs): + + B, nc, w, h = x.shape + + x = self.conv1(x) # shape = [*, width, grid, grid] + x = x.reshape(x.shape[0], x.shape[1], -1) # shape = [*, width, grid ** 2] + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] + + x = torch.cat([self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1) # shape = [*, grid ** 2 + 1, width] + + if x.shape[1] != self.positional_embedding.shape[0]: + x = x + self.interpolate_pos_encoding(x, w, h).to(x.dtype) + else: + x = x + self.positional_embedding.to(x.dtype) + + x = self.ln_pre(x) + + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + + if return_all_tokens: + x = self.ln_post(x) + return x[:, 1:, :] + + if return_all_final_tokens: + return self.ln_post(x) @ self.proj + + if return_final_tokens_no_cls: + return self.ln_post(x)[:, 1:, :] @ self.proj + + x = self.ln_post(x[:, 0, :]) + + if self.proj is not None: + x = x @ self.proj + + return x + + def interpolate_pos_encoding(self, x, w, h): + npatch = x.shape[1] - 1 + N = self.positional_embedding.shape[0] - 1 # 256 for large + if npatch == N and w == h: + return self.positional_embedding + class_pos_embed = self.positional_embedding[[0]] + patch_pos_embed = self.positional_embedding[1:] + dim = x.shape[-1] + w0 = w // self.patch_size + h0 = h // self.patch_size + # we add a small number to avoid floating point error in the interpolation + # see discussion at https://github.com/facebookresearch/dino/issues/8 + w0, h0 = w0 + 0.1, h0 + 0.1 + patch_pos_embed = nn.functional.interpolate( + patch_pos_embed.reshape(1, int(math.sqrt(N)), int(math.sqrt(N)), dim).permute(0, 3, 1, 2), + scale_factor=(w0 / math.sqrt(N), h0 / math.sqrt(N)), + mode='bicubic', + ) + assert int(w0) == patch_pos_embed.shape[-2] and int(h0) == patch_pos_embed.shape[-1] + patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim) + return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1) + + +class CLIP(nn.Module): + def __init__(self, + embed_dim: int, # 512 + # vision + image_resolution: int, # 224 + vision_layers: Union[Tuple[int, int, int, int], int], # 12 + vision_width: int, # 768 + vision_patch_size: int, # 16 + # text + context_length: int, # 77 + vocab_size: int, # 49408 + transformer_width: int, # 512 + transformer_heads: int, # 8 + transformer_layers: int # 12 + ): + super().__init__() + self.context_length = context_length + + if isinstance(vision_layers, (tuple, list)): + vision_heads = vision_width * 32 // 64 + self.visual = ModifiedResNet( + layers=vision_layers, + output_dim=embed_dim, + heads=vision_heads, + input_resolution=image_resolution, + width=vision_width + ) + else: + vision_heads = vision_width // 64 + self.visual = VisionTransformer( + input_resolution=image_resolution, + patch_size=vision_patch_size, + width=vision_width, + layers=vision_layers, + heads=vision_heads, + output_dim=embed_dim + ) + + self.transformer = Transformer( + width=transformer_width, + layers=transformer_layers, + heads=transformer_heads, + attn_mask=self.build_attention_mask() + ) + + self.vocab_size = vocab_size + self.token_embedding = nn.Embedding(vocab_size, transformer_width) + self.positional_embedding = nn.Parameter(torch.empty(self.context_length, transformer_width)) + self.ln_final = LayerNorm(transformer_width) + + self.text_projection = nn.Parameter(torch.empty(transformer_width, embed_dim)) + self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) + + self.initialize_parameters() + + def initialize_parameters(self): + nn.init.normal_(self.token_embedding.weight, std=0.02) + nn.init.normal_(self.positional_embedding, std=0.01) + + if isinstance(self.visual, ModifiedResNet): + if self.visual.attnpool is not None: + std = self.visual.attnpool.c_proj.in_features ** -0.5 + nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std) + + for resnet_block in [self.visual.layer1, self.visual.layer2, self.visual.layer3, self.visual.layer4]: + for name, param in resnet_block.named_parameters(): + if name.endswith("bn3.weight"): + nn.init.zeros_(param) + + proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5) + attn_std = self.transformer.width ** -0.5 + fc_std = (2 * self.transformer.width) ** -0.5 + for block in self.transformer.resblocks: + nn.init.normal_(block.attn.in_proj_weight, std=attn_std) + nn.init.normal_(block.attn.out_proj.weight, std=proj_std) + nn.init.normal_(block.mlp.c_fc.weight, std=fc_std) + nn.init.normal_(block.mlp.c_proj.weight, std=proj_std) + + if self.text_projection is not None: + nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5) + + def build_attention_mask(self): + # lazily create causal attention mask, with full attention between the vision tokens + # pytorch uses additive attention mask; fill with -inf + mask = torch.empty(self.context_length, self.context_length) + mask.fill_(float("-inf")) + mask.triu_(1) # zero out the lower diagonal + return mask + + @property + def dtype(self): + return self.visual.conv1.weight.dtype + + def encode_image(self, image, return_side_out=False, return_all_tokens=False, return_all_final_tokens=False, **kwargs): + return self.visual(image.type(self.dtype), return_all_tokens, return_all_final_tokens, **kwargs) + + def encode_text(self, text, return_all_tokens=False, return_patch_tokens=False): + x = self.token_embedding(text).type(self.dtype) # [batch_size, n_ctx, d_model] + + x = x + self.positional_embedding.type(self.dtype) + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + x = self.ln_final(x).type(self.dtype) + + if return_patch_tokens: + return x + # x.shape = [batch_size, n_ctx, transformer.width] + # take features from the eot embedding (eot_token is the highest number in each sequence) + if return_all_tokens: + x = x @ self.text_projection + else: + x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection + return x + + def forward(self, image, text): + image_features = self.encode_image(image) + text_features = self.encode_text(text) + + # normalized features + image_features = image_features / image_features.norm(dim=-1, keepdim=True) + text_features = text_features / text_features.norm(dim=-1, keepdim=True) + + # cosine similarity as logits + logit_scale = self.logit_scale.exp() + logits_per_image = logit_scale * image_features @ text_features.t() + logits_per_text = logits_per_image.t() + + # shape = [global_batch_size, global_batch_size] + return logits_per_image, logits_per_text + + +def convert_weights(model: nn.Module): + """Convert applicable model parameters to fp16""" + + def _convert_weights_to_fp16(l): + if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Linear)): + l.weight.data = l.weight.data.half() + if l.bias is not None: + l.bias.data = l.bias.data.half() + + if isinstance(l, nn.MultiheadAttention): + for attr in [*[f"{s}_proj_weight" for s in ["in", "q", "k", "v"]], "in_proj_bias", "bias_k", "bias_v"]: + tensor = getattr(l, attr) + if tensor is not None: + tensor.data = tensor.data.half() + + for name in ["text_projection", "proj"]: + if hasattr(l, name): + attr = getattr(l, name) + if attr is not None: + attr.data = attr.data.half() + + model.apply(_convert_weights_to_fp16) + + +def build_model(state_dict: dict): + vit = "visual.proj" in state_dict + + if vit: + vision_width = state_dict["visual.conv1.weight"].shape[0] + vision_layers = len([k for k in state_dict.keys() if k.startswith("visual.") and k.endswith(".attn.in_proj_weight")]) + vision_patch_size = state_dict["visual.conv1.weight"].shape[-1] + grid_size = round((state_dict["visual.positional_embedding"].shape[0] - 1) ** 0.5) + image_resolution = vision_patch_size * grid_size + else: + counts: list = [len(set(k.split(".")[2] for k in state_dict if k.startswith(f"visual.layer{b}"))) for b in [1, 2, 3, 4]] + vision_layers = tuple(counts) + vision_width = state_dict["visual.layer1.0.conv1.weight"].shape[0] + output_width = round((state_dict["visual.attnpool.positional_embedding"].shape[0] - 1) ** 0.5) + vision_patch_size = None + assert output_width ** 2 + 1 == state_dict["visual.attnpool.positional_embedding"].shape[0] + image_resolution = output_width * 32 + + embed_dim = state_dict["text_projection"].shape[1] + context_length = state_dict["positional_embedding"].shape[0] + vocab_size = state_dict["token_embedding.weight"].shape[0] + transformer_width = state_dict["ln_final.weight"].shape[0] + transformer_heads = transformer_width // 64 + transformer_layers = len(set(k.split(".")[2] for k in state_dict if k.startswith(f"transformer.resblocks"))) + + model = CLIP( + embed_dim, + image_resolution, vision_layers, vision_width, vision_patch_size, + context_length, vocab_size, transformer_width, transformer_heads, transformer_layers + ) + + for key in ["input_resolution", "context_length", "vocab_size"]: + if key in state_dict: + del state_dict[key] + + convert_weights(model) + model.load_state_dict(state_dict) + return model.eval() diff --git a/fourm/utils/clip/simple_tokenizer.py b/fourm/utils/clip/simple_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..00a2dc38e3973b78eda74372868823ad1ddbb8ae --- /dev/null +++ b/fourm/utils/clip/simple_tokenizer.py @@ -0,0 +1,137 @@ +# -------------------------------------------------------- +# Based on the CLIP code base +# https://github.com/openai/CLIP +# -------------------------------------------------------- + +import gzip +import html +import os +from functools import lru_cache + +import ftfy +import regex as re + + +@lru_cache() +def default_bpe(): + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz") + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8+n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """Return set of symbol pairs in a word. + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + + +class SimpleTokenizer(object): + def __init__(self, bpe_path: str = default_bpe()): + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + merges = gzip.open(bpe_path).read().decode("utf-8").split('\n') + merges = merges[1:49152-256-2+1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v+'' for v in vocab] + for merge in merges: + vocab.append(''.join(merge)) + vocab.extend(['<|startoftext|>', '<|endoftext|>']) + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|endoftext|>'} + self.pat = re.compile(r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + ( token[-1] + '',) + pairs = get_pairs(word) + + if not pairs: + return token+'' + + while True: + bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except: + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word)-1 and word[i+1] == second: + new_word.append(first+second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = ' '.join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8')) + bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' ')) + return bpe_tokens + + def decode(self, tokens): + text = ''.join([self.decoder[token] for token in tokens]) + text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('', ' ') + return text diff --git a/fourm/utils/data_constants.py b/fourm/utils/data_constants.py new file mode 100644 index 0000000000000000000000000000000000000000..dbdca5367ed7a56f31d5a6a5cb4264c852420265 --- /dev/null +++ b/fourm/utils/data_constants.py @@ -0,0 +1,40 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +DEFAULT_CROP_PCT = 0.875 +IMAGENET_DEFAULT_MEAN = (0.485, 0.456, 0.406) +IMAGENET_DEFAULT_STD = (0.229, 0.224, 0.225) +IMAGENET_INCEPTION_MEAN = (0.5, 0.5, 0.5) +IMAGENET_INCEPTION_STD = (0.5, 0.5, 0.5) +IMAGENET_DPN_MEAN = (124 / 255, 117 / 255, 104 / 255) +IMAGENET_DPN_STD = tuple([1 / (.0167 * 255)] * 3) + +IMAGENET_SURFACE_NORMAL_MEAN = (0.501, 0.405, 0.137) +IMAGENET_SURFACE_NORMAL_STD = (0.114, 0.165, 0.081) + +SEG_IGNORE_INDEX = 255 +SEG_IGNORE_INDEX_V2 = 0 +PAD_MASK_VALUE = 254 +COCO_SEMSEG_NUM_CLASSES = 133 + 1 # One extra class for no-class +ADE20K_SEMSEG_NUM_CLASSES = 150 + 1 # One extra class for no-class +HYPERSIM_SEMSEG_NUM_CLASSES = 41 + + +IMAGE_TASKS = {'rgb', 'depth', 'semseg', 'semseg_hypersim', 'semseg_coco', 'semseg_ade20k', 'normal'} +DETECTION_TASKS = {'det'} # 'det_coco', 'det_lvis' +TEXT_TASKS = {'caption'} +VISION_TASKS = IMAGE_TASKS | DETECTION_TASKS +SEQUENCE_TASKS = DETECTION_TASKS | TEXT_TASKS + +NYU_MEAN = 2070.7764 +NYU_STD = 777.5723 diff --git a/fourm/utils/dist.py b/fourm/utils/dist.py new file mode 100644 index 0000000000000000000000000000000000000000..c6f2ca98c6e0d2f2ef978adaeaaa4ac462655828 --- /dev/null +++ b/fourm/utils/dist.py @@ -0,0 +1,100 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -------------------------------------------------------- +# Based on DETR and MMCV code bases +# https://github.com/facebookresearch/detr +# https://github.com/open-mmlab/mmcv +# -------------------------------------------------------- + +import os +import pickle +import shutil +import sys +import tempfile +import datetime + +import torch +import torch.distributed as dist + + +def setup_for_distributed(is_main): + """ + This function disables printing when not in main process + """ + import builtins as __builtin__ + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop('force', False) + if is_main or force or kwargs.get('file', None) == sys.stderr: + builtin_print(*args, **kwargs) + + __builtin__.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_main(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + +def save_on_all(*args, **kwargs): + torch.save(*args, **kwargs) + + +def init_distributed_mode(args): + if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ: + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ['WORLD_SIZE']) + args.gpu = int(os.environ['LOCAL_RANK']) + else: + print('Not using distributed mode') + args.distributed = False + return + + args.distributed = True + + torch.cuda.set_device(args.gpu) + args.dist_backend = 'nccl' + print('| distributed init (rank {}): {}, gpu {}'.format( + args.rank, args.dist_url, args.gpu), flush=True) + # Set timeout to 1h20 in case some long download of dataset has to happen + torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank, + timeout=datetime.timedelta(4800)) + torch.distributed.barrier() + if ("print_all" not in args) or (not args.print_all): + setup_for_distributed(args.rank == 0) \ No newline at end of file diff --git a/fourm/utils/fsdp_utils.py b/fourm/utils/fsdp_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..15e4220e828841e8170b2116e69b2b5075d76eca --- /dev/null +++ b/fourm/utils/fsdp_utils.py @@ -0,0 +1,116 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from pathlib import Path + +import torch + +from .dist import save_on_main, is_main_process +from .s3_utils import save_on_s3 + + +from torch.distributed.fsdp import ( + FullyShardedDataParallel as FSDP, + FullStateDictConfig, + StateDictType, +) + +from torch.distributed.fsdp.api import FullOptimStateDictConfig + + + +def save_model_fsdp(args, epoch, model, optimizer, model_ema=None, ckpt_name=None, use_s3=False): + output_dir = Path(args.output_dir) + epoch_name = str(epoch) + ckpt_name = ckpt_name or epoch_name + + + with FSDP.state_dict_type(model, + StateDictType.FULL_STATE_DICT, + FullStateDictConfig(offload_to_cpu=True, rank0_only=True), + FullOptimStateDictConfig(offload_to_cpu=True, rank0_only=True), + ): + + model_state_dict = model.state_dict() + if optimizer is not None: + optimizer_state_dict = FSDP.optim_state_dict(model, optimizer) + else: + optimizer_state_dict = None + + # Only create the save_dict on the main process, not needed or recommended to do so on all ranks + # This make save_on_main() redundant + if is_main_process(): + checkpoint_path = os.path.join(output_dir, f'checkpoint-{ckpt_name}.pth') + + to_save = { + 'model': model_state_dict, + 'epoch': epoch, + 'args': args, + } + + if optimizer is not None: + to_save['optimizer'] = optimizer_state_dict + + if model_ema is not None: + print("Model EMA is currently not supported for FSDP") + # to_save['model_ema'] = get_state_dict(model_ema) + + save_on_main(to_save, checkpoint_path) + + if use_s3: + s3_path = os.path.join(args.s3_save_dir, f'checkpoint-{ckpt_name}.pth') + save_on_s3(checkpoint_path, s3_path, args.s3_endpoint) + + +def auto_load_model_fsdp(args, model, optimizer, model_ema=None): + output_dir = Path(args.output_dir) + if args.auto_resume and len(args.resume) == 0: + import glob + all_checkpoints = glob.glob(os.path.join(output_dir, 'checkpoint-*.pth')) + latest_ckpt = -1 + for ckpt in all_checkpoints: + t = ckpt.split('-')[-1].split('.')[0] + if t.isdigit(): + latest_ckpt = max(int(t), latest_ckpt) + if latest_ckpt >= 0: + args.resume = os.path.join(output_dir, 'checkpoint-%d.pth' % latest_ckpt) + print("Auto resume checkpoint: %s" % args.resume) + + if args.resume: + if args.resume.startswith('https'): + checkpoint = torch.hub.load_state_dict_from_url( + args.resume, map_location='cpu') + else: + checkpoint = torch.load(args.resume, map_location='cpu') + + with FSDP.state_dict_type( + model, + StateDictType.FULL_STATE_DICT, + FullStateDictConfig(rank0_only=False), + FullOptimStateDictConfig(rank0_only=False), + ): + + model.load_state_dict(checkpoint['model']) + print("Resume checkpoint %s" % args.resume) + + + if 'optimizer' in checkpoint and 'epoch' in checkpoint: + optimizer_state_dict = FSDP.optim_state_dict_to_load(checkpoint['optimizer'], model, optimizer) + optimizer.load_state_dict(optimizer_state_dict) + args.start_epoch = checkpoint['epoch'] + 1 + + print("With optim & sched!") + + if hasattr(args, 'model_ema') and args.model_ema: + print("Model EMA is currently not supported for FSDP") \ No newline at end of file diff --git a/fourm/utils/generation.py b/fourm/utils/generation.py new file mode 100644 index 0000000000000000000000000000000000000000..2c5da3447afffb8184855efb66b70b94587e8ac3 --- /dev/null +++ b/fourm/utils/generation.py @@ -0,0 +1,99 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np +import math + + +def sample_to_batch(mod_dict, device, domains): + mod_dict = { + modality: {k: v.unsqueeze(0).to(device, non_blocking=True) for k, v in d.items()} + for modality, d in mod_dict.items() if modality in domains + } + + return mod_dict + + +def unbatch(tensor): + return tensor.detach().squeeze(0).cpu() + + +def batch_to_sample(mod_dict, domains): + mod_dict = { + modality: {k: unbatch(v) for k, v in d.items()} + for modality, d in mod_dict.items() if modality in domains + } + + return mod_dict + + +def batch_to_device(mod_dict, device, domains): + mod_dict = { + modality: {k: v.to(device, non_blocking=True) for k, v in d.items()} + for modality, d in mod_dict.items() if modality in domains + } + + return mod_dict + + +def cosine_schedule(num_steps, total_tokens): + iters = np.arange(num_steps) + base_value = 1 + final_value = 0 + schedule = np.array( + [final_value + 0.5 * (base_value - final_value) * (1 + math.cos(math.pi * i / (len(iters)))) for i in iters]) + schedule_tokens = [round(total_tokens * i) for i in (schedule[:-1] - schedule[1:])] + schedule_tokens.append(total_tokens - sum(schedule_tokens)) + return np.array(schedule_tokens) + + +def linear_schedule(num_steps, total_tokens): + schedule = np.linspace(0, total_tokens, num_steps + 1, dtype=int) + schedule_tokens = np.diff(schedule)[::-1] + schedule_tokens.sort() # Sorts the array in ascending order. + schedule_tokens = schedule_tokens[::-1] # Reverses the array to descending order. + return np.trim_zeros(schedule_tokens, 'b') # Trims trailing zeros. + + +def continue_schedule(schedule, num_current_tokens): + schedule_cumsum = np.cumsum(schedule) + keep_mask = schedule_cumsum > num_current_tokens + diff = schedule_cumsum[keep_mask][0] - num_current_tokens + new_schedule = schedule[keep_mask] + new_schedule[0] = diff + return new_schedule + + +def decreasing_temp_schedule(max, min, token_schedule): + schedule_cumsum = np.cumsum(token_schedule) / np.sum(token_schedule) + temp_schedule = np.array([min + (max - min) * (1 - s) for s in schedule_cumsum]) + return temp_schedule + + +def onex_temp_schedule(max_t, min_t, token_schedule, power=0.5, min_linspace=1, max_linspace=100): + """Abitrary temperature schedule for one over x""" + x = np.linspace(min_linspace, max_linspace, num=sum(token_schedule)) + y = 1/(x**power) + y = y - min(y) + y = y / max(y) + unscaled_schedule = y + schedule_cumsum = np.cumsum(token_schedule) / np.sum(token_schedule) + unscaled_schedule = [(1 - cs) * us for us, cs in zip(unscaled_schedule, schedule_cumsum)] + + temp_schedule = np.array([min_t + (max_t - min_t) * s for s in unscaled_schedule]).clip(min=1e-9) + return temp_schedule + + +def linear_temp_schedule(temp, token_schedule): + """ Temperature that decays the temperature inversely proportional to the token schedule. """ + return np.concatenate([np.array([temp * 1.0]), (temp * (token_schedule.sum() - token_schedule.cumsum()) / token_schedule.sum())[:-1]]).clip(min=1e-9) diff --git a/fourm/utils/generation_datasets/PartiPrompts.tsv b/fourm/utils/generation_datasets/PartiPrompts.tsv new file mode 100644 index 0000000000000000000000000000000000000000..1dcd196cad7a43b9b8de7522d11442ef6c4070a9 --- /dev/null +++ b/fourm/utils/generation_datasets/PartiPrompts.tsv @@ -0,0 +1,1633 @@ +Prompt Category Challenge Note +bond Abstract Basic Biology-inspired concepts with multiple meanings +element Abstract Basic Biology-inspired concepts with multiple meanings +molecule Abstract Basic Biology-inspired concepts with multiple meanings +life Abstract Basic Biology-inspired concepts with multiple meanings +protein Abstract Basic Biology-inspired concepts with multiple meanings +yin-yang Abstract Basic Related to five elements +wood Abstract Basic Related to five elements +metal Abstract Basic Related to five elements +space Abstract Basic Related to five elements +air Abstract Basic Related to five elements +fire Abstract Basic Related to five elements +water Abstract Basic Related to five elements +earth Abstract Basic Related to five elements +force Abstract Basic Physics concepts +motion Abstract Basic Physics concepts +inertia Abstract Basic Physics concepts +energy Abstract Basic Physics concepts +black hole Abstract Basic Physics concepts +gravity Abstract Basic Physics concepts +peace Abstract Basic +fairness Abstract Basic +gender Abstract Basic +intelligence Abstract Basic +bias Abstract Basic +hate Abstract Basic +anger Abstract Basic +emotion Abstract Basic +feeling Abstract Basic +love Abstract Basic +artificial intelligence Abstract Basic +meaning of life Abstract Basic +42 Abstract Basic Simple numbers but challenging +0 Abstract Basic Simple numbers but challenging +infinity Abstract Basic Math concepts +imaginary numbers Abstract Basic Math concepts +Fibonacci number Abstract Basic Math concepts +golden ratio Abstract Basic Math concepts +an F1 Vehicles Basic +parallel lines Illustrations Basic Math concepts +concentric circles Illustrations Basic Math concepts +concurrent lines Illustrations Basic Math concepts +congruent triangles Illustrations Basic Math concepts +a hot air balloon Vehicles Basic +The Starry Night Arts Basic +300 Abstract Basic Simple numbers but challenging +101 Abstract Basic Simple numbers but challenging +U.S. 101 World Knowledge Basic Simple numbers but challenging +commonsense Abstract Basic +happiness Abstract Basic +hope Abstract Basic +insight Abstract Basic +inspiration Abstract Basic +derision Abstract Basic +Salvador Dalí People Basic +a shiba inu Animals Basic +a handpalm People Basic +an espresso machine Artifacts Basic +a propaganda poster Artifacts Basic +The Oriental Pearl World Knowledge Basic CogView +Ha Long Bay World Knowledge Basic +A Vietnam map World Knowledge Basic +A bowl of Pho Food & Beverage Basic +a snail Animals Basic +brain coral Animals Basic +a walnut Produce & Plants Basic +a capybara Animals Basic +a baby penguin Animals Basic +a cup of boba Food & Beverage Basic +a photo of san francisco's golden gate bridge World Knowledge Basic DALL-E +A picture of some food in the plate Food & Beverage Basic VQ-Diffusion +a chair Artifacts Basic +the Empire State Building World Knowledge Basic +the Sydney Opera House World Knowledge Basic +a hedgehog Animals Basic +a corgi Animals Basic +a robot Artifacts Basic +robots Artifacts Basic +a fall landscape Outdoor Scenes Basic +a sunset Outdoor Scenes Basic +a boat Vehicles Basic +a fox Animals Basic +a red cube Illustrations Basic +a panda Animals Basic +a space elevator Artifacts Basic GLIDE +a city Outdoor Scenes Basic +a fog Outdoor Scenes Basic +a clock Artifacts Basic +a phone Artifacts Basic +food Food & Beverage Basic +a store front Outdoor Scenes Basic +an armchair Artifacts Basic +a teapot Artifacts Basic +an illustration of a teapot Artifacts Basic DALL-E +a tiger Animals Basic +a bench Artifacts Basic +an orange Produce & Plants Basic +a laptop Artifacts Basic +an owl Animals Basic +a train Vehicles Basic +a cow Animals Basic +a submarine Vehicles Basic +a whale Animals Basic +a t-shirt Artifacts Basic +a bowl Artifacts Basic +a flag Artifacts Basic +a cat Animals Basic +a towel Artifacts Basic +a wall Artifacts Basic +a car Vehicles Basic +a giraffe Animals Basic +an eagle Animals Basic +a kangaroo Animals Basic +a canal Outdoor Scenes Basic +the grand canyon World Knowledge Basic +lily pads Produce & Plants Basic +a street Outdoor Scenes Basic +a house Outdoor Scenes Basic +a fish Animals Basic +a city intersection Outdoor Scenes Basic +a circle Illustrations Basic +a red circle Illustrations Basic +a box Illustrations Basic +a yellow box Illustrations Basic +a red box Illustrations Basic +a sphere Illustrations Basic +a red sphere Illustrations Basic +a large blue box Illustrations Basic +a blue metallic sphere Illustrations Basic +a horse Animals Basic +a pumpkin Produce & Plants Basic +a sword Artifacts Basic +a statue Artifacts Basic +a logo Illustrations Basic +a circular logo Illustrations Basic +a coffee mug Artifacts Basic +a pig Animals Basic +a squirrel Animals Basic +a hammer Artifacts Basic +a screwdriver Artifacts Basic +a handsaw Artifacts Basic +a power drill Artifacts Basic +a cocktail Food & Beverage Basic +a margarita Food & Beverage Basic +an avocado Produce & Plants Basic +a kitchen Indoor Scenes Basic +an iPhone case Artifacts Basic +a coffee maker Artifacts Basic +a banana Produce & Plants Basic +a violin Artifacts Basic +a room Indoor Scenes Basic +a mountain Outdoor Scenes Basic +a bird Animals Basic +a TV Artifacts Basic +a Christmas tree Artifacts Basic +The Statue of Liberty World Knowledge Basic +the Eiffel Tower World Knowledge Basic +a zebra Animals Basic +the city of London World Knowledge Basic +a koi fish Animals Basic +a pineapple Produce & Plants Basic +a toaster Artifacts Basic +a sign Artifacts Basic +a red lego block Artifacts Basic +teacup Artifacts Basic +chair Artifacts Basic +waterfall Outdoor Scenes Basic +a pirate ship Vehicles Basic +a dragon Animals Basic +a present Artifacts Basic +a bottle Artifacts Basic +a book cover Illustrations Basic +a sweatshirt Artifacts Basic +matching socks Artifacts Basic +cash Artifacts Basic +a wood cabin Indoor Scenes Basic +a clock tower Artifacts Basic +a chimpanzee Animals Basic +a hat Artifacts Basic +sneakers Artifacts Basic +a roast turkey Food & Beverage Basic +a turkey Animals Basic +a plate Artifacts Basic +a ladder Artifacts Basic +a Tyrannosaurus Rex Animals Basic +a Stegasaurus Animals Basic +a Triceratops Animals Basic +a Styracosaurus Animals Basic +a Diplodocus Animals Basic +a yellow sticky note Artifacts Basic +a ball Artifacts Basic +red balls Artifacts Basic +an ostrich Animals Basic +a stone path Outdoor Scenes Basic +a wooden deck Outdoor Scenes Basic +an F1 race car Vehicles Basic +a taxi Vehicles Basic +a road Outdoor Scenes Basic +a volcano Outdoor Scenes Basic +a large open book Artifacts Basic +a living room Indoor Scenes Basic +an elephant Animals Basic +a tree Produce & Plants Basic +a dolphin Animals Basic +a rowboat Vehicles Basic +a crown Artifacts Basic +a pick-up truck Vehicles Basic +a key Artifacts Basic +a goat Animals Basic +a chest Artifacts Basic +a coffee table Artifacts Basic +toy cars Artifacts Basic +a bookshelf Artifacts Basic +the moon Outdoor Scenes Basic +the Earth Outdoor Scenes Basic +the International Space Station World Knowledge Basic +the planet Jupiter Outdoor Scenes Basic +a tornado Outdoor Scenes Basic +a tidal wave Outdoor Scenes Basic +a laptop screen Artifacts Basic +an airplane Vehicles Basic +a butterfly Animals Basic +a lizard Animals Basic +a turtle Animals Basic +an octopus Animals Basic +a book Artifacts Basic +a moose Animals Basic +a kachina doll World Knowledge Basic +a doorknocker Artifacts Basic +a rabbit Animals Basic +orange juice Food & Beverage Basic +a green pepper Food & Beverage Basic +beer Food & Beverage Basic +Times Square World Knowledge Basic +the Great Wall World Knowledge Basic +the Kremlin World Knowledge Basic +a pickup truck Vehicles Basic +a shoe Artifacts Basic +a plant Produce & Plants Basic +a flower Produce & Plants Basic +a pair of headphones Artifacts Basic +a chemtrail Outdoor Scenes Basic +a tennis court Outdoor Scenes Basic +a ceiling fan Indoor Scenes Basic +a fire hydrant Artifacts Basic +a wooden post Artifacts Basic +a trash bin Artifacts Basic +a piano Artifacts Basic +a musical note Illustrations Basic +a harp Artifacts Basic +a tuba Artifacts Basic +The Alamo World Knowledge Basic +a windmill Outdoor Scenes Basic +the geyser Old Faithful World Knowledge Basic +a motorcycle Vehicles Basic +a painting of black and white Illustrations Basic +the Great Pyramid World Knowledge Basic +the Parthenon World Knowledge Basic +the Millennium Wheel World Knowledge Basic +a marina Outdoor Scenes Basic +a team People Basic +a child People Basic +a man People Basic +a woman People Basic +a girl People Basic +a person People Basic +a boy People Basic +a boy and a tiger People Basic +a family People Basic +a father and a son People Basic +a portrait of an old man People Basic +a portrait of young girl People Basic +a scientist People Basic +A dignified beaver wearing glasses, a vest, and colorful neck tie. He stands next to a tall stack of books in a library. Animals Complex +A solitary figure shrouded in mists peers up from the cobble stone street at the imposing and dark gothic buildings surrounding it. an old-fashioned lamp shines nearby. oil painting. Arts Complex +A punk rock squirrel in a studded leather jacket shouting into a microphone while standing on a stump and holding a beer on dark stage. Animals Complex +Horses pulling a carriage on the moon's surface, with the Statue of Liberty and Great Pyramid in the background. The Planet Earth can be seen in the sky. World Knowledge Complex +A robot painted as graffiti on a brick wall. The words "Fly an airplane" are written on the wall. A sidewalk is in front of the wall, and grass is growing out of cracks in the concrete. Outdoor Scenes Complex +A warrior wombat holding a sword and shield in a fighting stance. The wombat stands in front of the Arc de Triomphe on a day shrouded mist with the sun high in the sky. World Knowledge Complex +A set of 2x2 emoji icons with happy, angry, surprised and sobbing faces. The emoji icons look like dogs. All of the dogs are wearing blue turtlenecks. Illustrations Complex +A set of 2x2 emoji icons with happy, angry, surprised and sobbing faces. The emoji icons look like macaroons. All of the macaroons are wearing cowboy hats. Illustrations Complex +A set of 2x2 emoji icons with happy, angry, surprised and sobbing faces. The emoji icons look like colorful macarons. All of the macarons are wearing cowboy hats. Illustrations Complex +A set of 2x2 emoji icons with happy, angry, surprised and sobbing faces. The emoji icons look like pandas. All of the pandas are wearing colorful sunglasses. Illustrations Complex +A set of 2x2 emoji icons with happy, angry, surprised and sobbing faces. The emoji icons look like pigs. All of the pigs are wearing crowns. Illustrations Complex +A richly textured oil painting of a young badger delicately sniffing a yellow rose next to a tree trunk. A small waterfall can be seen in the background. Arts Complex +A portrait of a metal statue of a pharaoh wearing steampunk glasses and a leather jacket over a white t-shirt that has a drawing of a space shuttle on it. World Knowledge Complex +A photograph of the inside of a subway train. There are frogs sitting on the seats. One of them is reading a newspaper. The window shows the river in the background. Vehicles Complex +A photograph of the inside of a subway train. There are lobsters sitting on the seats. One of them is reading a newspaper. The window shows the ocean in the background. Vehicles Complex +A photograph of the inside of a subway train. There are raccoons sitting on the seats. One of them is reading a newspaper. The window shows the city in the background. Vehicles Complex +A photograph of the inside of a subway train. There are red pandas sitting on the seats. One of them is reading a newspaper. The window shows the jungle in the background. Vehicles Complex +A raccoon wearing formal clothes, wearing a tophat and holding a cane. The raccoon is holding a garbage bag. Oil painting in the style of abstract cubism. Arts Complex +A raccoon wearing formal clothes, wearing a tophat and holding a cane. The raccoon is holding a garbage bag. Oil painting in the style of Egyptian tomp hieroglyphics. Arts Complex +A raccoon wearing formal clothes, wearing a tophat and holding a cane. The raccoon is holding a garbage bag. Oil painting in the style of Hokusai. Arts Complex +A raccoon wearing formal clothes, wearing a tophat and holding a cane. The raccoon is holding a garbage bag. Oil painting in the style of Madhubani art. Arts Complex +A raccoon wearing formal clothes, wearing a tophat and holding a cane. The raccoon is holding a garbage bag. Oil painting in the style of pixel art. Arts Complex +A raccoon wearing formal clothes, wearing a tophat and holding a cane. The raccoon is holding a garbage bag. Oil painting in the style of pointilism. Arts Complex +A raccoon wearing formal clothes, wearing a tophat and holding a cane. The raccoon is holding a garbage bag. Oil painting in the style of Rembrandt. Arts Complex +A raccoon wearing formal clothes, wearing a tophat and holding a cane. The raccoon is holding a garbage bag. Oil painting in the style of traditional Chinese painting. Arts Complex +A raccoon wearing formal clothes, wearing a tophat and holding a cane. The raccoon is holding a garbage bag. Oil painting in the style of Vincent Van Gogh. Arts Complex +A raccoon wearing formal clothes, wearing a top hat and holding a cane. The raccoon is holding a garbage bag. Oil painting in the style of Vincent Van Gogh. Arts Complex +a statue of Abraham Lincoln wearing an opaque and shiny astronaut's helmet. The statue sits on the moon, with the planet Earth in the sky. World Knowledge Complex +a portrait of a statue of the Egyptian god Anubis wearing aviator goggles, white t-shirt and leather jacket. The city of Los Angeles is in the background. World Knowledge Complex +A smiling sloth wearing a leather jacket, a cowboy hat, a kilt and a bowtie. The sloth is holding a quarterstaff and a big book. A shiny VW van with a cityscape painted on it and parked on grass. World Knowledge Complex +A smiling sloth wearing a leather jacket, a cowboy hat, a kilt and a bowtie. The sloth is holding a quarterstaff and a big book. The sloth stands a few feet in front of a shiny VW van. The van has a cityscape painted on it and parked on grass. World Knowledge Complex +A DSLR photo of a shiny VW van that has a cityscape painted on it. A smiling sloth stands on grass in front of the van and is wearing a leather jacket, a cowboy hat, a kilt and a bowtie. The sloth is holding a quarterstaff and a big book. World Knowledge Complex +a poodle wearing a baseball cap holding a dictionary in hand and writing bonez on a chalkboard Animals Complex +a portrait of a statue of anubis with a crown and wearing a yellow t-shirt that has a space shuttle drawn on it World Knowledge Complex +an oil surrealist painting of a dreamworld on a seashore where clocks and watches appear to be inexplicably limp and melting in the desolate landscape. a table on the left, with a golden watch swarmed by ants. a strange fleshy creature in the center of the painting Arts Complex Description of The Persistence of Memory +a portrait of a statue of the Egyptian god Anubis wearing aviator goggles, white t-shirt and leather jacket. A full moon over the city of Los Angeles is in the background at night. World Knowledge Complex +a portrait of a statue of the Egyptian god Anubis wearing aviator goggles, white t-shirt and leather jacket. the skyline of Los Angeles at night can be seen in the background. World Knowledge Complex +a portrait of a statue of the Egyptian god Anubis wearing aviator goggles, white t-shirt and leather jacket, flying over the city of Mars. World Knowledge Complex +A photo of the Space Shuttle Endeavor painted yellow and flying over the Earth. The continent of South America is visible. World Knowledge Complex +A close-up photo of a wombat wearing a red backpack and raising both arms in the air. Mount Rushmore is in the background. World Knowledge Complex +A penguin wearing aviator goggles and flying confidently next to a bemused eagle. Animals Complex +A donkey is playing tug-of-war against an octopus. The donkey holds the rope in its mouth. A cat is jumping over the rope. Animals Complex +A wall in a royal castle. There are two paintings on the wall. The one on the left a detailed oil painting of the royal raccoon king. The one on the right a detailed oil painting of the royal raccoon queen. A cute dog looking at the two paintings, holding a sign saying 'plz conserve' Indoor Scenes Complex +A single beam of light enter the room from the ceiling. The beam of light is illuminating an easel. On the easel there is a Rembrandt painting of a raccoon Indoor Scenes Complex +The Millennium Wheel next to the Statue of Liberty. The Sagrada Familia church is also visible. World Knowledge Complex +A train ride in the monsoon rain in Kerala. With a Koala bear wearing a hat looking out of the window. There is a lot of coconut trees out of the window. World Knowledge Complex +A photograph of a portrait of a statue of a pharaoh wearing steampunk glasses, white t-shirt and leather jacket. World Knowledge Complex +An oil painting of two rabbits in the style of American Gothic, wearing the same clothes as in the original. Arts Complex +a blue cow is standing next to a tree with red leaves and yellow fruit. the cow is standing in a field with white flowers. impressionistic painting Animals Complex +A wombat sits in a yellow beach chair, while sipping a martini that is on his laptop keyboard. The wombat is wearing a white panama hat and a floral Hawaiian shirt. Out-of-focus palm trees in the background. Animals Complex +A high-contrast photo of a panda riding a horse. The panda is wearing a wizard hat and is reading a book. The horse is standing on a street against a gray concrete wall. Colorful flowers and the word "PEACE" are painted on the wall. Green grass grows from cracks in the street. Animals Complex +The buildings of downtown Manhattan situated at below Mount Everest. The Great Pyramid is in the foreground. World Knowledge Complex +Anime illustration of the Great Pyramid sitting next to the Parthenon under a blue night sky of roiling energy, exploding yellow stars, and chromatic blue swirls World Knowledge Complex +An anime illustration of Sydney Opera House sitting next to Eiffel tower, under a blue night sky of roiling energy, exploding yellow stars, and radiating swirls of blu World Knowledge Complex +A close-up high-contrast photo of Sydney Opera House sitting next to Eiffel tower, under a blue night sky of roiling energy, exploding yellow stars, and radiating swirls of blue World Knowledge Complex +the Sydney Opera House with the Eiffel tower sitting on the right, and Mount Everest rising above World Knowledge Complex +Greek statue of a man comforting a cat. The cat has a big head. The man looks angry. People Complex +a robot painted as graffiti on a brick wall. a sidewalk is in front of the wall, and grass is growing out of cracks in the concrete. Outdoor Scenes Complex +a portrait of a statue of a pharaoh wearing steampunk glasses, white t-shirt and leather jacket. dslr photograph. People Complex +a real flamingo reading a large open book. a big stack of books is piled up next to it. dslr photograph. Animals Complex +A photo of a hamburger fighting a hot dog in a boxing ring. The hot dog is tired and up against the ropes. Food & Beverage Complex +a blue cow is standing next to a tree with red leaves and yellow fruit. the cow is standing in a field with white flowers. impressionistic painting. Animals Complex +A cozy living room with a painting of a corgi on the wall above a couch and a round coffee table in front of a couch and a vase of flowers on a coffee table Indoor Scenes Complex +Photograph of a wall along a city street with a watercolor mural of foxes in a jazz band. Outdoor Scenes Complex +A high resolution photo of a donkey in a clown costume giving a lecture at the front of a lecture hall. The blackboard has mathematical equations on it. There are many students in the lecture hall. Indoor Scenes Complex +A black dog sitting on a wooden chair. A white cat with black ears is standing up with its paws on the chair. Animals Complex +a Saint Bernard standing up with its paws in the air. A young girl is seated on the dog's shoulders. People Complex +a photograph of a squirrel holding an arrow above its head and holding a longbow in its left hand Animals Complex +An empty fireplace with a television above it. The TV shows a lion hugging a giraffe. Indoor Scenes Complex +an invisible man wearing horn-rimmed glasses and a pearl bead necklase while looking at his phone People Complex +a blue semi-truck and its trailer jumping over a row of motorcycles. there are metal ramps on either side of the motorcycles. Vehicles Complex +a white rabbit in blue jogging clothes doubled over in pain while a turtle wearing a red tank top dashes confidently through the finish line Animals Complex +a hot air balloon with a yin-yang symbol, with the moon visible in the daytime sky Vehicles Complex +a photograph of a fiddle next to a basketball on a ping pong table Artifacts Complex +a racoon detective using a microscope while riding in a train Animals Complex +a photograph of an ostrich wearing a fedora and singing soulfully into a microphone Animals Complex +a cream-colored labradoodle wearing glasses and black beret teaching calculus at a blackboard Animals Complex +a basketball to the left of two soccer balls on a gravel driveway Artifacts Complex +a cat patting a crystal ball with the number 7 written on it in black marker Animals Complex +a cat licking a large felt ball with a drawing of the Eiffel Tower on it World Knowledge Complex +a giraffe wearing a white bathing suit and carefully stepping to the edge of a diving board and preparing to dive Animals Complex +a paranoid android freaking out and jumping into the air because it is surrounded by colorful Easter eggs Artifacts Complex +a mixed media image with a photograph of a woman with long orange hair over a background that is a sketch of a city skyline People Complex +a black and orange yin-yang symbol with tiger's heads instead of circles Illustrations Complex +a painting of an ornate treasure chest with a broad sword propped up against it, glowing in a dark cave Arts Complex +Renaissance portrayals of the Virgin Mary, seated in a loggia. Behind her is a hazy and seemingly isolated landscape imagined by the artist and painted using sfumato. People Complex Description of Mona Lisa +Oil-on-canvas painting of a blue night sky with roiling energy. A fuzzy and bright yellow crescent moon shining at the top. Below the exploding yellow stars and radiating swirls of blue, a distant village sits quietly on the right. Connecting earth and sky is a flame-like cypress tree with curling and swaying branches on the left. A church spire rises as a beacon over rolling blue hills Arts Complex Description of The Starry Night +Painting of a panic-stricken creature, simultaneously corpselike and reminiscent of a sperm or fetus, whose contours are echoed in the swirling lines of the blood-red sky Arts Complex Description of The Scream +a propaganda poster depicting a cat dressed as french emperor napoleon holding a piece of cheese Artifacts Complex DALL-E 2 +a white bird in front of a dinosaur standing by some trees Outdoor Scenes Complex +Snow mountain and tree reflection in the lake Outdoor Scenes Complex VQ-Diffusion +a white robot, a red robot and a black robot standing together Artifacts Complex +a tree reflected in the hood of a blue car Produce & Plants Complex +a tree reflected in the sunroof of a blue car Produce & Plants Complex +a monarch butterfly hatching from its chrysalis Animals Complex +a glass of orange juice to the right of a plate with buttered toast on it Food & Beverage Complex +a bottle of beer next to an ashtray with a half-smoked cigarrette Food & Beverage Complex +a pineapple with one beer to its left and two beers on its right Food & Beverage Complex +a pickup truck with a horse on its left and two dogs on its right Vehicles Complex +a brown trash bin with a green compost bin on its left and a blue recycling bin on its right Artifacts Complex +a woman with long hair next to a luminescent bird People Complex +a tall man stooping down to enter a low red sports car People Complex +a black dog jumping up to hug a woman wearing a red sweater People Complex +a man and a woman standing in the back up an old pickup truck People Complex +a boy sitting on the shoulders of a woman who is wearing an elegant dress People Complex +a girl with curly black hair jumping off a boulder People Complex +a grandmother reading a book to her grandson and granddaughter People Complex +a young woman with glasses reading a thick book at a mahogany desk People Complex +a man in a business suit on a ladder that is leaning up against the side of a white house People Complex +a man pouring milk into a coffee cup to make a latte with a beatiful design People Complex +a woman using a sledgehammer to smash an ice sculpture of a goose People Complex +a painting of the mona lisa on a white wall World Knowledge Complex +a man eating a glazed donut and a woman eating a chocolate cake People Complex +A teddy bear wearing a motorcycle helmet and cape is standing in front of Loch Awe with Kilchurn Castle behind him World Knowledge Fine-grained Detail +A teddy bear wearing a motorcycle helmet and cape is driving a speed boat near the Golden Gate Bridge World Knowledge Fine-grained Detail +A teddy bear wearing a motorcycle helmet and cape is car surfing on a taxi cab in New York City World Knowledge Fine-grained Detail +A teddy bear wearing a motorcycle helmet and cape is riding a motorcycle in Rio de Janeiro with Dois Irmãos in the background World Knowledge Fine-grained Detail +A punk rock squirrel in a studded leather jacket shouting into a microphone while standing on a stump Animals Fine-grained Detail +A punk rock squirrel in a studded leather jacket shouting into a microphone while standing on a lily pad Animals Fine-grained Detail +A punk rock squirrel in a studded leather jacket shouting into a microphone while standing on a boulder Animals Fine-grained Detail +A punk rock frog in a studded leather jacket shouting into a microphone while standing on a stump Animals Fine-grained Detail +A punk rock frog in a studded leather jacket shouting into a microphone while standing on a lily pad Animals Fine-grained Detail +A punk rock frog in a studded leather jacket shouting into a microphone while standing on a boulder Animals Fine-grained Detail +A punk rock platstumppus in a studded leather jacket shouting into a microphone while standing on a stump Animals Fine-grained Detail +A punk rock platypus in a studded leather jacket shouting into a microphone while standing on a lily pad Animals Fine-grained Detail +A punk rock platypus in a studded leather jacket shouting into a microphone while standing on a boulder Animals Fine-grained Detail +A map of the United States made out sushi. It is on a table next to a glass of red wine. World Knowledge Fine-grained Detail +A blue Porsche 356 parked in front of a yellow brick wall World Knowledge Fine-grained Detail +A helicopter flies over the Arches National Park. World Knowledge Fine-grained Detail +A helicopter flies over the Grand Canyon. World Knowledge Fine-grained Detail +A helicopter flies over Yosemite. World Knowledge Fine-grained Detail +A sunken ship becomes the homeland of fish. Vehicles Fine-grained Detail + A sloth in a go kart on a race track. The sloth is holding a banana in one hand. There is a banana peel on the track in the background. Animals Fine-grained Detail +a photograph of a bird wearing headphones and speaking into a microphone in a recording studio Animals Fine-grained Detail +A teddybear on a skateboard in Times Square, doing tricks on a cardboard box ramp. World Knowledge Fine-grained Detail +A smiling sloth wearing a bowtie and holding a quarterstaff and a big book. A shiny VW van parked on grass. World Knowledge Fine-grained Detail +A smiling sloth wearing a bowtie and holding a quarterstaff and a big book. Animals Fine-grained Detail +A smiling sloth wearing a leather jacket, a cowboy hat, a kilt and a bowtie. The sloth is holding a quarterstaff and a big book. Animals Fine-grained Detail +A smiling sloth wearing a leather jacket, a cowboy hat and a kilt. Animals Fine-grained Detail +A shiny VW van with a cityscape painted on it and parked on grass. World Knowledge Fine-grained Detail +The Statue of Liberty with the Manhattan skyline in the background. World Knowledge Fine-grained Detail +a cream colored labradoodle next to a white cat with black-tipped ears Animals Fine-grained Detail +The Great Pyramid of Giza situated in front of Mount Everest World Knowledge Fine-grained Detail +A photo of a Ming Dynasty vase on a leather topped table. World Knowledge Fine-grained Detail +A table full of food. There is a plate of chicken rice, a bowl of bak chor mee, and a bowl of laksa. Food & Beverage Fine-grained Detail +A map of the United States with a pin on San Francisco World Knowledge Fine-grained Detail +A map of the United States made out of sushi on the table. World Knowledge Fine-grained Detail +a cute wooden owl statue holding a large globe of the Earth above its head Animals Fine-grained Detail +a photograph of the mona lisa drinking coffee as she has her breakfast. her plate has an omelette and croissant. People Fine-grained Detail +the mona lisa wearing a cowboy hat and screaming a punk song into a microphone People Fine-grained Detail +a young badger delicately sniffing a yellow rose, richly textured oil painting Arts Fine-grained Detail +a photograph of a blue porsche 356 coming around a bend in the road Vehicles Fine-grained Detail +fairy cottage with smoke coming up chimney and a squirrel looking from the window Outdoor Scenes Fine-grained Detail +purple lego dollhouse with a pool and a swing Artifacts Fine-grained Detail +black bearded dog with an injured leg wearing a cone Animals Fine-grained Detail +brown white and black white guinea pigs eating parsley handed to them Animals Fine-grained Detail +The Rosetta Stone lying on the ground, covered in snow. World Knowledge Fine-grained Detail +a high-quality photograph of an armadillo playing a bagpipe while standing on one leg Animals Fine-grained Detail +a white robot with a red mohawk painted as graffiti on a red brick wall Artifacts Fine-grained Detail +a panda bear playing ping pong using a blue paddle against an ostrich using a red paddle Animals Fine-grained Detail +Anubis wearing sunglasses and sitting astride a hog motorcyle World Knowledge Fine-grained Detail +a photograph of sand with a bucket, lots of scattered shells but no sandpipers Outdoor Scenes Fine-grained Detail +a shiba inu wearing a beret and black turtleneck Animals Fine-grained Detail DALL-E 2 +a handpalm with leaves growing from it People Fine-grained Detail DALL-E 2 +panda mad scientist mixing sparkling chemicals Animals Fine-grained Detail DALL-E 2 +a corgi’s head depicted as an explosion of a nebula Animals Fine-grained Detail DALL-E 2 +a dolphin in an astronaut suit on saturn Animals Fine-grained Detail DALL-E 2 +a teddy bear on a skateboard in times square World Knowledge Fine-grained Detail DALL-E 2 +A Big Ben clock towering over the city of London World Knowledge Fine-grained Detail CogView +A Vietnam map showing Ha Long Bay World Knowledge Fine-grained Detail +A bowl of Pho served with bean sprouts on top Food & Beverage Fine-grained Detail +a capybara sitting in a field Animals Fine-grained Detail DALL-E +a baby penguin wearing a blue hat, red gloves, green shirt, and yellow pants Animals Fine-grained Detail DALL-E +a collection of glasses is sitting on a table Artifacts Fine-grained Detail DALL-E +A bare kitchen has wood cabinets and white appliances Indoor Scenes Fine-grained Detail VQ-Diffusion +A black and white landscape photograph of a black tree Outdoor Scenes Fine-grained Detail VQ-Diffusion +an ornate, high-backed mahogany chair with a red cushion Artifacts Fine-grained Detail +a whale breaching in front of the Sydney Opera House World Knowledge Fine-grained Detail +a white country home with a wrap-around porch Outdoor Scenes Fine-grained Detail +a corgi wearing a red bowtie and a purple party hat Animals Fine-grained Detail GLIDE +robots meditating in a vipassana retreat Artifacts Fine-grained Detail GLIDE +a white robot passing a soccer ball to a red robot Artifacts Fine-grained Detail +a fall landscape with a small cottage next to a lake Outdoor Scenes Fine-grained Detail GLIDE +a cat looking out of a window at a squirrel on a fence Animals Fine-grained Detail +a white cat with black ears and markings Animals Fine-grained Detail +a white cat and a tabby cat looking at each other Animals Fine-grained Detail +a cat sitting on a stairway railing Animals Fine-grained Detail +a professional photo of a sunset behind the grand canyon Outdoor Scenes Fine-grained Detail GLIDE +a large white yacht in a calm bay on a sunny day Vehicles Fine-grained Detail +a large white yacht tossed about in a stormy sea Vehicles Fine-grained Detail +a baby daikon radish in a tutu walking a dog Produce & Plants Fine-grained Detail DALL-E +an illustration of a baby daikon radish in a tutu walking a dog Produce & Plants Fine-grained Detail DALL-E +a long wooden bench in front of a brick wall Artifacts Fine-grained Detail +an ornate metal bench by a nature path Outdoor Scenes Fine-grained Detail +a white plastic bench with a high arched back Artifacts Fine-grained Detail +a tall horse next to a red car Animals Fine-grained Detail +an owl standing on a wire Animals Fine-grained Detail +an owl standing on a telephone wire Animals Fine-grained Detail +the skyline of New York City World Knowledge Fine-grained Detail +beautiful fireworks in the sky with red, white and blue Outdoor Scenes Fine-grained Detail +a steam locomotive speeding through a desert Vehicles Fine-grained Detail +a submarine floating past a shark Vehicles Fine-grained Detail +a whale breaching near a mountain Animals Fine-grained Detail +the United States flag next to the flag of Texas World Knowledge Fine-grained Detail +a giraffe with an owl on its head Animals Fine-grained Detail +an owl on top of an elephant's back Animals Fine-grained Detail +an eagle swooping down to catch a mouse Animals Fine-grained Detail +an owl with its wings spread out swooping over a tree Animals Fine-grained Detail +a giraffe eating the bark of a tree Animals Fine-grained Detail +a car and a truck on the road next to a traffic light Vehicles Fine-grained Detail +a cityscape at night with a full moon Outdoor Scenes Fine-grained Detail +an owl gripping a squirrel in its talons Animals Fine-grained Detail +a bloody mary cocktail next to a napkin Food & Beverage Fine-grained Detail +an old-fashioned cocktail next to a napkin Food & Beverage Fine-grained Detail +a long-island ice tea cocktail next to a napkin Food & Beverage Fine-grained Detail +a margarita next to a napkin Food & Beverage Fine-grained Detail +a room with two chairs and a painting of the Statue of Liberty Indoor Scenes Fine-grained Detail +a mountain and its reflection in a lake Outdoor Scenes Fine-grained Detail +a bird and its reflection in a fountain Outdoor Scenes Fine-grained Detail +The Statue of Liberty on a cloudy day World Knowledge Fine-grained Detail +The Statue of Liberty surrounded by helicopters World Knowledge Fine-grained Detail +the Eiffel Tower in winter World Knowledge Fine-grained Detail +a sunken ship at the bottom of the ocean Vehicles Fine-grained Detail +a sunken submarine at the bottom of the ocean Vehicles Fine-grained Detail +the Eiffel Tower in a desert World Knowledge Fine-grained Detail +Golden Gate bridge on the surface of Mars World Knowledge Fine-grained Detail +the city of London on Mars World Knowledge Fine-grained Detail +the city of London on the moon World Knowledge Fine-grained Detail +a stop sign with a large tree behind it Artifacts Fine-grained Detail +a stop sign knocked over on a sidewalk Artifacts Fine-grained Detail +teacups surounding a kettle Artifacts Fine-grained Detail +a dragon breathing fire Animals Fine-grained Detail +a dragon breathing fire on a castle Animals Fine-grained Detail +a dragon breathing fire onto a knight Animals Fine-grained Detail +a view of the Big Dipper in the night sky World Knowledge Fine-grained Detail +a view of the Orion constellation in the night sky World Knowledge Fine-grained Detail +a teddy bear to the right of a toy car Artifacts Fine-grained Detail +a toy car in front of a teddy bear Artifacts Fine-grained Detail +a large present with a red ribbon Artifacts Fine-grained Detail +a large present with a red ribbon to the left of a Christmas tree Indoor Scenes Fine-grained Detail +a half empty bottle of red wine Food & Beverage Fine-grained Detail +a wine bottle with a lit candle stuck in its spout Food & Beverage Fine-grained Detail +a wine bottle with a red ribbon wrapped around it Food & Beverage Fine-grained Detail +a kids' book cover with an illustration of white dog driving a red pickup truck Illustrations Fine-grained Detail +milk pouring into a large glass Food & Beverage Fine-grained Detail +milk pouring from a glass into a bowl Food & Beverage Fine-grained Detail +matching socks with cute cats on them Artifacts Fine-grained Detail +cash on a wooden table Artifacts Fine-grained Detail +a wood cabin with a fire pit in front of it Indoor Scenes Fine-grained Detail +view of a clock tower on a cloudy day Artifacts Fine-grained Detail +a chimpanzee wearing a bowtie and playing a piano Animals Fine-grained Detail +a black baseball hat with a flame decal on it Artifacts Fine-grained Detail +black hi-top sneakers with the Nike swoosh Artifacts Fine-grained Detail +a roast turkey being taken out of the oven Food & Beverage Fine-grained Detail +a bamboo ladder propped up against an oak tree Artifacts Fine-grained Detail +a Tyrannosaurus Rex roaring in front of a palm tree Animals Fine-grained Detail +a Stegasaurus eating ferns Animals Fine-grained Detail +a Triceratops charging down a hill Animals Fine-grained Detail +a Styracosaurus displaying its horns Animals Fine-grained Detail +a Diplodocus standing in a lake Animals Fine-grained Detail +a tabby cat coming through a door Animals Fine-grained Detail +a cat's tail showing under a couch Animals Fine-grained Detail +a light shining on a giraffe in a street Animals Fine-grained Detail +a barred owl peeking out from dense tree branches Animals Fine-grained Detail +a great gray owl with a mouse in its beak Animals Fine-grained Detail +a snowy owl standing in a grassy field Animals Fine-grained Detail +a stone path leading away from a fountain Outdoor Scenes Fine-grained Detail +a wooden deck overlooking a mountain valley Outdoor Scenes Fine-grained Detail +a mouse sitting next to a computer mouse Animals Fine-grained Detail +a tiny dragon landing on a knight's shield Animals Fine-grained Detail +an F1 race car in a Manhattan street World Knowledge Fine-grained Detail +a chess queen to the right of a chess knight Artifacts Fine-grained Detail +a white pawn attacking a black bishop Artifacts Fine-grained Detail +a mountain stream with salmon leaping out of it Outdoor Scenes Fine-grained Detail +a living room with a large Egyptian statue in the corner Indoor Scenes Fine-grained Detail +a stone bust next to an egg and an eggplant Artifacts Fine-grained Detail +scraps of paper drifting in the wind Outdoor Scenes Fine-grained Detail +the silhouette of an elephant on the full moon Animals Fine-grained Detail +a tree growing out of the middle of an intersection Outdoor Scenes Fine-grained Detail +a can of Spam on an elegant plate Food & Beverage Fine-grained Detail +a dolphin jumping over a rowboat Animals Fine-grained Detail +a pick-up truck rolling over a grassy field Vehicles Fine-grained Detail +a drop-top sports car coming around a bend in the road Vehicles Fine-grained Detail +the Taj Mahal at sunrise World Knowledge Fine-grained Detail +a small garden with an apple tree behind it Produce & Plants Fine-grained Detail +a old-time car with a large front grille Vehicles Fine-grained Detail +a blue wall with a large framed watercolor painting of a mountain Indoor Scenes Fine-grained Detail +a yellow wall with a large framed oil painting of a car Indoor Scenes Fine-grained Detail +an old-fashioned phone next to a sleek laptop Artifacts Fine-grained Detail +a motorcycle parked in an ornate bank lobby Vehicles Fine-grained Detail +a hot air balloon landing in a corn field Vehicles Fine-grained Detail +a wooden toy horse with a mane made of rope Artifacts Fine-grained Detail +a beach with a cruise ship passing by Outdoor Scenes Fine-grained Detail +a group of penguins in a snowstorm Animals Fine-grained Detail +a Scottish castle next to a loch World Knowledge Fine-grained Detail +a full moon peeking through clouds at night Outdoor Scenes Fine-grained Detail +a crescent moon viewed between tree branches at night Outdoor Scenes Fine-grained Detail +a full moon rising above a mountain at night Outdoor Scenes Fine-grained Detail +the International Space Station flying in front of the moon World Knowledge Fine-grained Detail +a tornado passing over a corn field Outdoor Scenes Fine-grained Detail +a tidal wave approaching a coastal road Outdoor Scenes Fine-grained Detail +an antique car by a beach Vehicles Fine-grained Detail +a water tower next to a deserted road Outdoor Scenes Fine-grained Detail +a small airplane flying over rolling hills Vehicles Fine-grained Detail +an airplane flying into a cloud that looks like monster Vehicles Fine-grained Detail +a blue airplane taxiing on a runway with the sun behind it Vehicles Fine-grained Detail +a car with tires that have yellow rims Vehicles Fine-grained Detail +a turtle upside down and spinning on its shell Animals Fine-grained Detail +an octopus fleeing and squirting black ink Animals Fine-grained Detail +a moose by a mountain stream Animals Fine-grained Detail +a kachina doll with feathers on its head and wearing a white dress and brown boots. World Knowledge Fine-grained Detail +a kachina doll standing in sand World Knowledge Fine-grained Detail +a doorknocker shaped like a lion's head Artifacts Fine-grained Detail +a silver doorknocker on a mahoghany door Artifacts Fine-grained Detail +a rabbit with white fur and black-tipped ears Animals Fine-grained Detail +a glass of orange juice next to an empty pitcher Food & Beverage Fine-grained Detail +a glass of orange juice with an orange peel stuck on the rim Food & Beverage Fine-grained Detail +a bottle of light beer with a lemon slice wedged in the rim Food & Beverage Fine-grained Detail +a helicopter hovering over Times Square World Knowledge Fine-grained Detail +a traffic jam at Times Square World Knowledge Fine-grained Detail +a prop plane flying low over the Great Wall World Knowledge Fine-grained Detail +a view of the Kremlin on a sunny day World Knowledge Fine-grained Detail +a view of the Kremlin with snow falling World Knowledge Fine-grained Detail +a palm tree forest in front of the Kremlin World Knowledge Fine-grained Detail +a Ferrari Testarossa in front of the Kremlin World Knowledge Fine-grained Detail +a rusty red pickup truck with white wheel rims Vehicles Fine-grained Detail +a blue pickup truck with a rhinoceros in its flatbed Vehicles Fine-grained Detail +an orange pickup truck next to a yellow Porsche 911 Vehicles Fine-grained Detail +a pickup truck kicking up dust on a back road Vehicles Fine-grained Detail +a pickup truck going up a mountain switchback Vehicles Fine-grained Detail +a pickup truck at the beach at sunrise Vehicles Fine-grained Detail +a pickup truck under street lights at night Vehicles Fine-grained Detail +a pair of shoes on a tennis racquet Artifacts Fine-grained Detail +a shoe with a sock draped over it Artifacts Fine-grained Detail +a plant growing on the side of a brick wall Produce & Plants Fine-grained Detail +a plant at the bottom of a shallow stream Produce & Plants Fine-grained Detail +a cow eating a green leafy plant Animals Fine-grained Detail +a bundle of blue and yellow flowers in a vase Produce & Plants Fine-grained Detail +a horse chewing a large blue flower Animals Fine-grained Detail +a yellow diamond-shaped sign with a deer silhouette Artifacts Fine-grained Detail +a yellow diamond-shaped sign with a turtle silhouette Artifacts Fine-grained Detail +a yellow diamond-shaped sign with a puma silhouette Artifacts Fine-grained Detail +a yellow diamond-shaped sign with a wooly mammoth silhouette Artifacts Fine-grained Detail +a pair of headphones on a pumpkin Artifacts Fine-grained Detail +a pair of headphones on a guitar Artifacts Fine-grained Detail +a pair of headphones on a statue of a horse Artifacts Fine-grained Detail +a pair of headphones dangling from a tree branch Artifacts Fine-grained Detail +a car's wheel crushing a pair of headphones Vehicles Fine-grained Detail +a chemtrail passing between two clouds Outdoor Scenes Fine-grained Detail +a long chemtrail trailing an airplane in a blue sky Outdoor Scenes Fine-grained Detail +a tennis court with a basketball hoop in one corner Outdoor Scenes Fine-grained Detail +a tennis court that is very wet from lots of rain Outdoor Scenes Fine-grained Detail +a rocking chair next to the net of a tennis court Outdoor Scenes Fine-grained Detail +a ceiling fan with an ornate light fixture Indoor Scenes Fine-grained Detail +red and yellow balloons hanging from a ceiling fan Indoor Scenes Fine-grained Detail +a sidewalk next to a wooden post with a blue '5' painted on top Outdoor Scenes Fine-grained Detail +a harp with a carved eagle figure at the top Artifacts Fine-grained Detail +a tuba with red flowers protruding from its bell Artifacts Fine-grained Detail +an elephant using its trunk to blow into a tuba Animals Fine-grained Detail +The Alamo with bright white clouds above it World Knowledge Fine-grained Detail +a flock of geese in front of The Alamo World Knowledge Fine-grained Detail +an old-fashioned windmill surrounded by flowers Outdoor Scenes Fine-grained Detail +a windmill farm next to a country road Outdoor Scenes Fine-grained Detail +a family of bears passing by the geyser Old Faithful World Knowledge Fine-grained Detail +an old red truck parked by the geyser Old Faithful World Knowledge Fine-grained Detail +an inflatable rabbit held up in the air by the geyser Old Faithful World Knowledge Fine-grained Detail +a Harley-Davidson motorcycle with a flame decal Vehicles Fine-grained Detail +a motorcycle in front of an rhinoceros Vehicles Fine-grained Detail +a chopper decorated with the Stars and Stripes Vehicles Fine-grained Detail +a painting of black and white with a red flower in the right corner Illustrations Fine-grained Detail +a gorilla climbing up the side of the Great Pyramid World Knowledge Fine-grained Detail +a beat-up truck at the base of the Great Pyramid World Knowledge Fine-grained Detail +snow covering the Great Pyramid World Knowledge Fine-grained Detail +the sun setting behind the Parthenon World Knowledge Fine-grained Detail +fireworks above the Parthenon World Knowledge Fine-grained Detail +the Millennium Wheel in a snow storm World Knowledge Fine-grained Detail +a marina with a herd of dolphins playing in it Outdoor Scenes Fine-grained Detail +a volcano exploding next to a marina Outdoor Scenes Fine-grained Detail +a team playing baseball at the beach People Fine-grained Detail +a man standing on a street corner People Fine-grained Detail +a crowd of people watching fireworks by a park People Fine-grained Detail +a crowd of people watching fireworks by a city People Fine-grained Detail +a smiling man with wavy brown hair and trimmed beard People Fine-grained Detail +a woman with long black hair and dark skin People Fine-grained Detail +a woman with long black hair and dark skin in a long white dress People Fine-grained Detail +a man with long blonde hair, brown eyes and blue jeans People Fine-grained Detail +an elderly woman with straight hair and metal-rimmed glasses People Fine-grained Detail +a man wearing sunglasses and business suit People Fine-grained Detail +a girl with long curly blonde hair and sunglasses People Fine-grained Detail +an old man with a long grey beard and green eyes People Fine-grained Detail +a woman with tan skin in blue jeans and yellow shirt People Fine-grained Detail +a woman with a dog puppet and a cat puppet People Fine-grained Detail +a group of skiers are preparing to walk up a sand dune People Fine-grained Detail +a child eating a birthday cake near some palm trees People Fine-grained Detail +a man with wild hair looking into a crystal ball People Fine-grained Detail +Gandalf saying you shall not pass World Knowledge Fine-grained Detail +a mosquito biting a man People Fine-grained Detail +a witch riding a broom People Fine-grained Detail +a knight holding a long sword People Fine-grained Detail +a man reading a book with a prism on its cover People Fine-grained Detail +a cricket team walking on to the pitch People Fine-grained Detail +a cricketer standing next to a wicket People Fine-grained Detail +an elder politician giving a campaign speech People Fine-grained Detail +a young girl playing piano People Fine-grained Detail +a girl getting a kite out of a tree People Fine-grained Detail +a politician giving a speech at a podium People Fine-grained Detail +a politician speaking to a large crowd People Fine-grained Detail +a child in the air while jumping on a trampoline People Fine-grained Detail +the Beatles crossing Abbey road People Fine-grained Detail +a girl riding an ostrich People Fine-grained Detail +a river with people swimming in it as a boat goes by People Fine-grained Detail +a selfie of an old man with a white beard People Fine-grained Detail +a young girl wearing a tiara and frilly dress People Fine-grained Detail +a ballet dancer next to a waterfall People Fine-grained Detail +a judge delivering a sentence to the defendant People Fine-grained Detail +people packed on a double-decker bus People Fine-grained Detail +a man riding a camel on the beach People Fine-grained Detail +a woman singing into a microphone People Fine-grained Detail +a child unraveling a roll of toilet paper People Fine-grained Detail +a girl examining an ammonite fossil People Fine-grained Detail +a fairy flying over a girl's shoulder People Fine-grained Detail +Tibetan priests ringing a bell People Fine-grained Detail +a man sleeping in a hammock People Fine-grained Detail +a library filled with kids reading books People Fine-grained Detail +a scientist accepting an award People Fine-grained Detail +A tornado made of sharks crashing into a skyscraper. painting in the style of Hokusai. Arts Imagination +A tornado made of sharks crashing into a skyscraper. painting in the style of abstract cubism. Arts Imagination +A tornado made of sharks crashing into a skyscraper. painting in the style of watercolor. Arts Imagination +A tornado made of tigers crashing into a skyscraper. painting in the style of Hokusai. Arts Imagination +A tornado made of tigers crashing into a skyscraper. painting in the style of abstract cubism. Arts Imagination +A tornado made of tigers crashing into a skyscraper. painting in the style of watercolor. Arts Imagination +A tornado made of bees crashing into a skyscraper. painting in the style of Hokusai. Arts Imagination +A tornado made of bees crashing into a skyscraper. painting in the style of abstract cubism. Arts Imagination +A tornado made of bees crashing into a skyscraper. painting in the style of watercolor. Arts Imagination +A television made of water that displays an image of a cityscape at night. Artifacts Imagination +A photo of a light bulb in outer space traveling the galaxy with a sailing boat inside the light bulb. Artifacts Imagination +A shiny robot wearing a race car suit and black visor stands proudly in front of an F1 race car. The sun is setting on a cityscape in the background. comic book illustration. Illustrations Imagination +A horse sitting on an astronaut's shoulders. People Imagination +The collision of two black holes in the center of a galaxy. Abstract Imagination +a super math wizard cat, richly textured oil painting Arts Imagination +A group of farm animals (cows, sheep, and pigs) made out of cheese and ham, on a wooden board. There is a dog in the background eyeing the board hungrily. Food & Beverage Imagination +A giant cobra snake made from corn Animals Imagination +A giant cobra snake made from sushi Animals Imagination +A giant cobra snake made from pancakes Animals Imagination +A giant cobra snake made from salad Animals Imagination +A photo of an astronaut riding a horse in the forest. There is a river in front of them with water lilies. People Imagination +A rhino beetle this size of a tank grapples a real life passenger airplane on the tarmac Animals Imagination +a massive robot with a coffee cup head. it is standing in the street with one foot smashing a car. Artifacts Imagination +A large city fountain that has milk instead of water. Several cats are leaning into the fountain. Outdoor Scenes Imagination +A bowl of soup that looks like a monster knitted out of wool Food & Beverage Imagination +A bowl of soup that looks like a monster made out of plasticine Food & Beverage Imagination +A bowl of soup that looks like a monster spray-painted on a wall Food & Beverage Imagination +A bowl of soup that looks like a monster with tofu says deep learning Food & Beverage Imagination +The 1970s logo for a london-area football club called "The Rumbury Wanderers" World Knowledge Imagination +a swordfish and a narwhal fencing in an underwater sandy arena. a crap and a lobster are cheering. Animals Imagination +A castle made of tortilla chips, in a river made of salsa. There are tiny burritos walking around the castle Food & Beverage Imagination +A city in 4-dimensional space-time Abstract Imagination +A high resolution photo of a rat working out in a gym. Animals Imagination +A high resolution photo of a chicken working out in a gym. Animals Imagination +a dump truck filled with soccer balls scuba diving in a coral reef. Vehicles Imagination +a yellow dump truck filled with soccer balls driving in a coral reef. a blue whale looms in the background. Vehicles Imagination +A photo of llama wearing sunglasses standing on the deck of a spaceship with the Earth in the background. Animals Imagination +A high resolution photo of a large bowl of ramen. There are several origami boats in the ramen of different colors. Food & Beverage Imagination +an elf drinking orange juice through a straw of a giant orange next to a squirrel and an owl watching from above Animals Imagination Concept from an old children's book +An alien octopus floats through a portal reading a newspaper. DSLR photo. Animals Imagination +A photo of an astronaut riding a horse in the forest. People Imagination +A photograph of a bird made of wheat bread and an egg. Animals Imagination +a horse corral with tigers standing in each stall Animals Imagination +a bike rack with some bike locks attached to it but no bicycles Vehicles Imagination +A red dragon dressed in a tuxedo and playing chess. The chess pieces are fashioned after robots. Animals Imagination +An oil painting of a two-story house lifting off the ground like a rocket. Arts Imagination +a portrait of a postal worker who has forgotten their mailbag People Imagination +a politician wearing a soccer jersey and holding a volleyball while giving a speech on a stage People Imagination +a kangaroo hopping in the air between two identical large statues of warrior rabbits Animals Imagination +a impressionistic painting of a scholarly badger reading the Rosetta Stone World Knowledge Imagination +A black dragon perched on top of a tall Egyptian obelisk and breathing flames at a knight on the ground Animals Imagination +a photograph of a knight in shining armor holding a basketball People Imagination +Dogs playing poker Animals Imagination Description of "Dogs Playing Poker" +Dogs sitting around a poker table Animals Imagination Description of "Dogs Playing Poker" +Dogs sitting around a poker table with beer bottles and chips. Their hands are holding cards. Animals Imagination Description of "Dogs Playing Poker" +Salvador Dalí with a robotic half face People Imagination DALL-E 2 +an espresso machine that makes coffee from human souls Artifacts Imagination DALL-E 2 +panda mad scientist Animals Imagination DALL-E 2 +a snail made of harp Animals Imagination DALL-E +a giraffe made of turtle Animals Imagination DALL-E +a giraffe imitating a turtle Animals Imagination DALL-E +a cube made of porcupine Illustrations Imagination DALL-E +a watermelon chair Artifacts Imagination +a giant gorilla at the top of the Empire State Building World Knowledge Imagination +a spaceship that looks like the Sydney Opera House Vehicles Imagination +a hedgehog using a calculator Animals Imagination GLIDE +a cat playing checkers Animals Imagination +a hamster dragon Animals Imagination +a futuristic city Outdoor Scenes Imagination GLIDE +corgi pizza Animals Imagination +a clock with no hands Artifacts Imagination +a baby daikon radish in a tutu Produce & Plants Imagination DALL-E +a chimpanzee sitting on a wooden bench Animals Imagination +a smiling banana wearing a bandana Produce & Plants Imagination +an orange wearing a cowboy hat Produce & Plants Imagination +a cat drinking a pint of beer Animals Imagination +a train going to the moon Vehicles Imagination +a subway train in an empty station Vehicles Imagination +a cow jumping over the moon Animals Imagination +a cat sitting in a car seat Animals Imagination +a friendly car Vehicles Imagination +two cats doing research Animals Imagination +a red train is coming down the beach Vehicles Imagination +a small kitchen with a white goat in it Indoor Scenes Imagination +a giraffe with a funny face Animals Imagination +a peaceful lakeside landscape with migrating herd of sauropods Outdoor Scenes Imagination +a cat standing on a horse Animals Imagination +a horse standing on an elephant Animals Imagination +a cat chasing a horse Animals Imagination +a horse chasing a cat Animals Imagination +a cat with four eyes Animals Imagination +a sword slicing through a loaf of bread Artifacts Imagination +a sword slicing through pouring milk Artifacts Imagination +a coffee mug floating in the sky Artifacts Imagination +a pig face with an eye patch Animals Imagination +a stork playing a violin Animals Imagination +a TV on a horse Artifacts Imagination +a Christmas tree on a toy train Artifacts Imagination +The Statue of Liberty with the face of an owl World Knowledge Imagination +a zebra with blue stripes Animals Imagination +a zebra with alternating blue and red stripes Animals Imagination +a yellow tiger with blue stripes Animals Imagination +a tiger wearing a tuxedo Animals Imagination +the statue of Liberty next to the Washington Monument World Knowledge Imagination +a koi fish flying in the sky Animals Imagination +a pineapple surfing on a wave Produce & Plants Imagination +a toaster shaking hands with a microwave Artifacts Imagination +blue apples on a tree with yellow leaves Produce & Plants Imagination +square blue apples on a tree with circular yellow leaves Produce & Plants Imagination +square red apples on a tree with circular green leaves Produce & Plants Imagination +a pirate ship landing on the moon Vehicles Imagination +a turkey walking in the kitchen Animals Imagination +a horned owl with a graduation cap and diploma Animals Imagination +a panda bear with aviator glasses on its head Animals Imagination +a nerdy bear wearing glasses and a bowtie Animals Imagination +a volcano spewing fish into the sky Outdoor Scenes Imagination +a tree with leaves that look like purple balloons Produce & Plants Imagination +a pony with a shooting star on its flank Animals Imagination +a half moon in the day sky Outdoor Scenes Imagination +the moon with a smiling face Outdoor Scenes Imagination +a tornado with a house carried along at its top Outdoor Scenes Imagination +a penguin that is a car Animals Imagination +a bicycle wheel that is made of red yarn Vehicles Imagination +a rabbit sitting on a turtle's back Animals Imagination +a rabbit wearing a black tophat and monocle Animals Imagination +a full pitcher of beer with an elephant's trunk in it Food & Beverage Imagination +an elephant walking on the Great Wall World Knowledge Imagination +a spaceship landing on the Great Wall World Knowledge Imagination +an ocean at the base of the Great Wall World Knowledge Imagination +a herd of buffalo stampeding at the Kremlin World Knowledge Imagination +a pickup truck missing its wheels Vehicles Imagination +a flower with a cat's face in the middle Produce & Plants Imagination +a flower with large red petals growing on the moon's surface Produce & Plants Imagination +a grand piano next to the net of a tennis court Outdoor Scenes Imagination +a piano tumbling down a hill Artifacts Imagination +a drawing of a series of musical notes wrapped around the Earth Illustrations Imagination +a tuba made of flower petals Artifacts Imagination +a spaceship hovering over The Alamo World Knowledge Imagination +a motorcycle hanging on a garage wall Vehicles Imagination +a dragon perched on top of the Great Pyramid World Knowledge Imagination +a flying boat gliding past the Parthenon Vehicles Imagination +the Parthenon in front of the Great Pyramid World Knowledge Imagination +a diplodocus standing in front of the Millennium Wheel World Knowledge Imagination +a child and a penguin sitting on the moon People Imagination +Superman shaking hands with Spiderman People Imagination +a man with puppet that looks like a king People Imagination +a person with arms like a tree branch People Imagination +a man riding a cat People Imagination +a triangle with a smiling face Illustrations Imagination +a square with an angry face Illustrations Imagination +a concert without any fans People Linguistic Structures Negation +a summer tree without any leaves Produce & Plants Linguistic Structures Negation +a classroom without any students People Linguistic Structures Negation +a bookshelf without any books on it Artifacts Linguistic Structures Negation +a shoe rack without any pairs of shoes on it Artifacts Linguistic Structures Negation +a closet without clothes Artifacts Linguistic Structures Negation +a plate that has no bananas on it. there is a glass without orange juice next to it. Food & Beverage Linguistic Structures Negation +A bird gives an apple to a squirrel Animals Linguistic Structures Ditransitive verbs +A squirrel gives an apple to a bird Animals Linguistic Structures Ditransitive verbs +A dog gives an apple to a squirrel Animals Linguistic Structures Ditransitive verbs +The sculpture rolled off the shelf because it wasn't level Artifacts Linguistic Structures Winograd Schema Challenge +The sculpture rolled off the shelf because it wasn't anchored Artifacts Linguistic Structures Winograd Schema Challenge +The large ball crashed right through the table because it was made of styrofoam Artifacts Linguistic Structures Winograd Schema Challenge +The large ball crashed right through the table because it was made of steel Artifacts Linguistic Structures Winograd Schema Challenge +The trophy doesn't fit into the brown suitcase because it's too small Artifacts Linguistic Structures Winograd Schema Challenge +The trophy doesn't fit into the brown suitcase because it's too large Artifacts Linguistic Structures Winograd Schema Challenge +The coach smiled at the player tossed the frisbee People Linguistic Structures https://en.wikipedia.org/wiki/List_of_linguistic_example_sentences +The rat the cat the dog bit chased escaped. Animals Linguistic Structures https://en.wikipedia.org/wiki/List_of_linguistic_example_sentences +To a squirrel, a dog gives an apple. Animals Linguistic Structures Ditransitive verbs +A man gives a woman a laptop and a boy a book. People Linguistic Structures Ditransitive verbs +A robot gives a wombat an orange and a lemur a banana. Animals Linguistic Structures Ditransitive verbs +A man sips a latte and a woman a beer. Food & Beverage Linguistic Structures +A rabbit checks its watch, and so does a gecko. Animals Linguistic Structures +A boy holds and a girl paints a piece of wood. People Linguistic Structures +A cat that has black fur bats a red Christmas ornament that has silver sparkles. Animals Linguistic Structures +The mouse the cat watches is jumping in the air. Animals Linguistic Structures +Four deer surrounding a moose. Animals Linguistic Structures +supercalifragilisticexpialidocious Abstract Linguistic Structures Mary Poppins described it as the word to use “when you have nothing to say.” +Incomprehensibilities Abstract Linguistic Structures +Pneumonoultramicroscopicsilicovolcanoconiosis Abstract Linguistic Structures Longest word: "a lung disease caused by inhalation of very fine silicate or quartz dust." +a large stone pedestal without a statue of a horse on it Artifacts Linguistic Structures Negation +a brown mouse laughing at a gray cat because a 16 ton weight is about to fall on its head Animals Linguistic Structures +a black dog sitting between a bush and a pair of green pants standing up with nobody inside them Animals Linguistic Structures +The horse raced past the barn fell Animals Linguistic Structures Garden-path sentences +The old man the boat. People Linguistic Structures Garden-path sentences +We painted the wall with cracks. People Linguistic Structures Garden-path sentences +The raft floated down the river sank. Vehicles Linguistic Structures Garden-path sentences +Mary gave the child the dog bit a Band-Aid. People Linguistic Structures Garden-path sentences +The man who whistles tunes pianos. People Linguistic Structures Garden-path sentences +One morning I chased an elephant in my pajamas People Linguistic Structures Syntactic Ambiguities +A tourist is looking at a whale using a binocular People Linguistic Structures Syntactic Ambiguities +The dog chased the cat, which ran up a tree. It waited at the top. Animals Linguistic Structures Winograd Schema Challenge +The dog chased the cat, which ran up a tree. It waited at the bottom. Animals Linguistic Structures Winograd Schema Challenge +An aerial view of Ha Long Bay without any boats World Knowledge Linguistic Structures Negation +a bench without any cats on it Artifacts Linguistic Structures Negation +a laptop no letters on its keyboard Artifacts Linguistic Structures Negation +a subway train with no cows in it Vehicles Linguistic Structures Negation +a kitchen without a refrigerator Indoor Scenes Linguistic Structures Negation +several lily pads without frogs Produce & Plants Linguistic Structures Negation +a street without vehicles Outdoor Scenes Linguistic Structures Negation +a house with no windows Outdoor Scenes Linguistic Structures Negation +a car with no windows Vehicles Linguistic Structures Negation +a fish without eyes Animals Linguistic Structures Negation +a city intersection without cars Outdoor Scenes Linguistic Structures Negation +a horse without a rider Animals Linguistic Structures Negation +a banana without its peel Produce & Plants Linguistic Structures Negation +a bat landing on a baseball bat Animals Linguistic Structures +a bird landing on a bat Animals Linguistic Structures +a rowboat without paddles Vehicles Linguistic Structures Negation +a harp without any strings Artifacts Linguistic Structures Negation +a marina without any boats in it Outdoor Scenes Linguistic Structures Negation +Zoomed out view of a giraffe and a zebra in the middle of a field covered with colorful flowers Animals Perspective +Three-quarters front view of a blue 1977 Corvette coming around a curve in a mountain road and looking over a green valley on a cloudy day. Vehicles Perspective +Three-quarters front view of a blue 1977 Ford F-150 coming around a curve in a mountain road and looking over a green valley on a cloudy day. Vehicles Perspective +Three-quarters front view of a blue 1977 Porsche 911 coming around a curve in a mountain road and looking over a green valley on a cloudy day. Vehicles Perspective +Three-quarters front view of a red 1997 Corvette coming around a curve in a mountain road and looking over a green valley on a cloudy day. Vehicles Perspective +Three-quarters front view of a red 1997 Ford F-150 coming around a curve in a mountain road and looking over a green valley on a cloudy day. Vehicles Perspective +Three-quarters front view of a red 1997 Porsche 911 coming around a curve in a mountain road and looking over a green valley on a cloudy day. Vehicles Perspective +Three-quarters front view of a yellow 2017 Corvette coming around a curve in a mountain road and looking over a green valley on a cloudy day. Vehicles Perspective +Three-quarters front view of a yellow 2017 Ford F-150 coming around a curve in a mountain road and looking over a green valley on a cloudy day. Vehicles Perspective +Three-quarters front view of a yellow 2017 Porsche 911 coming around a curve in a mountain road and looking over a green valley on a cloudy day. Vehicles Perspective +A photo of a frog reading the newspaper named "Toaday" written on it. There is a frog printed on the newspaper too. Animals Perspective +The frog found itself in the newspaper Animals Perspective +A cat dreaming about becoming a tiger Animals Perspective +A close-up of two chameleons wearing karate uniforms and fighting, jumping over a waterfall. Animals Perspective +A close-up of two mantis wearing karate uniforms and fighting, jumping over a waterfall. Animals Perspective +A close-up of two beetles wearing karate uniforms and fighting, jumping over a waterfall. Animals Perspective +A robot with a black visor and the number 42 on its chest. It stands proudly in front of an F1 race car. The sun is setting on a cityscape in the background. wide-angle view. comic book illustration. Illustrations Perspective +A smiling sloth is wearing a leather jacket, a cowboy hat, a kilt and a bowtie. The sloth is holding a quarterstaff and a big book. The sloth is standing on grass a few feet in front of a shiny VW van with flowers painted on it. wide-angle lens from below. World Knowledge Perspective +A smiling sloth wearing a leather jacket, bowtie, kilt and cowboy hat. The sloth is holding a quarterstaff and a big book. The sloth is standing on grass a few feet in front of a rusty old VW van with flowers painted on it. wide-angle lens from below. World Knowledge Perspective +view of a giraffe and a zebra in the middle of a field Animals Perspective +Aerial view of downtown Manhattan, but with Millennium Wheel next to the Statue of Liberty. The Great Pyramid is on a sandy island near the buildings. World Knowledge Perspective +Ground view of the Great Pyramids and Sphinx on the moon's surface. The back of an astronaut is in the foreground. The planet Earth looms in the sky. World Knowledge Perspective +Saturn rises on the horizon. Outdoor Scenes Perspective +Mars rises on the horizon. Outdoor Scenes Perspective +Jupiter rises on the horizon. Outdoor Scenes Perspective +a close-up of a blue dragonfly on a daffodil Animals Perspective +long shards of a broken mirror reflecting the eyes of a great horned owl Artifacts Perspective +side view of a brown horse wearing a saddle. the number 55 is stamped in white on its rear flank. Animals Perspective +three quarters view of a rusty old red pickup truck with white doors and a smashed windshield Vehicles Perspective +view from below of a tall white ladder with just one rung leaning up against a yellow brick wall Artifacts Perspective +a photo of the back of a covered wagon. a polar bear is sticking it's head out of the wagon. Vehicles Perspective +a close up of a handpalm with leaves growing from it People Perspective DALL-E 2 +a corgi’s head Animals Perspective +An aerial view of Ha Long Bay World Knowledge Perspective +a macro photograph of brain coral Animals Perspective DALL-E +a cross-section view of a walnut Produce & Plants Perspective DALL-E +an extreme close-up view of a capybara sitting in a field Animals Perspective DALL-E +an overhead view of the Empire State Building World Knowledge Perspective +a pig face Animals Perspective +a close-up of a bloody mary cocktail Food & Beverage Perspective +a close-up of an old-fashioned cocktail Food & Beverage Perspective +a close-up of a long-island ice tea cocktail Food & Beverage Perspective +a close-up of a margarita Food & Beverage Perspective +the back of a violin Artifacts Perspective +view of a clock tower from above Artifacts Perspective +view of a clock tower from below Artifacts Perspective +an aerial photo of a carnival at night Outdoor Scenes Perspective +an aerial photo of a baseball stadium Outdoor Scenes Perspective +an aerial photo of a sandy island in the ocean Outdoor Scenes Perspective +an ostrich's face Animals Perspective +a close-up of an ostrich's face Animals Perspective +the eyes of an owl Animals Perspective +a close-up of the eyes of an owl Animals Perspective +a top down view of a horse running in a field Animals Perspective +tall buildings seen through a window with rain on it Outdoor Scenes Perspective +trees seen through a car window on a rainy day Outdoor Scenes Perspective +a view of the Earth from the moon Outdoor Scenes Perspective +an image of the moons of the planet Jupiter Outdoor Scenes Perspective +the eye of the planet Jupiter Outdoor Scenes Perspective +a close-up of the eye of the planet Jupiter Outdoor Scenes Perspective +an aerial view of the Great Wall World Knowledge Perspective +view of the Great Wall from its base World Knowledge Perspective +an overhead view of a pickup truck with boxes in its flatbed Vehicles Perspective +a close-up of the keys of a piano Artifacts Perspective +an aerial view of the Great Pyramid World Knowledge Perspective +a view of the Milllenium Wheel from the Thames World Knowledge Perspective +overhead view of three people looking down at the street from the top of a tall building People Perspective +two people facing each other People Perspective +two people facing the viewer People Perspective +a three quarters view of a man getting into a car People Perspective +a stack of three red cubes with a blue sphere on the right and two green cones on the left Illustrations Properties & Positioning +A photo of a Persian Metal Engraving vase sitting to the left of a bunch of orange flowers. World Knowledge Properties & Positioning +A photo of a Japanese Porcelain Imari vase on the ground below a wooden chair. World Knowledge Properties & Positioning +a pen-and-ink crosshatched drawing of a sphere with dark square on it Illustrations Properties & Positioning +a metallic blue sphere to the left of a yellow box made of felt Illustrations Properties & Positioning +a blue wooden pyramid on top of a red plastic box Illustrations Properties & Positioning +concentric squares fading from yellow on the outside to deep orange on the inside Illustrations Properties & Positioning +A green heart with shadow Illustrations Properties & Positioning VQ-Diffusion +a red cube on top of a blue cube Illustrations Properties & Positioning GLIDE +a white flag with a red circle next to a solid blue flag Artifacts Properties & Positioning +a red circle on top of a blue square Illustrations Properties & Positioning +a yellow box to the right of a blue sphere Illustrations Properties & Positioning +a red box to the right of a green sphere Illustrations Properties & Positioning +a red box next to a blue box Illustrations Properties & Positioning +a red sphere on top of a yellow box Illustrations Properties & Positioning +a large blue box with three small yellow boxes on it Illustrations Properties & Positioning +several red lego blocks and one blue one Artifacts Properties & Positioning +a red block to the left of a blue pyramid Illustrations Properties & Positioning +three small yellow boxes on a large blue box Illustrations Properties & Positioning +a black background with a large yellow circle Illustrations Properties & Positioning +a white background with a large blue circle Illustrations Properties & Positioning +a black background with a large yellow square Illustrations Properties & Positioning +a white background with a large blue square Illustrations Properties & Positioning +a black background with a large yellow circle and a small red square Illustrations Properties & Positioning +a white background with a large blue circle and a small green square Illustrations Properties & Positioning +a large yellow triangle above a green square and red rectangle Illustrations Properties & Positioning +two small circles to the left of a red triangle that is on a green rectangle Illustrations Properties & Positioning +a green pyramid in front of a blue box Illustrations Properties & Positioning +a large yellow sphere behind a small purple pyramid Illustrations Properties & Positioning +a metallic blue sphere to the left of a brown cardboard box Illustrations Properties & Positioning +a yellow swirl next to a blue dashed line. Illustrations Properties & Positioning +a red swirl above a black dashed line. Illustrations Properties & Positioning +a green pepper to the left of a red pepper Food & Beverage Properties & Positioning +a brown trash bin to the left of a blue recycling bin Artifacts Properties & Positioning +a circular brown trash bin in front of a brick wall Artifacts Properties & Positioning +ten red apples Produce & Plants Quantity +Four dragons surrounding a dinosaur Animals Quantity +Four cats surrounding a dog Animals Quantity +two baseballs to the left of three tennis balls Artifacts Quantity +a group of not more than five meerkats standing with the sun setting behind them Animals Quantity +a basketball game between a team of four cats and a team of three dogs Animals Quantity +the hands of a single person holding a basketball People Quantity +several people putting their hands on a basketball People Quantity +a tiny football in front of three yellow tennis balls Artifacts Quantity +two beat-up baseballs on either side of a yellow basketball Artifacts Quantity +two baseballs next to three tennis balls Artifacts Quantity +7 dogs sitting around a poker table Animals Quantity Description of "Dogs Playing Poker" +7 dogs sitting around a poker table, two of which are turning away. Animals Quantity Description of "Dogs Playing Poker" +300 movie titles Artifacts Quantity Simple numbers but challenging +a collection of glasses Artifacts Quantity +two chairs Artifacts Quantity +three chairs Artifacts Quantity +five chairs Artifacts Quantity +two wooden chairs and three metal chairs Artifacts Quantity +three black cats standing next to two orange cats Animals Quantity +a bunch of laptops piled on a sofa Indoor Scenes Quantity +four owls standing on a telephone wire Animals Quantity +an owl family Animals Quantity +a comic about an owl family in the forest Artifacts Quantity +two cats Animals Quantity +a comic about two cats doing research Artifacts Quantity +three elephants standing on top of each other Animals Quantity +two red boxes Illustrations Quantity +two violins standing up Artifacts Quantity +two violins standing up with their bows on the ground in front of them Artifacts Quantity +three violins lying on the floor Indoor Scenes Quantity +three red lego blocks Artifacts Quantity +four teacups surounding a kettle Artifacts Quantity +three small yellow boxes Illustrations Quantity +bottles Artifacts Quantity +two frosted glass bottles Artifacts Quantity +five frosted glass bottles Artifacts Quantity +three green glass bottles Artifacts Quantity +four green glass bottles Artifacts Quantity +two wine bottles Artifacts Quantity +three wine bottles Artifacts Quantity +four wine bottles Artifacts Quantity +ten wine bottles Artifacts Quantity +two wine bottles and three beer cans Artifacts Quantity +a pair of matching socks with cute cats on them Artifacts Quantity +a pile of cash on a wooden table Artifacts Quantity +a pile of cash on a stone floor Artifacts Quantity +two red balls Artifacts Quantity +two red balls on a table Artifacts Quantity +five red balls Artifacts Quantity +five red balls on a table Artifacts Quantity +three yellow balls and two red boxes on a table Artifacts Quantity +an antique chest with three drawers Artifacts Quantity +ten triangles and five squares on a black background Illustrations Quantity +a pile of toy cars Artifacts Quantity +a full bookshelf with three shelves Artifacts Quantity +a bookshelf with ten books stacked vertically Artifacts Quantity +three airplanes parked in a row at a terminal Vehicles Quantity +three green peppers Food & Beverage Quantity +five green peppers to the right of two red onions Food & Beverage Quantity +Times Square with thousands of dogs running around World Knowledge Quantity +three pickup trucks piled on top of each other Vehicles Quantity +a pair of brown suede shoes Artifacts Quantity +two red flowers and three white flowers Produce & Plants Quantity +two parallel chemtrails in blue sky Outdoor Scenes Quantity +two chemtrails forming an X in blue sky Outdoor Scenes Quantity +a tennis court with three yellow cones on it Outdoor Scenes Quantity +a tennis court with tennis balls scattered all over it Outdoor Scenes Quantity +a ceiling fan with four white blades Indoor Scenes Quantity +a ceiling fan with five brown blades Indoor Scenes Quantity +two pianos next to each other Indoor Scenes Quantity +a series of musical notes Illustrations Quantity +a field with ten massive modern windmills Outdoor Scenes Quantity +two motorcycles facing each other Vehicles Quantity +children People Quantity +one child on a couch People Quantity +three children on a couch People Quantity +ten children on a couch People Quantity +a crowd of people watching fireworks People Quantity +a parade with cars and people waving People Quantity +a family of four posing in front of a house People Quantity +a family of four posing at the Grand Canyon People Quantity +a family of four posing at Mount Rushmore People Quantity +a family of four posing on the moon People Quantity +a family of four walking at the beach with waves covering their feet People Quantity +a jungle gym with three kids on it People Quantity +four people riding in a convertible car People Quantity +two runners crossing the finish line together People Quantity +a four-piece band on a stage in front of a small crowd People Quantity +two sets of identical twins People Quantity +Siberian husky playing the piano. Animals Simple Detail +A van parked on grass Vehicles Simple Detail +a smiling sloth Animals Simple Detail +A shiny VW van parked on grass. World Knowledge Simple Detail +a farm scene with cows, ducks and a tractor. Outdoor Scenes Simple Detail +a lavender backpack with a triceratops stuffed animal head on top Artifacts Simple Detail +a dolphin in an astronaut suit Animals Simple Detail DALL-E 2 +a teddy bear on a skateboard Artifacts Simple Detail DALL-E 2 +A tiger is playing football Animals Simple Detail CogView +A Google map highlighting Vietnam World Knowledge Simple Detail +A bowl of Chicken Pho Food & Beverage Simple Detail +A bowl of Beef Pho Food & Beverage Simple Detail +a lovestruck cup of boba Food & Beverage Simple Detail DALL-E +a horse in a field of flowers Outdoor Scenes Simple Detail +A green heart Illustrations Simple Detail +The sunset on the beach is wonderful Outdoor Scenes Simple Detail VQ-Diffusion +A red bus is driving on the road Vehicles Simple Detail VQ-Diffusion +A picture of a very tall stop sign Outdoor Scenes Simple Detail VQ-Diffusion +A movie poster of mountain and sea Artifacts Simple Detail VQ-Diffusion +A giraffe walking through a green grass covered field Animals Simple Detail VQ-Diffusion +A green train is coming down the tracks Vehicles Simple Detail VQ-Diffusion +A living area with a television and a table Indoor Scenes Simple Detail VQ-Diffusion +A red hydrant on the grass Outdoor Scenes Simple Detail VQ-Diffusion +Sunset over the sea Outdoor Scenes Simple Detail VQ-Diffusion +A small house in the wilderness Outdoor Scenes Simple Detail VQ-Diffusion +Sunset over the skyline of a city Outdoor Scenes Simple Detail VQ-Diffusion +A heart made of chocolate Illustrations Simple Detail VQ-Diffusion +A heart made of water Illustrations Simple Detail VQ-Diffusion +A heart made of wood Illustrations Simple Detail VQ-Diffusion +A heart made of cookie Illustrations Simple Detail VQ-Diffusion +a white country home Outdoor Scenes Simple Detail +robots meditating Artifacts Simple Detail +a robot kicking a soccer ball Artifacts Simple Detail +a cat looking out of a window Animals Simple Detail +a cat coming through a cat door Animals Simple Detail +a cat jumping in the air Animals Simple Detail +a cat jumping in the air to catch a bird Animals Simple Detail +a cat jumping in the air to get onto a table Animals Simple Detail +a sunset behind the grand canyon World Knowledge Simple Detail GLIDE +a boat in the canals of venice World Knowledge Simple Detail GLIDE +a large white yacht Vehicles Simple Detail +a panda eating bamboo Animals Simple Detail GLIDE +a fog rolling into new york World Knowledge Simple Detail GLIDE +a green clock Artifacts Simple Detail +a photo of a phone from the 20s Artifacts Simple Detail DALL-E +a photo of the food of china Food & Beverage Simple Detail DALL-E +a baby daikon radish Produce & Plants Simple Detail DALL-E +a very fancy French restaurant Indoor Scenes Simple Detail +a tiger standing by some flowers Animals Simple Detail +a bench next to a flower pot Artifacts Simple Detail +a bird standing on a stick Animals Simple Detail +a blue sports car on the road Vehicles Simple Detail +a red sports car on the road Vehicles Simple Detail +a soccer ball flying over a car Outdoor Scenes Simple Detail +the finale of a fireworks display Outdoor Scenes Simple Detail +beautiful fireworks in the sky Outdoor Scenes Simple Detail +a subway train Vehicles Simple Detail +a subway train coming out of a tunnel Vehicles Simple Detail +orange jello Food & Beverage Simple Detail +a yellow t-shirt Artifacts Simple Detail +a blue t-shirt Artifacts Simple Detail +a flag furling in the wind Artifacts Simple Detail +a white towel Artifacts Simple Detail +a white towel with a cat on it Artifacts Simple Detail +a black towel Artifacts Simple Detail +a black towel with a dog on it Artifacts Simple Detail +a yellow wall Artifacts Simple Detail +a friendly car in the city Vehicles Simple Detail +an owl family in the forest Animals Simple Detail +a kangaroo jumping through the park Animals Simple Detail +a street in Paris World Knowledge Simple Detail +a canal in Venice World Knowledge Simple Detail +a small house Outdoor Scenes Simple Detail +a small house on a mountain top Outdoor Scenes Simple Detail +the grand canyon on a cloudy day World Knowledge Simple Detail +a house on a mountain Outdoor Scenes Simple Detail +a photograph of a house on a mountain Outdoor Scenes Simple Detail +a peaceful lakeside landscape Outdoor Scenes Simple Detail +a cat reading a book Animals Simple Detail +a cat reading a newspaper Animals Simple Detail +a cat reading a comic book Animals Simple Detail +a horse reading a book Animals Simple Detail +a horse reading a newspaper Animals Simple Detail +a horse reading a comic book Animals Simple Detail +a cat jumping down from a wall Animals Simple Detail +a pumpkin with a candle in it Produce & Plants Simple Detail +a sword in a stone Artifacts Simple Detail +an Egyptian statue World Knowledge Simple Detail +an Egyptian statue in the desert World Knowledge Simple Detail +a massive statue in a temple Artifacts Simple Detail +a circular logo on a coffee mug Illustrations Simple Detail +a pig in a field Animals Simple Detail +a horse in a field Animals Simple Detail +a squirrel in a field Animals Simple Detail +a tiger in a field Animals Simple Detail +a pig in a forest Animals Simple Detail +a horse in a forest Animals Simple Detail +a squirrel in a forest Animals Simple Detail +a tiger in a forest Animals Simple Detail +a hammer on a table Artifacts Simple Detail +a screwdriver on a table Artifacts Simple Detail +a handsaw on a table Artifacts Simple Detail +a power drill on a table Artifacts Simple Detail +a bloody mary cocktail Food & Beverage Simple Detail +an old-fashioned cocktail Food & Beverage Simple Detail +a long-island ice tea cocktail Food & Beverage Simple Detail +a tree surrounded by flowers Produce & Plants Simple Detail +a street with several cars on it Outdoor Scenes Simple Detail +an avocado on a table Produce & Plants Simple Detail +a kitchen with a large refrigerator Indoor Scenes Simple Detail +weeds in the cracks of a sidewalk Artifacts Simple Detail +a wood treehouse in an oak tree Artifacts Simple Detail +a metal treehouse in an oak tree Artifacts Simple Detail +a violin next to an apple Artifacts Simple Detail +a room with two chairs and a painting Indoor Scenes Simple Detail +a canal in Amsterdam World Knowledge Simple Detail +red apples on a tree with green leaves Produce & Plants Simple Detail +a stop sign Artifacts Simple Detail +a yield sign Artifacts Simple Detail +a mountain with a cloud hanging over it Outdoor Scenes Simple Detail +a bottle of red wine Food & Beverage Simple Detail +wine bottles Food & Beverage Simple Detail +the cover of a book about gardening Illustrations Simple Detail +a milk container on a table Artifacts Simple Detail +a milk container in a refrigerator Artifacts Simple Detail +a marine iguana crossing the street Animals Simple Detail +cash on a stone floor Artifacts Simple Detail +a half-peeled banana Produce & Plants Simple Detail +a black baseball hat Artifacts Simple Detail +black hi-top sneakers Artifacts Simple Detail +a roast turkey on the table Food & Beverage Simple Detail +a plate with white rice topped by cooked vegetables Food & Beverage Simple Detail +slices of avocado on a piece of toast Food & Beverage Simple Detail +slices of mango on a piece of toast Food & Beverage Simple Detail +a bamboo ladder Artifacts Simple Detail +a cat jumps over a baby gate Animals Simple Detail +an ostrich standing on a couch Animals Simple Detail +a taxi driving in the countryside Vehicles Simple Detail +a two lane road with a bright yellow line Outdoor Scenes Simple Detail +a road ruined by an earthquake Outdoor Scenes Simple Detail +a volcano erupting near a small town Outdoor Scenes Simple Detail +a volcano with lava pouring down its slopes Outdoor Scenes Simple Detail +a horse running in a field Animals Simple Detail +a robot cooking Artifacts Simple Detail +a robot cooking in the kitchen Indoor Scenes Simple Detail +a compass next to a piece of fruit Artifacts Simple Detail +the silhouette of an elephant Animals Simple Detail +a tree growing through a fence Produce & Plants Simple Detail +a butterfly kite stuck high in a tree Produce & Plants Simple Detail +a unicorn with a multicolored mane Animals Simple Detail +a crown with a ruby in its center Artifacts Simple Detail +an ornate jewel-encrusted key Artifacts Simple Detail +a goat wearing headphones Animals Simple Detail +a small tree covered in white blossoms Produce & Plants Simple Detail +an antique chest Artifacts Simple Detail +a coffee table with a magazine on it Indoor Scenes Simple Detail +a yellow wall with two framed sketches Indoor Scenes Simple Detail +a squirrell driving a toy car Animals Simple Detail +a penguin standing on a sidewalk Animals Simple Detail +a basketball hoop with a large blue ball stuck in it Outdoor Scenes Simple Detail +a racoon holding a shiny red apple over its head Animals Simple Detail +a pair of glasses under a computer monitor Artifacts Simple Detail +a throw rug on a stone floor Artifacts Simple Detail +a beach with apartment buildings next to it Outdoor Scenes Simple Detail +a moose standing over a fox Animals Simple Detail +a massive statue of Hanuman World Knowledge Simple Detail +a black t-shirt Artifacts Simple Detail +a small airplane Vehicles Simple Detail +an airplane taking off of a runway Vehicles Simple Detail +a monarch butterfly Animals Simple Detail +a lizard that just lost its tail Animals Simple Detail +a golden doorknocker on a red door Artifacts Simple Detail +a pitcher of orange juice Food & Beverage Simple Detail +a glass of orange juice Food & Beverage Simple Detail +a green pepper cut in half on a plate Food & Beverage Simple Detail +a green pepper sliced into many pieces Food & Beverage Simple Detail +a full pint of IPA Food & Beverage Simple Detail +a half-full pitcher of stout Food & Beverage Simple Detail +Times Square during the day World Knowledge Simple Detail +Times Square at night World Knowledge Simple Detail +the Kremlin at night World Knowledge Simple Detail +a hot air balloon floating by the Kremlin World Knowledge Simple Detail +a black shoe with a lightning bolt on it Artifacts Simple Detail +a plant with small flowers with purple petals Produce & Plants Simple Detail +a plant with orange flowers shaped like stars Produce & Plants Simple Detail +a flower with large yellow petals Produce & Plants Simple Detail +a yellow diamond-shaped sign Artifacts Simple Detail +an elephant in a tennis court Animals Simple Detail +a silver fire hydrant next to a sidewalk Artifacts Simple Detail +a yellow fire hydrant in grass Artifacts Simple Detail +a red fire hydrant by a brick wall Artifacts Simple Detail +a rusty fire hydrant surrounded by dirt Artifacts Simple Detail +a fire hydrant with graffiti on it Artifacts Simple Detail +a snake curled around a wooden post Animals Simple Detail +a grand piano with a white bench Indoor Scenes Simple Detail +a piano with Christmas lights all over it Indoor Scenes Simple Detail +a piano with an open song book above the keys Indoor Scenes Simple Detail +an ornate gold harp Artifacts Simple Detail +a massive modern windmill Outdoor Scenes Simple Detail +a red sport bike Vehicles Simple Detail +a painting of black and white vertical stripes Illustrations Simple Detail +a painting of black and white checkerboard Illustrations Simple Detail +a painting of black and white with a red border Illustrations Simple Detail +a team playing baseball People Simple Detail +children on a couch People Simple Detail +a woman with long hair People Simple Detail +a man standing under a tree People Simple Detail +a smiling man People Simple Detail +an elderly woman People Simple Detail +an old man People Simple Detail +a man with puppet People Simple Detail +a boy going to school People Simple Detail +a girl going to a farm People Simple Detail +a family on a road trip People Simple Detail +a father and a son playing tennis People Simple Detail +a man heading a soccer ball People Simple Detail +a sad man with green hair People Simple Detail +a woman with sunglasses and red hair People Simple Detail +a man looking at a distant mountain People Simple Detail +a woman looking at a house People Simple Detail +a man riding a horse People Simple Detail +a man chasing a cat People Simple Detail +a man chasing a horse People Simple Detail +a pumpkin on a man's head People Simple Detail +an angry man People Simple Detail +a frustrated child People Simple Detail +a laughing woman People Simple Detail +a woman running on a trail People Simple Detail +a boy jumping off a wall People Simple Detail +a girl diving into a pool People Simple Detail +a man banging on a door People Simple Detail +a comic book supervillian Artifacts Simple Detail +Downtown Istanbul at sunrise. detailed ink wash. World Knowledge Style & Format +Downtown Seattle at sunrise. detailed ink wash. World Knowledge Style & Format +Downtown Beijing at sunrise. detailed ink wash. World Knowledge Style & Format +Downtown Rio de Janeiro at sunrise. detailed ink wash. World Knowledge Style & Format +Downtown LA at sunrise. detailed ink wash. World Knowledge Style & Format +Downtown Sydney at sunrise. detailed ink wash. World Knowledge Style & Format +Downtown Sanfrancisco at sunrise. detailed ink wash. World Knowledge Style & Format +Downtown Singapore at sunrise. detailed ink wash. World Knowledge Style & Format +Downtown NYC at sunrise. detailed ink wash. World Knowledge Style & Format +Downtown Austin at sunrise. detailed ink wash. World Knowledge Style & Format +Downtown Shanghai at sunrise. detailed ink wash. World Knowledge Style & Format +Downtown Saigon at sunrise. detailed ink wash. World Knowledge Style & Format +A section of the Great Wall in the mountains. detailed charcoal sketch. World Knowledge Style & Format +The Great Hypostyle Hall of Karnak. detailed charcoal sketch. World Knowledge Style & Format +A Mesoamerican pyramid surrounded by jungle. detailed charcoal sketch. World Knowledge Style & Format +Taj Mahal with its reflection. detailed charcoal sketch. World Knowledge Style & Format +A spaceship made of cardboard. Artifacts Style & Format +A old phonograph made of cardboard. Artifacts Style & Format +A castle made of cardboard. Artifacts Style & Format +A photo of a dragonfly made of water. Animals Style & Format +A photo of a panda made of water. Animals Style & Format +A photo of a teddy bear made of water. Animals Style & Format +A photo of a crocodile made of water. Animals Style & Format +A photo of a lotus flower made of water. Produce & Plants Style & Format +A photo of a maple leaf made of water. Produce & Plants Style & Format +A photo of a palm tree made of water. Produce & Plants Style & Format +A photo of a four-leaf clover made of water. Produce & Plants Style & Format +an abstract oil painting in deep red and black with a thick patches of white Arts Style & Format +light and happiness throughout and finding its way to every corner of the world, abstract oil painting Arts Style & Format +trying to find my way in a big confusing world, abstract oil painting Arts Style & Format +the door of knowing, a portal brightly opening the way through darkness. abstract anime landscape oil painting. Arts Style & Format +A soft beam of light shines down on an armored granite wombat warrior statue holding a broad sword. The statue stands an ornate pedestal in the cella of a temple. wide-angle lens. anime oil painting. Animals Style & Format +Oil painting generated by artificial intelligence Abstract Style & Format +A heavy metal tiger standing on a rooftop while singing and jamming on an electric guitar under a spotlight. anime illustration. Animals Style & Format +A funny Rube Goldberg machine made out of metal World Knowledge Style & Format +A funny Rube Goldberg machine made out of paper World Knowledge Style & Format +A funny Rube Goldberg machine made out of wood World Knowledge Style & Format +Oil painting of a giant robot made of sushi, holding chopsticks. Artifacts Style & Format +Portrait of a gecko wearing a train conductor’s hat and holding a flag that has a yin-yang symbol on it. Charcoal. Animals Style & Format +Portrait of a gecko wearing a train conductor’s hat and holding a flag that has a yin-yang symbol on it. Child's crayon drawing. Animals Style & Format +Portrait of a gecko wearing a train conductor’s hat and holding a flag that has a yin-yang symbol on it. Chinese ink. Animals Style & Format +Portrait of a gecko wearing a train conductor’s hat and holding a flag that has a yin-yang symbol on it. Comic. Animals Style & Format +Portrait of a gecko wearing a train conductor’s hat and holding a flag that has a yin-yang symbol on it. Marble statue. Animals Style & Format +Portrait of a gecko wearing a train conductor’s hat and holding a flag that has a yin-yang symbol on it. Oil on canvas. Animals Style & Format +Portrait of a gecko wearing a train conductor’s hat and holding a flag that has a yin-yang symbol on it. Photograph. Animals Style & Format +Portrait of a gecko wearing a train conductor’s hat and holding a flag that has a yin-yang symbol on it. Watercolor. Animals Style & Format +Portrait of a gecko wearing a train conductor’s hat and holding a flag that has a yin-yang symbol on it. Woodcut. Animals Style & Format +A rusty spaceship blasts off in the foreground. A city with tall skyscrapers is in the distance, with a mountain and ocean in the background. A dark moon is in the sky. realistic high-contrast anime illustration. Vehicles Style & Format +A photo of an Athenian vase with a painting of pandas playing basketball in the style of Egyptian hieroglyphics World Knowledge Style & Format +A photo of an Athenian vase with a painting of pandas playing soccer in the style of Egyptian hieroglyphics World Knowledge Style & Format +A photo of an Athenian vase with a painting of pandas playing tennis in the style of Egyptian hieroglyphics World Knowledge Style & Format +A photo of an Athenian vase with a painting of pangolins playing basketball in the style of Egyptian hieroglyphics World Knowledge Style & Format +A photo of an Athenian vase with a painting of pangolins playing soccer in the style of Egyptian hieroglyphics World Knowledge Style & Format +A photo of an Athenian vase with a painting of pangolins playing tennis in the style of Egyptian hieroglyphics World Knowledge Style & Format +A photo of an Athenian vase with a painting of toucans playing basketball in the style of Egyptian hieroglyphics World Knowledge Style & Format +A photo of an Athenian vase with a painting of toucans playing soccer in the style of Egyptian hieroglyphics World Knowledge Style & Format +A photo of an Athenian vase with a painting of toucans playing tennis in the style of Egyptian hieroglyphics World Knowledge Style & Format +A shiny VW van in front of a cityscape. A smiling sloth stands on grass in front of the van and is wearing a leather jacket, a cowboy hat, a kilt and a bowtie. The sloth is holding a quarterstaff and a big book. High-contrast oil painting. World Knowledge Style & Format +A shiny VW van that has flowers painted on it. A smiling sloth stands on grass in front of the van and is wearing a leather jacket, a cowboy hat, a kilt and a bowtie. The sloth is holding a quarterstaff and a big book. ink sketch. World Knowledge Style & Format +Wood engraving of the Greek letter Omega World Knowledge Style & Format +a satellite image of a costal french city there is a large park on the west side and a mountain to the north. There is a cloud covering part of the image Illustrations Style & Format +A gundam stands tall with its sword raised. A city with tall skyscrapers is in the distance, with a mountain and ocean in the background. A dark moon is in the sky. realistic high-contrast anime illustration. World Knowledge Style & Format +Hokusai's version of Animal Farm Arts Style & Format +A photograph in a temple of a wall painting depicting pandas playing tennis, in the style of Egyptian tomb hieroglyphics World Knowledge Style & Format +an old raccoon wearing a top hat and holding an apple, oil painting in the style of van gogh Arts Style & Format +portrait of a well-dressed raccoon, oil painting in the style of Rembrandt Arts Style & Format +close-up portrait of a smiling businesswoman holding a cell phone, oil painting in the style of Rembrandt People Style & Format +a portrait of a man wearing sunglasses and a business suit, painting in pop art style People Style & Format +a sport car melting into a clock, surrealist painting in the style of Salvador Dali Arts Style & Format +a stained glass window depicting a calm tyrannosaurus rex Illustrations Style & Format +a tennis match in the style of Egyptian hieroglyphic tomb paintings World Knowledge Style & Format +vibrant portrait painting of Salvador Dalí with a robotic half face People Style & Format DALL-E 2 +an espresso machine that makes coffee from human souls, high-contrast painting Artifacts Style & Format DALL-E 2 +panda mad scientist mixing sparkling chemicals, high-contrast painting Animals Style & Format DALL-E 2 +a dolphin in an astronaut suit on saturn, high-contrast painting Animals Style & Format DALL-E 2 +The Oriental Pearl in sketch style World Knowledge Style & Format CogView +The Oriental Pearl in Chinese painting World Knowledge Style & Format CogView +The Oriental Pearl in oil painting World Knowledge Style & Format CogView +a capybara made of voxels sitting in a field Animals Style & Format DALL-E +an emoji of a baby penguin wearing a blue hat, red gloves, green shirt, and yellow pants Illustrations Style & Format DALL-E +a professional high quality emoji of a lovestruck cup of boba Illustrations Style & Format DALL-E +a dutch baroque painting of a horse in a field of flowers Arts Style & Format +Icon of a red heart Illustrations Style & Format VQ-Diffusion +A cartoon house with red roof Illustrations Style & Format VQ-Diffusion +A cartoon tiger face Animals Style & Format VQ-Diffusion +Face of an orange frog in cartoon style Animals Style & Format VQ-Diffusion +A vector illustration of a tree Illustrations Style & Format VQ-Diffusion +an abstract painting of the Empire State Building World Knowledge Style & Format +an abstract painting of the Sydney Opera House World Knowledge Style & Format +a painting of a white country home with a wrap-around porch Outdoor Scenes Style & Format +an oil painting of a cat playing checkers Arts Style & Format +a surrealist dream-like oil painting by salvador dalí of a cat playing checkers Arts Style & Format GLIDE +an oil painting of a hamster dragon Arts Style & Format GLIDE +a high-quality oil painting of a psychedelic hamster dragon Arts Style & Format GLIDE +a painting of a fox in the style of starry night Arts Style & Format GLIDE +a stained glass window of a panda eating bamboo Artifacts Style & Format GLIDE +a crayon drawing of a space elevator Vehicles Style & Format GLIDE +a futuristic city in synthwave style Outdoor Scenes Style & Format GLIDE +a pixel art corgi pizza Illustrations Style & Format GLIDE +a green clock in the shape of a pentagon Artifacts Style & Format DALL-E +a painting of a phone from the 20s Artifacts Style & Format DALL-E +a painting of the food of china Food & Beverage Style & Format DALL-E +a photograph of a bust of homer People Style & Format DALL-E +a drawing of a bust of homer People Style & Format DALL-E +an armchair in the shape of an avocado Artifacts Style & Format DALL-E +a painting of the skyline of New York City World Knowledge Style & Format +a drawing of the skyline of New York City World Knowledge Style & Format +a cartoon of a train going to the moon Vehicles Style & Format +a cartoon of a cow jumping over the moon Animals Style & Format +orange jello in the shape of a man Food & Beverage Style & Format +a comic about a friendly car in the city Artifacts Style & Format +a painting of street in Paris World Knowledge Style & Format +a painting of a canal in Venice World Knowledge Style & Format +an abstract painting with blue, red and black Arts Style & Format +an abstract painting of a waterfall Arts Style & Format +an abstract painting of a spaceship Arts Style & Format +an abstract painting of a pond with a bridge Arts Style & Format +a cartoon of an angry shark Animals Style & Format +a cartoon of a happy car on the road Vehicles Style & Format +a cartoon of a bear birthday party Animals Style & Format +a drawing of a house on a mountain Arts Style & Format +a cartoon of a house on a mountain Illustrations Style & Format +a painting of a house on a mountain Arts Style & Format +an abstract painting of a house on a mountain Arts Style & Format +a painting of a peaceful lakeside landscape Arts Style & Format +a drawing of a peaceful lakeside landscape Arts Style & Format +a map of Australia World Knowledge Style & Format +a map of Italy World Knowledge Style & Format +a map of South America World Knowledge Style & Format +a map of Texas World Knowledge Style & Format +a map of Manhattan World Knowledge Style & Format +an impressionistic painting of tree and a building Arts Style & Format +an abstract painting of a tree and a building Arts Style & Format +a watercolor painting of a tree and a building Arts Style & Format +an oil painting of a tree and a building Arts Style & Format +an abstract painting of three triangles in blue, red and white Arts Style & Format +an abstract painting of three triangles in blue, yellow and red Arts Style & Format +an abstract painting of three squares in blue, red and white Arts Style & Format +an abstract painting of three squares in blue, yellow and red Arts Style & Format +a painting of a cute owl on a box Arts Style & Format +a horse in a field in Minecraft style Animals Style & Format +a drawing of a pig face with an eye patch Illustrations Style & Format +a drawing of a hammer Illustrations Style & Format +a drawing of a screwdriver Illustrations Style & Format +a drawing of a handsaw Illustrations Style & Format +a drawing of a power drill Illustrations Style & Format +a cloud in the shape of a teacup Outdoor Scenes Style & Format +a cloud in the shape of a elephant Outdoor Scenes Style & Format +a cloud in the shape of a castle Outdoor Scenes Style & Format +a footprint shaped like a peanut Artifacts Style & Format +a drawing of a stork playing a violin Illustrations Style & Format +The Statue of Liberty in Minecraft World Knowledge Style & Format +An oil painting of the Statue of Liberty World Knowledge Style & Format +An abstract painting of the Statue of Liberty World Knowledge Style & Format +a diagram of brain function Illustrations Style & Format +a diagram of the human digestive system Illustrations Style & Format +a diagram of a suburban house Illustrations Style & Format +a stop sign with a blue background Artifacts Style & Format +a painting of a sport car in the style of Monet Arts Style & Format +a painting of a sport car in the style of Dali Arts Style & Format +a drawing of a space shuttle in the style of da Vinci Arts Style & Format +a coloring book page of a horse next to a stream Artifacts Style & Format +a sketch of a horse Illustrations Style & Format +a sketch of a skyscraper Illustrations Style & Format +a sketch of a train Illustrations Style & Format +a sketch of a camel next to a stream Illustrations Style & Format +a diagram of the star constellations Illustrations Style & Format +a thumbnail image of an ice cream cone Illustrations Style & Format +a thumbnail image of a horse and cart Illustrations Style & Format +background pattern with alternating roses and skulls Illustrations Style & Format +graffiti of a funny dog on a street wall Illustrations Style & Format +graffiti of a rocket ship on a brick wall Illustrations Style & Format +a watercolor painting of a snowy owl standing in a grassy field Arts Style & Format +a cute illustration of a horned owl with a graduation cap and diploma Illustrations Style & Format +a cardboard spaceship Vehicles Style & Format +the flag of the United Kingdom painted in rusty corrugated iron World Knowledge Style & Format +a diagram of the solar system Illustrations Style & Format +a drawing of a pint of beer on a brick wall Illustrations Style & Format +an abstract painting of the lights at Times Square World Knowledge Style & Format +an abstract drawing of the Great Wall World Knowledge Style & Format +a diagram of the Great Wall World Knowledge Style & Format +a map showing the Great Wall World Knowledge Style & Format +a still life painting of a pair of shoes Artifacts Style & Format +an impressionist painting of the geyser Old Faithful World Knowledge Style & Format +a diagram of how the geyser Old Faithful works World Knowledge Style & Format +an abstract painting of the Great Pyramid World Knowledge Style & Format +a diagram of the inside of the Great Pyramid World Knowledge Style & Format +the silhouette of the Milllenium Wheel at dusk World Knowledge Style & Format +a painting of ten children on a couch People Style & Format +a painting of a man standing on a street corner People Style & Format +a painting of a man standing under a tree People Style & Format +a drawing of a man standing under a tree People Style & Format +a cartoon of a man standing under a tree People Style & Format +a comic about a boy going to school People Style & Format +a comic about a girl going to a farm People Style & Format +a comic about a boy and a tiger People Style & Format +a comic about a family on a road trip People Style & Format +a comic about a father and a son playing tennis People Style & Format +a cartoon of a boy playing with a tiger People Style & Format +the mona lisa People Style & Format +a painting of the Mona Lisa with a frown People Style & Format +a painting of the Mona Lisa with New York City behind her People Style & Format +the Mona Lisa in the style of Minecraft People Style & Format +a thumbnail image of a person skiing People Style & Format +a thumbnail image of a gingerbread man Illustrations Style & Format +the cover of The Beatle's album Revolver World Knowledge Style & Format +the saying "BE EXCELLENT TO EACH OTHER" on a rough wall with a graffiti image of a green alien wearing a tuxedo. Outdoor Scenes Writing & Symbols +A green sign that says "Very Deep Learning" and is at the edge of the Grand Canyon. World Knowledge Writing & Symbols +Portrait of a tiger wearing a train conductor's hat and holding a skateboard that has a yin-yang symbol on it. charcoal sketch Animals Writing & Symbols +Portrait of a tiger wearing a train conductor's hat and holding a skateboard that has a yin-yang symbol on it. child's crayon drawing Animals Writing & Symbols +Portrait of a tiger wearing a train conductor's hat and holding a skateboard that has a yin-yang symbol on it. Chinese ink and wash painting Animals Writing & Symbols +Portrait of a tiger wearing a train conductor's hat and holding a skateboard that has a yin-yang symbol on it. color ink-and-wash drawing Animals Writing & Symbols +Portrait of a tiger wearing a train conductor's hat and holding a skateboard that has a yin-yang symbol on it. comic book illustration Animals Writing & Symbols +Portrait of a tiger wearing a train conductor's hat and holding a skateboard that has a yin-yang symbol on it. marble statue Animals Writing & Symbols +Portrait of a tiger wearing a train conductor's hat and holding a skateboard that has a yin-yang symbol on it. oil painting Animals Writing & Symbols +Portrait of a tiger wearing a train conductor's hat and holding a skateboard that has a yin-yang symbol on it. photograph Animals Writing & Symbols +Portrait of a tiger wearing a train conductor's hat and holding a skateboard that has a yin-yang symbol on it. woodcut Animals Writing & Symbols +Two cups of coffee, one with latte art of a map of United States. The other has latte art of a map of Africa. World Knowledge Writing & Symbols +Two cups of coffee, one with latte art of a lovely princess. The other has latte art of a frog. Food & Beverage Writing & Symbols +Two cups of coffee, one with latte art of a heart. The other has latte art of stars. Food & Beverage Writing & Symbols +Two cups of coffee, one with latte art of the Eiffel tower. The other has latte art of the Statue of Liberty. Food & Beverage Writing & Symbols +A sign that says Deep Learning Artifacts Writing & Symbols +A bar of chocolate without a wrapper that has the word "WRAPPER" printed on it. Artifacts Writing & Symbols +A glass of red wine tipped over on a couch, with a stain that writes "OOPS" on the couch. Indoor Scenes Writing & Symbols +Two cups of coffee, one with latte art of yin yang symbol. The other has latte art of a heart. Food & Beverage Writing & Symbols +"G I G G L E" painted in thick colorful lettering as graffiti on a faded red brick wall with a splotch of exploding white paint. Illustrations Writing & Symbols +high-contrast image of the word "WOMBAT" written with thick colored graffiti letters on a white wall with dramatic splashes of paint Illustrations Writing & Symbols +A plush monkey fording the Charles River on a log while wearing a Boston Red Sox hat with MIT in the background. World Knowledge Writing & Symbols +the letters P A X forming a very simple outline of an elephant's shape. the elephant is facing left. vector art, orange logo. Illustrations Writing & Symbols +A burger patty, with the bottom bun and lettuce and tomatoes. "COFFEE" written on it in mustard Food & Beverage Writing & Symbols +Two cups of coffee, one with latte art of the words "LOVE" written in one. The other has latte art of the words "PEACE" written in the other. Artifacts Writing & Symbols +The saying "BE EXCELLENT TO EACH OTHER" written in a stained glass window. Illustrations Writing & Symbols +The saying "BE EXCELLENT TO EACH OTHER" written in faded paint on the hull of an old wooden boat and reflected in the water. Wide-angle lens. Illustrations Writing & Symbols +The saying "BE EXCELLENT TO EACH OTHER" written on a red brick wall with a graffiti image of a green alien wearing a tuxedo. A yellow fire hydrant is on a sidewalk in the foreground. Illustrations Writing & Symbols +The saying "BE EXCELLENT TO EACH OTHER" written with carved letters on driftwood. Illustrations Writing & Symbols +a hot air balloon with chameleon logo. the sun is shining and puffy white clouds are in the background. Vehicles Writing & Symbols +A portrait photo of a kangaroo wearing an orange hoodie and blue sunglasses standing on the grassin front of the Sydney Opera House holding a sign on the chest that says Welcome Friends! World Knowledge Writing & Symbols +Anime illustration of a kangaroo holding a sign that says "Starry Night", in front of the Sydney Opera House sitting next to the Eiffel Tower under a blue night sky of roiling energy, exploding yellow stars, and radiating swirls of blu World Knowledge Writing & Symbols +a group of cats in a meeting. there is a whiteboard with "stack more layers" written on it. Indoor Scenes Writing & Symbols +A high contrast portrait photo of a fluffy hamster wearing an orange beanie and sunglasses holding a sign that says "Let's PAINT!" Animals Writing & Symbols +a cartoon of a dog saying "I see what you did there" Animals Writing & Symbols +a robot holding a sign with "Let's PAINT!" written on it Artifacts Writing & Symbols +a dog wearing a baseball cap backwards and writing BONEZ on a chalkboard Animals Writing & Symbols +graffiti spelling BE KIND on white subway tile Illustrations Writing & Symbols +a grumpy porcupine handing a check for $10,000 to a smiling peacock Animals Writing & Symbols +a boat with 'BLUE GROOVE' written on its hull Vehicles Writing & Symbols +a store front that has the word ‘openai’ written on it. Outdoor Scenes Writing & Symbols DALL-E +a laptop with a maze sticker on it Artifacts Writing & Symbols +a yellow t-shirt with a dog on it Artifacts Writing & Symbols +a blue t-shirt with a dinosaur on it Artifacts Writing & Symbols +a bowl with a dinosaur on it Artifacts Writing & Symbols +a bowl with a cartoon dinosaur on it Artifacts Writing & Symbols +a flag with a dinosaur on it Illustrations Writing & Symbols +a flag with a drawing of a dinosaur on it Artifacts Writing & Symbols +a flag with three vertical stripes Illustrations Writing & Symbols +a flag with three vertical stripes that are blue, white and red Illustrations Writing & Symbols +a white towel with a cartoon of a cat on it Artifacts Writing & Symbols +a black towel with a cartoon of a dog on it Artifacts Writing & Symbols +a yellow wall with the word KA-BOOM on it Artifacts Writing & Symbols +a cat sitting on a box with a drawing of an elephant on it Animals Writing & Symbols +the word 'mardefly' on a coffee mug Artifacts Writing & Symbols +a stop sigh with 'ALL WAY' written below it Artifacts Writing & Symbols +a sticker stuck in the middle of a stop sign Artifacts Writing & Symbols +world's best brother t-shirt Artifacts Writing & Symbols +the cover of a book called 'Backpropaganda' by I.C. Gradients Illustrations Writing & Symbols +the saying 'do unto others as they would do unto you' written on a white background Artifacts Writing & Symbols +a t-shirt with Carpe Diem written on it Artifacts Writing & Symbols +a sweatshirt with 'Batch Norm' written on it Artifacts Writing & Symbols +a yellow sticky note with 'BUY MILK' written on it Artifacts Writing & Symbols +a large open book showing text and an illustration of a cat Artifacts Writing & Symbols +a yellow wall with 'Hello, world.' written on it Indoor Scenes Writing & Symbols +a motorcycle parked in an ornate bank lobby with "BUZZ" written on its body Vehicles Writing & Symbols +a logo of a wombat on a coffee mug Artifacts Writing & Symbols +a coffee mug with an ankh symbol on it Artifacts Writing & Symbols +a t-shirt with 'ANKH' written on it Artifacts Writing & Symbols +a t-shirt with 'Archaelogy Rocks!' written on it Artifacts Writing & Symbols +a black t-shirt with the peace sign on it Artifacts Writing & Symbols +a store front with 'Grassy Meadow' written on it Outdoor Scenes Writing & Symbols +a store front with 'AwesomePurchase' written on it Outdoor Scenes Writing & Symbols +a laptop screen showing an internet search Artifacts Writing & Symbols +a laptop screen showing a bunch of photographs Artifacts Writing & Symbols +a laptop screen showing a document being edited Artifacts Writing & Symbols +a book with the words 'Don't Panic!' written on it Artifacts Writing & Symbols +the words 'KEEP OFF THE GRASS' Illustrations Writing & Symbols +the words 'KEEP OFF THE GRASS' written on a brick wall Indoor Scenes Writing & Symbols +the words 'KEEP OFF THE GRASS' on a black sticker Illustrations Writing & Symbols +the words 'KEEP OFF THE GRASS' on a sign next to a lawn Outdoor Scenes Writing & Symbols +the word 'START' Illustrations Writing & Symbols +the word 'START' written on a street surface Illustrations Writing & Symbols +the word 'START' on a blue t-shirt Artifacts Writing & Symbols +the word 'START' written above the word 'SMILING' Illustrations Writing & Symbols +the word 'START' written in chalk on a sidewalk Outdoor Scenes Writing & Symbols +a wooden post with a yellow '3' painted on top Outdoor Scenes Writing & Symbols +a wooden post in front of a patch of tall grass Outdoor Scenes Writing & Symbols +a wooden post with a blue '5' painted on top Outdoor Scenes Writing & Symbols +a series of musical notes on a black t-shirt Artifacts Writing & Symbols +a series of musical notes on a computer screen Artifacts Writing & Symbols \ No newline at end of file diff --git a/fourm/utils/generation_datasets/__init__.py b/fourm/utils/generation_datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f1cf9252db30f26a8293fe8c9f695a87c47f46cd --- /dev/null +++ b/fourm/utils/generation_datasets/__init__.py @@ -0,0 +1,3 @@ +from .parti_prompts_dataset import * +from .empty_dataset import * +from .image_caption_dataset import * \ No newline at end of file diff --git a/fourm/utils/generation_datasets/empty_dataset.py b/fourm/utils/generation_datasets/empty_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..ad277d7d51c483ba93670adaca30c9850967a0f6 --- /dev/null +++ b/fourm/utils/generation_datasets/empty_dataset.py @@ -0,0 +1,27 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +from torch.utils.data import Dataset + +class EmptyDataset(Dataset): + """Empty dataset""" + + def __init__(self, dataset_size: int): + self.dataset_size = dataset_size + + def __getitem__(self, index): + return {} + + def __len__(self): + return self.dataset_size \ No newline at end of file diff --git a/fourm/utils/generation_datasets/image_caption_dataset.py b/fourm/utils/generation_datasets/image_caption_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..9f432c9c1232bc136662d7fddaf6905de81619b7 --- /dev/null +++ b/fourm/utils/generation_datasets/image_caption_dataset.py @@ -0,0 +1,99 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from torch.utils.data import Dataset +from typing import Any, Callable, Dict, List, Optional, Tuple, cast + +from fourm.data.multimodal_dataset_folder import make_dataset, UNIFIED_EXTENSIONS +from fourm.data.modality_transforms import get_transform_key, RGBTransform, CaptionTransform, UnifiedDataTransform + + +class ImageCaptionDataset(Dataset): + """ + Similar to MultiModalDatasetFolder, but specialized for image-caption datasets. + """ + def __init__(self, + root: str, + augmenter: Optional[Callable] = None, + modality_paths: Dict[str, str] = None, + is_valid_file: Optional[Callable[[str], bool]] = None, + cache=False): + self.root = root + self.modality_paths = modality_paths or {} + + self.modality_transforms = { + 'rgb': RGBTransform(imagenet_default_mean_and_std=False), + 'caption': CaptionTransform() + } + + self.transform = UnifiedDataTransform(transforms_dict=self.modality_transforms, image_augmenter=augmenter) + + classes, class_to_idx = self._find_classes(os.path.join(self.root, self.modality_paths.get('caption', 'caption'))) + extensions = UNIFIED_EXTENSIONS if is_valid_file is None else None + + samples = { + mod: make_dataset( + os.path.join(self.root, self.modality_paths.get(mod, mod)), + class_to_idx, + extensions, + is_valid_file, + cache_path=os.path.join(self.root, 'dataloader_cache', f'{self.modality_paths.get(mod, mod)}.pkl') if cache else None) + for mod in ['caption', 'rgb'] + } + + for mod, mod_samples in samples.items(): + if len(mod_samples) == 0: + msg = "Found 0 logs in subfolders of: {}\n".format(os.path.join(self.root, self.modality_paths.get(mod, mod))) + if extensions is not None: + msg += "Supported extensions are: {}".format(",".join(extensions)) + raise RuntimeError(msg) + + self.extensions = extensions + self.classes = classes + self.class_to_idx = class_to_idx + self.samples = samples + + def _find_classes(self, dir: str) -> Tuple[List[str], Dict[str, int]]: + """ + Finds the class folders in a dataset. + + Args: + dir (string): Root directory path. + + Returns: + tuple: (classes, class_to_idx) where classes are relative to (dir), and class_to_idx is a dictionary. + + Ensures: + No class is a subdirectory of another. + """ + classes = [d.name for d in os.scandir(dir) if d.is_dir()] + classes.sort() + class_to_idx = {cls_name: i for i, cls_name in enumerate(classes)} + return classes, class_to_idx + + def __getitem__(self, index): + + sample_dict = {} + for mod in ['caption', 'rgb']: + path, _ = self.samples[mod][index] + sample = self.modality_transforms[get_transform_key(mod)].load(path) + sample_dict[mod] = sample + + if self.transform is not None: + sample_dict = self.transform(sample_dict) + + return sample_dict + + def __len__(self) -> int: + return len(list(self.samples.values())[0]) \ No newline at end of file diff --git a/fourm/utils/generation_datasets/parti_prompts_dataset.py b/fourm/utils/generation_datasets/parti_prompts_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..79712d4d23a190539cb1519df00c7daaa8cf2754 --- /dev/null +++ b/fourm/utils/generation_datasets/parti_prompts_dataset.py @@ -0,0 +1,114 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +import pandas as pd +import numpy as np +from torch.utils.data import Dataset + + +class PartiPromptsDataset(Dataset): + """ + Parti Prompts caption dataset. + + Args: + text_tokenizer (tokenizers.Tokenizer): The tokenizer to use for encoding the captions. + max_length (int): The maximum sequence length of the captions. + parti_prompts_csv (str): The path to the Parti Prompts dataset. + """ + def __init__(self, text_tokenizer, max_length=128, parti_prompts_csv='fourm/utils/generation_datasets/PartiPrompts.tsv', parti_prompts_t5_embs=None, llm_embedder=None): + self.text_tokenizer = text_tokenizer + self.max_length = max_length + self.parti_prompts = pd.read_csv(parti_prompts_csv, sep='\t') + + self.pad_id = text_tokenizer.token_to_id("[PAD]") + self.eos_id = text_tokenizer.token_to_id("[EOS]") + if parti_prompts_t5_embs is not None: + # T5 Embeddings are saved as a numpy array, so we need to load it + self.t5_embs = np.load(parti_prompts_t5_embs)['emb'] + self.t5_masks = np.load(parti_prompts_t5_embs)['mask_valid'] + self.llm_embedder = None + elif llm_embedder is not None: + self.t5_embs = None + self.llm_embedder = llm_embedder + else: + self.t5_embs = None + self.llm_embedder = None + + def __getitem__(self, index): + text = self.parti_prompts.Prompt[index] + seq_ids = self.text_tokenizer.encode(text).ids + [self.eos_id] + + tensor = torch.ones(self.max_length, dtype=torch.int) * self.pad_id + tensor[:len(seq_ids)] = torch.tensor(seq_ids, dtype=torch.int) + + out = {} + out['caption'] = {'tensor': tensor} + + if self.t5_embs is not None: + + t5_emb = torch.tensor(self.t5_embs[index], dtype=torch.float32) + t5_emb = pad_or_truncate(t5_emb, self.max_length) + + t5_mask = torch.tensor(self.t5_masks[index], dtype=torch.bool) + t5_mask = pad_or_truncate(t5_mask, self.max_length) + + ascii_tensor = text_to_tensor(text, max_length=self.max_length * 10) # Save ASCII as tensor + + out['t5_caption'] = { + 'tensor': t5_emb, + 'mask_valid': t5_mask, + 'ascii_tensor': ascii_tensor, + } + elif self.llm_embedder is not None: + t5_emb, _, t5_mask = self.llm_embedder.get_text_embeddings([text]) + t5_emb = pad_or_truncate(t5_emb.squeeze(0), self.max_length) + t5_mask = pad_or_truncate(t5_mask.bool().squeeze(0), self.max_length) + ascii_tensor = text_to_tensor(text, max_length=self.max_length * 10) # Save ASCII as tensor + + out['t5_caption'] = { + 'tensor': t5_emb, + 'mask_valid': t5_mask, + 'ascii_tensor': ascii_tensor, + } + + return out + + def __len__(self): + return len(self.parti_prompts) + + +def pad_or_truncate(tensor, fixed_length, padding_value=0): + current_length = tensor.shape[0] + + if current_length < fixed_length: + # Calculate padding sizes for all dimensions, but only pad along dim=0 + padding_sizes = [0] * 2 * len(tensor.shape) + padding_sizes[1] = fixed_length - current_length + return torch.nn.functional.pad(tensor, padding_sizes, 'constant', padding_value) + else: + return tensor[:fixed_length] + +def text_to_tensor(text, max_length=None): + """Converts plaintext to a tensor with optional padding.""" + ascii_values = [ord(c) for c in text] + if max_length: + while len(ascii_values) < max_length: + ascii_values.append(0) # Using 0 as the padding value + return torch.tensor(ascii_values, dtype=torch.int) + + +def tensor_to_text(tensor): + """Converts tensor back to plaintext. Assumes padding with zeros.""" + ascii_values = tensor.tolist() + return ''.join(chr(val) for val in ascii_values if val != 0) diff --git a/fourm/utils/hmr2_utils/hmr2/__init__.py b/fourm/utils/hmr2_utils/hmr2/__init__.py new file mode 100755 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/fourm/utils/hmr2_utils/hmr2/models/__init__.py b/fourm/utils/hmr2_utils/hmr2/models/__init__.py new file mode 100755 index 0000000000000000000000000000000000000000..d81169998d524007a689e5d6020462d589c5d35c --- /dev/null +++ b/fourm/utils/hmr2_utils/hmr2/models/__init__.py @@ -0,0 +1,2 @@ +from .smpl_wrapper import SMPL +from .hmr2 import HMR2 diff --git a/fourm/utils/hmr2_utils/hmr2/models/backbones/__init__.py b/fourm/utils/hmr2_utils/hmr2/models/backbones/__init__.py new file mode 100755 index 0000000000000000000000000000000000000000..69d1a138ee9d0eb5f5104bb04f33fb4ed7b21166 --- /dev/null +++ b/fourm/utils/hmr2_utils/hmr2/models/backbones/__init__.py @@ -0,0 +1,13 @@ +# -------------------------------------------------------- +# Based on the ViTPose and 4DHumans code bases +# https://github.com/ViTAE-Transformer/ViTPose/ +# https://github.com/shubham-goel/4D-Humans +# -------------------------------------------------------- + +from .vit import vit + +def create_backbone(cfg): + if cfg.MODEL.BACKBONE.TYPE == 'vit': + return vit(cfg) + else: + raise NotImplementedError('Backbone type is not implemented') diff --git a/fourm/utils/hmr2_utils/hmr2/models/backbones/vit.py b/fourm/utils/hmr2_utils/hmr2/models/backbones/vit.py new file mode 100755 index 0000000000000000000000000000000000000000..a54b0c2b9808bfab75505548af878363d3c00594 --- /dev/null +++ b/fourm/utils/hmr2_utils/hmr2/models/backbones/vit.py @@ -0,0 +1,353 @@ +# -------------------------------------------------------- +# Based on the ViTPose and 4DHumans code bases +# https://github.com/ViTAE-Transformer/ViTPose/ +# https://github.com/shubham-goel/4D-Humans +# -------------------------------------------------------- + +import math + +import torch +from functools import partial +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint + +from timm.models.layers import drop_path, to_2tuple, trunc_normal_ + +def vit(cfg): + return ViT( + img_size=(256, 192), + patch_size=16, + embed_dim=1280, + depth=32, + num_heads=16, + ratio=1, + use_checkpoint=False, + mlp_ratio=4, + qkv_bias=True, + drop_path_rate=0.55, + ) + +def get_abs_pos(abs_pos, h, w, ori_h, ori_w, has_cls_token=True): + """ + Calculate absolute positional embeddings. If needed, resize embeddings and remove cls_token + dimension for the original embeddings. + Args: + abs_pos (Tensor): absolute positional embeddings with (1, num_position, C). + has_cls_token (bool): If true, has 1 embedding in abs_pos for cls token. + hw (Tuple): size of input image tokens. + + Returns: + Absolute positional embeddings after processing with shape (1, H, W, C) + """ + cls_token = None + B, L, C = abs_pos.shape + if has_cls_token: + cls_token = abs_pos[:, 0:1] + abs_pos = abs_pos[:, 1:] + + if ori_h != h or ori_w != w: + new_abs_pos = F.interpolate( + abs_pos.reshape(1, ori_h, ori_w, -1).permute(0, 3, 1, 2), + size=(h, w), + mode="bicubic", + align_corners=False, + ).permute(0, 2, 3, 1).reshape(B, -1, C) + + else: + new_abs_pos = abs_pos + + if cls_token is not None: + new_abs_pos = torch.cat([cls_token, new_abs_pos], dim=1) + return new_abs_pos + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + def extra_repr(self): + return 'p={}'.format(self.drop_prob) + +class Mlp(nn.Module): + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.fc2(x) + x = self.drop(x) + return x + +class Attention(nn.Module): + def __init__( + self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., + proj_drop=0., attn_head_dim=None,): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.dim = dim + + if attn_head_dim is not None: + head_dim = attn_head_dim + all_head_dim = head_dim * self.num_heads + + self.scale = qk_scale or head_dim ** -0.5 + + self.qkv = nn.Linear(dim, all_head_dim * 3, bias=qkv_bias) + + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(all_head_dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x): + B, N, C = x.shape + qkv = self.qkv(x) + qkv = qkv.reshape(B, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) + + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, -1) + x = self.proj(x) + x = self.proj_drop(x) + + return x + +class Block(nn.Module): + + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, + drop=0., attn_drop=0., drop_path=0., act_layer=nn.GELU, + norm_layer=nn.LayerNorm, attn_head_dim=None + ): + super().__init__() + + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, + attn_drop=attn_drop, proj_drop=drop, attn_head_dim=attn_head_dim + ) + + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + def forward(self, x): + x = x + self.drop_path(self.attn(self.norm1(x))) + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + + +class PatchEmbed(nn.Module): + """ Image to Patch Embedding + """ + def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768, ratio=1): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + num_patches = (img_size[1] // patch_size[1]) * (img_size[0] // patch_size[0]) * (ratio ** 2) + self.patch_shape = (int(img_size[0] // patch_size[0] * ratio), int(img_size[1] // patch_size[1] * ratio)) + self.origin_patch_shape = (int(img_size[0] // patch_size[0]), int(img_size[1] // patch_size[1])) + self.img_size = img_size + self.patch_size = patch_size + self.num_patches = num_patches + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=(patch_size[0] // ratio), padding=4 + 2 * (ratio//2-1)) + + def forward(self, x, **kwargs): + B, C, H, W = x.shape + x = self.proj(x) + Hp, Wp = x.shape[2], x.shape[3] + + x = x.flatten(2).transpose(1, 2) + return x, (Hp, Wp) + + +class HybridEmbed(nn.Module): + """ CNN Feature Map Embedding + Extract feature map from CNN, flatten, project to embedding dim. + """ + def __init__(self, backbone, img_size=224, feature_size=None, in_chans=3, embed_dim=768): + super().__init__() + assert isinstance(backbone, nn.Module) + img_size = to_2tuple(img_size) + self.img_size = img_size + self.backbone = backbone + if feature_size is None: + with torch.no_grad(): + training = backbone.training + if training: + backbone.eval() + o = self.backbone(torch.zeros(1, in_chans, img_size[0], img_size[1]))[-1] + feature_size = o.shape[-2:] + feature_dim = o.shape[1] + backbone.train(training) + else: + feature_size = to_2tuple(feature_size) + feature_dim = self.backbone.feature_info.channels()[-1] + self.num_patches = feature_size[0] * feature_size[1] + self.proj = nn.Linear(feature_dim, embed_dim) + + def forward(self, x): + x = self.backbone(x)[-1] + x = x.flatten(2).transpose(1, 2) + x = self.proj(x) + return x + + +class ViT(nn.Module): + + def __init__(self, + img_size=224, patch_size=16, in_chans=3, num_classes=80, embed_dim=768, depth=12, + num_heads=12, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0., + drop_path_rate=0., hybrid_backbone=None, norm_layer=None, use_checkpoint=False, + frozen_stages=-1, ratio=1, last_norm=True, + patch_padding='pad', freeze_attn=False, freeze_ffn=False, + ): + # Protect mutable default arguments + super(ViT, self).__init__() + norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) + self.num_classes = num_classes + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + self.frozen_stages = frozen_stages + self.use_checkpoint = use_checkpoint + self.patch_padding = patch_padding + self.freeze_attn = freeze_attn + self.freeze_ffn = freeze_ffn + self.depth = depth + + if hybrid_backbone is not None: + self.patch_embed = HybridEmbed( + hybrid_backbone, img_size=img_size, in_chans=in_chans, embed_dim=embed_dim) + else: + self.patch_embed = PatchEmbed( + img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim, ratio=ratio) + num_patches = self.patch_embed.num_patches + + # since the pretraining model has class token + self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim)) + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule + + self.blocks = nn.ModuleList([ + Block( + dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, + ) + for i in range(depth)]) + + self.last_norm = norm_layer(embed_dim) if last_norm else nn.Identity() + + if self.pos_embed is not None: + trunc_normal_(self.pos_embed, std=.02) + + self._freeze_stages() + + def _freeze_stages(self): + """Freeze parameters.""" + if self.frozen_stages >= 0: + self.patch_embed.eval() + for param in self.patch_embed.parameters(): + param.requires_grad = False + + for i in range(1, self.frozen_stages + 1): + m = self.blocks[i] + m.eval() + for param in m.parameters(): + param.requires_grad = False + + if self.freeze_attn: + for i in range(0, self.depth): + m = self.blocks[i] + m.attn.eval() + m.norm1.eval() + for param in m.attn.parameters(): + param.requires_grad = False + for param in m.norm1.parameters(): + param.requires_grad = False + + if self.freeze_ffn: + self.pos_embed.requires_grad = False + self.patch_embed.eval() + for param in self.patch_embed.parameters(): + param.requires_grad = False + for i in range(0, self.depth): + m = self.blocks[i] + m.mlp.eval() + m.norm2.eval() + for param in m.mlp.parameters(): + param.requires_grad = False + for param in m.norm2.parameters(): + param.requires_grad = False + + def init_weights(self): + """Initialize the weights in backbone. + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + def _init_weights(m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + self.apply(_init_weights) + + def get_num_layers(self): + return len(self.blocks) + + @torch.jit.ignore + def no_weight_decay(self): + return {'pos_embed', 'cls_token'} + + def forward_features(self, x): + B, C, H, W = x.shape + x, (Hp, Wp) = self.patch_embed(x) + + if self.pos_embed is not None: + # fit for multiple GPU training + # since the first element for pos embed (sin-cos manner) is zero, it will cause no difference + x = x + self.pos_embed[:, 1:] + self.pos_embed[:, :1] + + for blk in self.blocks: + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x) + else: + x = blk(x) + + x = self.last_norm(x) + + xp = x.permute(0, 2, 1).reshape(B, -1, Hp, Wp).contiguous() + + return xp + + def forward(self, x): + x = self.forward_features(x) + return x + + def train(self, mode=True): + """Convert the model into training mode.""" + super().train(mode) + self._freeze_stages() diff --git a/fourm/utils/hmr2_utils/hmr2/models/components/__init__.py b/fourm/utils/hmr2_utils/hmr2/models/components/__init__.py new file mode 100755 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/fourm/utils/hmr2_utils/hmr2/models/components/pose_transformer.py b/fourm/utils/hmr2_utils/hmr2/models/components/pose_transformer.py new file mode 100755 index 0000000000000000000000000000000000000000..9eea7de21b2289f1bc1838695d7591989d967dc5 --- /dev/null +++ b/fourm/utils/hmr2_utils/hmr2/models/components/pose_transformer.py @@ -0,0 +1,363 @@ +# -------------------------------------------------------- +# Based on the 4DHumans code base +# https://github.com/shubham-goel/4D-Humans +# -------------------------------------------------------- + +from inspect import isfunction +from typing import Callable, Optional + +import torch +from einops import rearrange +from einops.layers.torch import Rearrange +from torch import nn + +from .t_cond_mlp import ( + AdaptiveLayerNorm1D, + FrequencyEmbedder, + normalization_layer, +) +# from .vit import Attention, FeedForward + + +def exists(val): + return val is not None + + +def default(val, d): + if exists(val): + return val + return d() if isfunction(d) else d + + +class PreNorm(nn.Module): + def __init__(self, dim: int, fn: Callable, norm: str = "layer", norm_cond_dim: int = -1): + super().__init__() + self.norm = normalization_layer(norm, dim, norm_cond_dim) + self.fn = fn + + def forward(self, x: torch.Tensor, *args, **kwargs): + if isinstance(self.norm, AdaptiveLayerNorm1D): + return self.fn(self.norm(x, *args), **kwargs) + else: + return self.fn(self.norm(x), **kwargs) + + +class FeedForward(nn.Module): + def __init__(self, dim, hidden_dim, dropout=0.0): + super().__init__() + self.net = nn.Sequential( + nn.Linear(dim, hidden_dim), + nn.GELU(), + nn.Dropout(dropout), + nn.Linear(hidden_dim, dim), + nn.Dropout(dropout), + ) + + def forward(self, x): + return self.net(x) + + +class Attention(nn.Module): + def __init__(self, dim, heads=8, dim_head=64, dropout=0.0): + super().__init__() + inner_dim = dim_head * heads + project_out = not (heads == 1 and dim_head == dim) + + self.heads = heads + self.scale = dim_head**-0.5 + + self.attend = nn.Softmax(dim=-1) + self.dropout = nn.Dropout(dropout) + + self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False) + + self.to_out = ( + nn.Sequential(nn.Linear(inner_dim, dim), nn.Dropout(dropout)) + if project_out + else nn.Identity() + ) + + def forward(self, x): + qkv = self.to_qkv(x).chunk(3, dim=-1) + q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=self.heads), qkv) + + dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale + + attn = self.attend(dots) + attn = self.dropout(attn) + + out = torch.matmul(attn, v) + out = rearrange(out, "b h n d -> b n (h d)") + return self.to_out(out) + + +class CrossAttention(nn.Module): + def __init__(self, dim, context_dim=None, heads=8, dim_head=64, dropout=0.0): + super().__init__() + inner_dim = dim_head * heads + project_out = not (heads == 1 and dim_head == dim) + + self.heads = heads + self.scale = dim_head**-0.5 + + self.attend = nn.Softmax(dim=-1) + self.dropout = nn.Dropout(dropout) + + context_dim = default(context_dim, dim) + self.to_kv = nn.Linear(context_dim, inner_dim * 2, bias=False) + self.to_q = nn.Linear(dim, inner_dim, bias=False) + + self.to_out = ( + nn.Sequential(nn.Linear(inner_dim, dim), nn.Dropout(dropout)) + if project_out + else nn.Identity() + ) + + def forward(self, x, context=None): + context = default(context, x) + k, v = self.to_kv(context).chunk(2, dim=-1) + q = self.to_q(x) + q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=self.heads), [q, k, v]) + + dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale + + attn = self.attend(dots) + attn = self.dropout(attn) + + out = torch.matmul(attn, v) + out = rearrange(out, "b h n d -> b n (h d)") + return self.to_out(out) + + +class Transformer(nn.Module): + def __init__( + self, + dim: int, + depth: int, + heads: int, + dim_head: int, + mlp_dim: int, + dropout: float = 0.0, + norm: str = "layer", + norm_cond_dim: int = -1, + ): + super().__init__() + self.layers = nn.ModuleList([]) + for _ in range(depth): + sa = Attention(dim, heads=heads, dim_head=dim_head, dropout=dropout) + ff = FeedForward(dim, mlp_dim, dropout=dropout) + self.layers.append( + nn.ModuleList( + [ + PreNorm(dim, sa, norm=norm, norm_cond_dim=norm_cond_dim), + PreNorm(dim, ff, norm=norm, norm_cond_dim=norm_cond_dim), + ] + ) + ) + + def forward(self, x: torch.Tensor, *args): + for attn, ff in self.layers: + x = attn(x, *args) + x + x = ff(x, *args) + x + return x + + +class TransformerCrossAttn(nn.Module): + def __init__( + self, + dim: int, + depth: int, + heads: int, + dim_head: int, + mlp_dim: int, + dropout: float = 0.0, + norm: str = "layer", + norm_cond_dim: int = -1, + context_dim: Optional[int] = None, + ): + super().__init__() + self.layers = nn.ModuleList([]) + for _ in range(depth): + sa = Attention(dim, heads=heads, dim_head=dim_head, dropout=dropout) + ca = CrossAttention( + dim, context_dim=context_dim, heads=heads, dim_head=dim_head, dropout=dropout + ) + ff = FeedForward(dim, mlp_dim, dropout=dropout) + self.layers.append( + nn.ModuleList( + [ + PreNorm(dim, sa, norm=norm, norm_cond_dim=norm_cond_dim), + PreNorm(dim, ca, norm=norm, norm_cond_dim=norm_cond_dim), + PreNorm(dim, ff, norm=norm, norm_cond_dim=norm_cond_dim), + ] + ) + ) + + def forward(self, x: torch.Tensor, *args, context=None, context_list=None): + if context_list is None: + context_list = [context] * len(self.layers) + if len(context_list) != len(self.layers): + raise ValueError(f"len(context_list) != len(self.layers) ({len(context_list)} != {len(self.layers)})") + + for i, (self_attn, cross_attn, ff) in enumerate(self.layers): + x = self_attn(x, *args) + x + x = cross_attn(x, *args, context=context_list[i]) + x + x = ff(x, *args) + x + return x + + +class DropTokenDropout(nn.Module): + def __init__(self, p: float = 0.1): + super().__init__() + if p < 0 or p > 1: + raise ValueError( + "dropout probability has to be between 0 and 1, " "but got {}".format(p) + ) + self.p = p + + def forward(self, x: torch.Tensor): + # x: (batch_size, seq_len, dim) + if self.training and self.p > 0: + zero_mask = torch.full_like(x[0, :, 0], self.p).bernoulli().bool() + # TODO: permutation idx for each batch using torch.argsort + if zero_mask.any(): + x = x[:, ~zero_mask, :] + return x + + +class ZeroTokenDropout(nn.Module): + def __init__(self, p: float = 0.1): + super().__init__() + if p < 0 or p > 1: + raise ValueError( + "dropout probability has to be between 0 and 1, " "but got {}".format(p) + ) + self.p = p + + def forward(self, x: torch.Tensor): + # x: (batch_size, seq_len, dim) + if self.training and self.p > 0: + zero_mask = torch.full_like(x[:, :, 0], self.p).bernoulli().bool() + # Zero-out the masked tokens + x[zero_mask, :] = 0 + return x + + +class TransformerEncoder(nn.Module): + def __init__( + self, + num_tokens: int, + token_dim: int, + dim: int, + depth: int, + heads: int, + mlp_dim: int, + dim_head: int = 64, + dropout: float = 0.0, + emb_dropout: float = 0.0, + emb_dropout_type: str = "drop", + emb_dropout_loc: str = "token", + norm: str = "layer", + norm_cond_dim: int = -1, + token_pe_numfreq: int = -1, + ): + super().__init__() + if token_pe_numfreq > 0: + token_dim_new = token_dim * (2 * token_pe_numfreq + 1) + self.to_token_embedding = nn.Sequential( + Rearrange("b n d -> (b n) d", n=num_tokens, d=token_dim), + FrequencyEmbedder(token_pe_numfreq, token_pe_numfreq - 1), + Rearrange("(b n) d -> b n d", n=num_tokens, d=token_dim_new), + nn.Linear(token_dim_new, dim), + ) + else: + self.to_token_embedding = nn.Linear(token_dim, dim) + self.pos_embedding = nn.Parameter(torch.randn(1, num_tokens, dim)) + if emb_dropout_type == "drop": + self.dropout = DropTokenDropout(emb_dropout) + elif emb_dropout_type == "zero": + self.dropout = ZeroTokenDropout(emb_dropout) + else: + raise ValueError(f"Unknown emb_dropout_type: {emb_dropout_type}") + self.emb_dropout_loc = emb_dropout_loc + + self.transformer = Transformer( + dim, depth, heads, dim_head, mlp_dim, dropout, norm=norm, norm_cond_dim=norm_cond_dim + ) + + def forward(self, inp: torch.Tensor, *args, **kwargs): + x = inp + + if self.emb_dropout_loc == "input": + x = self.dropout(x) + x = self.to_token_embedding(x) + + if self.emb_dropout_loc == "token": + x = self.dropout(x) + b, n, _ = x.shape + x += self.pos_embedding[:, :n] + + if self.emb_dropout_loc == "token_afterpos": + x = self.dropout(x) + x = self.transformer(x, *args) + return x + + +class TransformerDecoder(nn.Module): + def __init__( + self, + num_tokens: int, + token_dim: int, + dim: int, + depth: int, + heads: int, + mlp_dim: int, + dim_head: int = 64, + dropout: float = 0.0, + emb_dropout: float = 0.0, + emb_dropout_type: str = 'drop', + norm: str = "layer", + norm_cond_dim: int = -1, + context_dim: Optional[int] = None, + skip_token_embedding: bool = False, + ): + super().__init__() + if not skip_token_embedding: + self.to_token_embedding = nn.Linear(token_dim, dim) + else: + self.to_token_embedding = nn.Identity() + if token_dim != dim: + raise ValueError( + f"token_dim ({token_dim}) != dim ({dim}) when skip_token_embedding is True" + ) + + self.pos_embedding = nn.Parameter(torch.randn(1, num_tokens, dim)) + if emb_dropout_type == "drop": + self.dropout = DropTokenDropout(emb_dropout) + elif emb_dropout_type == "zero": + self.dropout = ZeroTokenDropout(emb_dropout) + elif emb_dropout_type == "normal": + self.dropout = nn.Dropout(emb_dropout) + + self.transformer = TransformerCrossAttn( + dim, + depth, + heads, + dim_head, + mlp_dim, + dropout, + norm=norm, + norm_cond_dim=norm_cond_dim, + context_dim=context_dim, + ) + + def forward(self, inp: torch.Tensor, *args, context=None, context_list=None): + x = self.to_token_embedding(inp) + b, n, _ = x.shape + + x = self.dropout(x) + x += self.pos_embedding[:, :n] + + x = self.transformer(x, *args, context=context, context_list=context_list) + return x + diff --git a/fourm/utils/hmr2_utils/hmr2/models/components/t_cond_mlp.py b/fourm/utils/hmr2_utils/hmr2/models/components/t_cond_mlp.py new file mode 100755 index 0000000000000000000000000000000000000000..f3411f410696684d09b981a390f8c4299bd91c02 --- /dev/null +++ b/fourm/utils/hmr2_utils/hmr2/models/components/t_cond_mlp.py @@ -0,0 +1,204 @@ +# -------------------------------------------------------- +# Based on the 4DHumans code base +# https://github.com/shubham-goel/4D-Humans +# -------------------------------------------------------- + +import copy +from typing import List, Optional + +import torch + + +class AdaptiveLayerNorm1D(torch.nn.Module): + def __init__(self, data_dim: int, norm_cond_dim: int): + super().__init__() + if data_dim <= 0: + raise ValueError(f"data_dim must be positive, but got {data_dim}") + if norm_cond_dim <= 0: + raise ValueError(f"norm_cond_dim must be positive, but got {norm_cond_dim}") + self.norm = torch.nn.LayerNorm( + data_dim + ) # TODO: Check if elementwise_affine=True is correct + self.linear = torch.nn.Linear(norm_cond_dim, 2 * data_dim) + torch.nn.init.zeros_(self.linear.weight) + torch.nn.init.zeros_(self.linear.bias) + + def forward(self, x: torch.Tensor, t: torch.Tensor) -> torch.Tensor: + # x: (batch, ..., data_dim) + # t: (batch, norm_cond_dim) + # return: (batch, data_dim) + x = self.norm(x) + alpha, beta = self.linear(t).chunk(2, dim=-1) + + # Add singleton dimensions to alpha and beta + if x.dim() > 2: + alpha = alpha.view(alpha.shape[0], *([1] * (x.dim() - 2)), alpha.shape[1]) + beta = beta.view(beta.shape[0], *([1] * (x.dim() - 2)), beta.shape[1]) + + return x * (1 + alpha) + beta + + +class SequentialCond(torch.nn.Sequential): + def forward(self, input, *args, **kwargs): + for module in self: + if isinstance(module, (AdaptiveLayerNorm1D, SequentialCond, ResidualMLPBlock)): + # print(f'Passing on args to {module}', [a.shape for a in args]) + input = module(input, *args, **kwargs) + else: + # print(f'Skipping passing args to {module}', [a.shape for a in args]) + input = module(input) + return input + + +def normalization_layer(norm: Optional[str], dim: int, norm_cond_dim: int = -1): + if norm == "batch": + return torch.nn.BatchNorm1d(dim) + elif norm == "layer": + return torch.nn.LayerNorm(dim) + elif norm == "ada": + assert norm_cond_dim > 0, f"norm_cond_dim must be positive, got {norm_cond_dim}" + return AdaptiveLayerNorm1D(dim, norm_cond_dim) + elif norm is None: + return torch.nn.Identity() + else: + raise ValueError(f"Unknown norm: {norm}") + + +def linear_norm_activ_dropout( + input_dim: int, + output_dim: int, + activation: torch.nn.Module = torch.nn.ReLU(), + bias: bool = True, + norm: Optional[str] = "layer", # Options: ada/batch/layer + dropout: float = 0.0, + norm_cond_dim: int = -1, +) -> SequentialCond: + layers = [] + layers.append(torch.nn.Linear(input_dim, output_dim, bias=bias)) + if norm is not None: + layers.append(normalization_layer(norm, output_dim, norm_cond_dim)) + layers.append(copy.deepcopy(activation)) + if dropout > 0.0: + layers.append(torch.nn.Dropout(dropout)) + return SequentialCond(*layers) + + +def create_simple_mlp( + input_dim: int, + hidden_dims: List[int], + output_dim: int, + activation: torch.nn.Module = torch.nn.ReLU(), + bias: bool = True, + norm: Optional[str] = "layer", # Options: ada/batch/layer + dropout: float = 0.0, + norm_cond_dim: int = -1, +) -> SequentialCond: + layers = [] + prev_dim = input_dim + for hidden_dim in hidden_dims: + layers.extend( + linear_norm_activ_dropout( + prev_dim, hidden_dim, activation, bias, norm, dropout, norm_cond_dim + ) + ) + prev_dim = hidden_dim + layers.append(torch.nn.Linear(prev_dim, output_dim, bias=bias)) + return SequentialCond(*layers) + + +class ResidualMLPBlock(torch.nn.Module): + def __init__( + self, + input_dim: int, + hidden_dim: int, + num_hidden_layers: int, + output_dim: int, + activation: torch.nn.Module = torch.nn.ReLU(), + bias: bool = True, + norm: Optional[str] = "layer", # Options: ada/batch/layer + dropout: float = 0.0, + norm_cond_dim: int = -1, + ): + super().__init__() + if not (input_dim == output_dim == hidden_dim): + raise NotImplementedError( + f"input_dim {input_dim} != output_dim {output_dim} is not implemented" + ) + + layers = [] + prev_dim = input_dim + for i in range(num_hidden_layers): + layers.append( + linear_norm_activ_dropout( + prev_dim, hidden_dim, activation, bias, norm, dropout, norm_cond_dim + ) + ) + prev_dim = hidden_dim + self.model = SequentialCond(*layers) + self.skip = torch.nn.Identity() + + def forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor: + return x + self.model(x, *args, **kwargs) + + +class ResidualMLP(torch.nn.Module): + def __init__( + self, + input_dim: int, + hidden_dim: int, + num_hidden_layers: int, + output_dim: int, + activation: torch.nn.Module = torch.nn.ReLU(), + bias: bool = True, + norm: Optional[str] = "layer", # Options: ada/batch/layer + dropout: float = 0.0, + num_blocks: int = 1, + norm_cond_dim: int = -1, + ): + super().__init__() + self.input_dim = input_dim + self.model = SequentialCond( + linear_norm_activ_dropout( + input_dim, hidden_dim, activation, bias, norm, dropout, norm_cond_dim + ), + *[ + ResidualMLPBlock( + hidden_dim, + hidden_dim, + num_hidden_layers, + hidden_dim, + activation, + bias, + norm, + dropout, + norm_cond_dim, + ) + for _ in range(num_blocks) + ], + torch.nn.Linear(hidden_dim, output_dim, bias=bias), + ) + + def forward(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor: + return self.model(x, *args, **kwargs) + + +class FrequencyEmbedder(torch.nn.Module): + def __init__(self, num_frequencies, max_freq_log2): + super().__init__() + frequencies = 2 ** torch.linspace(0, max_freq_log2, steps=num_frequencies) + self.register_buffer("frequencies", frequencies) + + def forward(self, x): + # x should be of size (N,) or (N, D) + N = x.size(0) + if x.dim() == 1: # (N,) + x = x.unsqueeze(1) # (N, D) where D=1 + x_unsqueezed = x.unsqueeze(-1) # (N, D, 1) + scaled = self.frequencies.view(1, 1, -1) * x_unsqueezed # (N, D, num_frequencies) + s = torch.sin(scaled) + c = torch.cos(scaled) + embedded = torch.cat([s, c, x_unsqueezed], dim=-1).view( + N, -1 + ) # (N, D * 2 * num_frequencies + D) + return embedded + diff --git a/fourm/utils/hmr2_utils/hmr2/models/heads/__init__.py b/fourm/utils/hmr2_utils/hmr2/models/heads/__init__.py new file mode 100755 index 0000000000000000000000000000000000000000..d993f8d62006070a7ef576b04a08da3ed5304ea2 --- /dev/null +++ b/fourm/utils/hmr2_utils/hmr2/models/heads/__init__.py @@ -0,0 +1 @@ +from .smpl_head import build_smpl_head diff --git a/fourm/utils/hmr2_utils/hmr2/models/heads/smpl_head.py b/fourm/utils/hmr2_utils/hmr2/models/heads/smpl_head.py new file mode 100755 index 0000000000000000000000000000000000000000..018a4dd74348a69f02c3124749cb404a83bae92a --- /dev/null +++ b/fourm/utils/hmr2_utils/hmr2/models/heads/smpl_head.py @@ -0,0 +1,116 @@ +# -------------------------------------------------------- +# Based on the 4DHumans code base +# https://github.com/shubham-goel/4D-Humans +# -------------------------------------------------------- + +import torch +import torch.nn as nn +import torch.nn.functional as F +import numpy as np +import einops + +from ...utils.geometry import rot6d_to_rotmat, aa_to_rotmat +from ..components.pose_transformer import TransformerDecoder + +def build_smpl_head(cfg): + smpl_head_type = cfg.MODEL.SMPL_HEAD.get('TYPE', 'hmr') + if smpl_head_type == 'transformer_decoder': + return SMPLTransformerDecoderHead(cfg) + else: + raise ValueError('Unknown SMPL head type: {}'.format(smpl_head_type)) + +class SMPLTransformerDecoderHead(nn.Module): + """ Cross-attention based SMPL Transformer decoder + """ + + def __init__(self, cfg): + super().__init__() + self.cfg = cfg + self.joint_rep_type = cfg.MODEL.SMPL_HEAD.get('JOINT_REP', '6d') + self.joint_rep_dim = {'6d': 6, 'aa': 3}[self.joint_rep_type] + npose = self.joint_rep_dim * (cfg.SMPL.NUM_BODY_JOINTS + 1) + self.npose = npose + self.input_is_mean_shape = cfg.MODEL.SMPL_HEAD.get('TRANSFORMER_INPUT', 'zero') == 'mean_shape' + transformer_args = dict( + num_tokens=1, + token_dim=(npose + 10 + 3) if self.input_is_mean_shape else 1, + dim=1024, + ) + transformer_args = (transformer_args | dict(cfg.MODEL.SMPL_HEAD.TRANSFORMER_DECODER)) + self.transformer = TransformerDecoder( + **transformer_args + ) + dim=transformer_args['dim'] + self.decpose = nn.Linear(dim, npose) + self.decshape = nn.Linear(dim, 10) + self.deccam = nn.Linear(dim, 3) + + if cfg.MODEL.SMPL_HEAD.get('INIT_DECODER_XAVIER', False): + # True by default in MLP. False by default in Transformer + nn.init.xavier_uniform_(self.decpose.weight, gain=0.01) + nn.init.xavier_uniform_(self.decshape.weight, gain=0.01) + nn.init.xavier_uniform_(self.deccam.weight, gain=0.01) + + mean_params = np.load(cfg.SMPL.MEAN_PARAMS) + init_body_pose = torch.from_numpy(mean_params['pose'].astype(np.float32)).unsqueeze(0) + init_betas = torch.from_numpy(mean_params['shape'].astype('float32')).unsqueeze(0) + init_cam = torch.from_numpy(mean_params['cam'].astype(np.float32)).unsqueeze(0) + self.register_buffer('init_body_pose', init_body_pose) + self.register_buffer('init_betas', init_betas) + self.register_buffer('init_cam', init_cam) + + def forward(self, x, **kwargs): + + batch_size = x.shape[0] + # vit pretrained backbone is channel-first. Change to token-first + x = einops.rearrange(x, 'b c h w -> b (h w) c') + + init_body_pose = self.init_body_pose.expand(batch_size, -1) + init_betas = self.init_betas.expand(batch_size, -1) + init_cam = self.init_cam.expand(batch_size, -1) + + # TODO: Convert init_body_pose to aa rep if needed + if self.joint_rep_type == 'aa': + raise NotImplementedError + + pred_body_pose = init_body_pose + pred_betas = init_betas + pred_cam = init_cam + pred_body_pose_list = [] + pred_betas_list = [] + pred_cam_list = [] + for i in range(self.cfg.MODEL.SMPL_HEAD.get('IEF_ITERS', 1)): + # Input token to transformer is zero token + if self.input_is_mean_shape: + token = torch.cat([pred_body_pose, pred_betas, pred_cam], dim=1)[:,None,:] + else: + token = torch.zeros(batch_size, 1, 1).to(x.device) + + # Pass through transformer + token_out = self.transformer(token, context=x) + token_out = token_out.squeeze(1) # (B, C) + + # Readout from token_out + pred_body_pose = self.decpose(token_out) + pred_body_pose + pred_betas = self.decshape(token_out) + pred_betas + pred_cam = self.deccam(token_out) + pred_cam + pred_body_pose_list.append(pred_body_pose) + pred_betas_list.append(pred_betas) + pred_cam_list.append(pred_cam) + + # Convert self.joint_rep_type -> rotmat + joint_conversion_fn = { + '6d': rot6d_to_rotmat, + 'aa': lambda x: aa_to_rotmat(x.view(-1, 3).contiguous()) + }[self.joint_rep_type] + + pred_smpl_params_list = {} + pred_smpl_params_list['body_pose'] = torch.cat([joint_conversion_fn(pbp).view(batch_size, -1, 3, 3)[:, 1:, :, :] for pbp in pred_body_pose_list], dim=0) + pred_smpl_params_list['betas'] = torch.cat(pred_betas_list, dim=0) + pred_smpl_params_list['cam'] = torch.cat(pred_cam_list, dim=0) + pred_body_pose = joint_conversion_fn(pred_body_pose).view(batch_size, self.cfg.SMPL.NUM_BODY_JOINTS+1, 3, 3) + + pred_smpl_params = {'global_orient': pred_body_pose[:, [0]], + 'body_pose': pred_body_pose[:, 1:], + 'betas': pred_betas} + return pred_smpl_params, pred_cam, pred_smpl_params_list diff --git a/fourm/utils/hmr2_utils/hmr2/models/hmr2.py b/fourm/utils/hmr2_utils/hmr2/models/hmr2.py new file mode 100755 index 0000000000000000000000000000000000000000..852429f791aded4be676d53635ad6d3f32e2de89 --- /dev/null +++ b/fourm/utils/hmr2_utils/hmr2/models/hmr2.py @@ -0,0 +1,117 @@ +# -------------------------------------------------------- +# Based on the 4DHumans code base +# https://github.com/shubham-goel/4D-Humans +# -------------------------------------------------------- + +import torch +from typing import Any, Dict, Mapping, Tuple + +from yacs.config import CfgNode + +from ..utils import SkeletonRenderer, MeshRenderer +from ..utils.geometry import perspective_projection +from .backbones import create_backbone +from .heads import build_smpl_head +from . import SMPL + +class HMR2(torch.nn.Module): + + def __init__(self, cfg: CfgNode, init_renderer: bool = True): + """ + Setup HMR2 model + Args: + cfg (CfgNode): Config file as a yacs CfgNode + """ + super().__init__() + + # Save hyperparameters + self.save_hyperparameters(logger=False, ignore=['init_renderer']) + + self.cfg = cfg + # Create backbone feature extractor + self.backbone = create_backbone(cfg) + if cfg.MODEL.BACKBONE.get('PRETRAINED_WEIGHTS', None): + self.backbone.load_state_dict(torch.load(cfg.MODEL.BACKBONE.PRETRAINED_WEIGHTS, map_location='cpu')['state_dict']) + + # Create SMPL head + self.smpl_head = build_smpl_head(cfg) + + # Instantiate SMPL model + smpl_cfg = {k.lower(): v for k,v in dict(cfg.SMPL).items()} + self.smpl = SMPL(**smpl_cfg) + + # Buffer that shows whetheer we need to initialize ActNorm layers + self.register_buffer('initialized', torch.tensor(False)) + # Setup renderer for visualization + if init_renderer: + self.renderer = SkeletonRenderer(self.cfg) + self.mesh_renderer = MeshRenderer(self.cfg, faces=self.smpl.faces) + else: + self.renderer = None + self.mesh_renderer = None + + # Disable automatic optimization since we use adversarial training + self.automatic_optimization = False + + def forward_step(self, batch: Dict, train: bool = False) -> Dict: + """ + Run a forward step of the network + Args: + batch (Dict): Dictionary containing batch data + train (bool): Flag indicating whether it is training or validation mode + Returns: + Dict: Dictionary containing the regression output + """ + + # Use RGB image as input + x = batch['img'] + batch_size = x.shape[0] + + # Compute conditioning features using the backbone + # if using ViT backbone, we need to use a different aspect ratio + conditioning_feats = self.backbone(x[:,:,:,32:-32]) + + pred_smpl_params, pred_cam, _ = self.smpl_head(conditioning_feats) + + # Store useful regression outputs to the output dict + output = {} + output['pred_cam'] = pred_cam + output['pred_smpl_params'] = {k: v.clone() for k,v in pred_smpl_params.items()} + + # Compute camera translation + device = pred_smpl_params['body_pose'].device + dtype = pred_smpl_params['body_pose'].dtype + focal_length = self.cfg.EXTRA.FOCAL_LENGTH * torch.ones(batch_size, 2, device=device, dtype=dtype) + pred_cam_t = torch.stack([pred_cam[:, 1], + pred_cam[:, 2], + 2*focal_length[:, 0]/(self.cfg.MODEL.IMAGE_SIZE * pred_cam[:, 0] +1e-9)],dim=-1) + output['pred_cam_t'] = pred_cam_t + output['focal_length'] = focal_length + + # Compute model vertices, joints and the projected joints + pred_smpl_params['global_orient'] = pred_smpl_params['global_orient'].reshape(batch_size, -1, 3, 3) + pred_smpl_params['body_pose'] = pred_smpl_params['body_pose'].reshape(batch_size, -1, 3, 3) + pred_smpl_params['betas'] = pred_smpl_params['betas'].reshape(batch_size, -1) + smpl_output = self.smpl(**{k: v.float() for k,v in pred_smpl_params.items()}, pose2rot=False) + pred_keypoints_3d = smpl_output.joints + pred_vertices = smpl_output.vertices + output['pred_keypoints_3d'] = pred_keypoints_3d.reshape(batch_size, -1, 3) + output['pred_vertices'] = pred_vertices.reshape(batch_size, -1, 3) + pred_cam_t = pred_cam_t.reshape(-1, 3) + focal_length = focal_length.reshape(-1, 2) + pred_keypoints_2d = perspective_projection(pred_keypoints_3d, + translation=pred_cam_t, + focal_length=focal_length / self.cfg.MODEL.IMAGE_SIZE) + + output['pred_keypoints_2d'] = pred_keypoints_2d.reshape(batch_size, -1, 2) + return output + + def forward(self, batch: Dict) -> Dict: + """ + Run a forward step of the network in val mode + Args: + batch (Dict): Dictionary containing batch data + Returns: + Dict: Dictionary containing the regression output + """ + return self.forward_step(batch, train=False) diff --git a/fourm/utils/hmr2_utils/hmr2/models/smpl_wrapper.py b/fourm/utils/hmr2_utils/hmr2/models/smpl_wrapper.py new file mode 100755 index 0000000000000000000000000000000000000000..bd7eaa8c399c5fb69ebd74acd96d7f028573fb5f --- /dev/null +++ b/fourm/utils/hmr2_utils/hmr2/models/smpl_wrapper.py @@ -0,0 +1,47 @@ +# -------------------------------------------------------- +# Based on the 4DHumans and ProHMR code bases +# https://github.com/shubham-goel/4D-Humans +# https://github.com/nkolot/ProHMR +# -------------------------------------------------------- + +import torch +import numpy as np +import pickle +from typing import Optional +import smplx +from smplx.lbs import vertices2joints +from smplx.utils import SMPLOutput + + +class SMPL(smplx.SMPLLayer): + def __init__(self, *args, joint_regressor_extra: Optional[str] = None, update_hips: bool = False, **kwargs): + """ + Extension of the official SMPL implementation to support more joints. + Args: + Same as SMPLLayer. + joint_regressor_extra (str): Path to extra joint regressor. + """ + super(SMPL, self).__init__(*args, **kwargs) + smpl_to_openpose = [24, 12, 17, 19, 21, 16, 18, 20, 0, 2, 5, 8, 1, 4, + 7, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34] + + if joint_regressor_extra is not None: + self.register_buffer('joint_regressor_extra', torch.tensor(pickle.load(open(joint_regressor_extra, 'rb'), encoding='latin1'), dtype=torch.float32)) + self.register_buffer('joint_map', torch.tensor(smpl_to_openpose, dtype=torch.long)) + self.update_hips = update_hips + + def forward(self, *args, **kwargs) -> SMPLOutput: + """ + Run forward pass. Same as SMPL and also append an extra set of joints if joint_regressor_extra is specified. + """ + smpl_output = super(SMPL, self).forward(*args, **kwargs) + joints = smpl_output.joints[:, self.joint_map, :] + if self.update_hips: + joints[:,[9,12]] = joints[:,[9,12]] + \ + 0.25*(joints[:,[9,12]]-joints[:,[12,9]]) + \ + 0.5*(joints[:,[8]] - 0.5*(joints[:,[9,12]] + joints[:,[12,9]])) + if hasattr(self, 'joint_regressor_extra'): + extra_joints = vertices2joints(self.joint_regressor_extra, smpl_output.vertices) + joints = torch.cat([joints, extra_joints], dim=1) + smpl_output.joints = joints + return smpl_output diff --git a/fourm/utils/hmr2_utils/hmr2/utils/__init__.py b/fourm/utils/hmr2_utils/hmr2/utils/__init__.py new file mode 100755 index 0000000000000000000000000000000000000000..9914c7a8376c206cad0401c8f178139b5611cd66 --- /dev/null +++ b/fourm/utils/hmr2_utils/hmr2/utils/__init__.py @@ -0,0 +1,31 @@ +# -------------------------------------------------------- +# Based on the 4DHumans and ProHMR code bases +# https://github.com/shubham-goel/4D-Humans +# https://github.com/nkolot/ProHMR +# -------------------------------------------------------- + +import torch +from typing import Any + +from .renderer import Renderer +from .mesh_renderer import MeshRenderer +from .skeleton_renderer import SkeletonRenderer +# from .pose_utils import eval_pose, Evaluator + +def recursive_to(x: Any, target: torch.device): + """ + Recursively transfer a batch of data to the target device + Args: + x (Any): Batch of data. + target (torch.device): Target device. + Returns: + Batch of data where all tensors are transfered to the target device. + """ + if isinstance(x, dict): + return {k: recursive_to(v, target) for k, v in x.items()} + elif isinstance(x, torch.Tensor): + return x.to(target) + elif isinstance(x, list): + return [recursive_to(i, target) for i in x] + else: + return x diff --git a/fourm/utils/hmr2_utils/hmr2/utils/geometry.py b/fourm/utils/hmr2_utils/hmr2/utils/geometry.py new file mode 100755 index 0000000000000000000000000000000000000000..e338cc3717cfd71f541b6f111d8459f0ef352db2 --- /dev/null +++ b/fourm/utils/hmr2_utils/hmr2/utils/geometry.py @@ -0,0 +1,109 @@ +# -------------------------------------------------------- +# Based on the 4DHumans, ProHMR, and SPIN code bases +# https://github.com/shubham-goel/4D-Humans +# https://github.com/nkolot/ProHMR +# https://github.com/nkolot/SPIN +# -------------------------------------------------------- + +from typing import Optional +import torch +from torch.nn import functional as F + +def aa_to_rotmat(theta: torch.Tensor): + """ + Convert axis-angle representation to rotation matrix. + Works by first converting it to a quaternion. + Args: + theta (torch.Tensor): Tensor of shape (B, 3) containing axis-angle representations. + Returns: + torch.Tensor: Corresponding rotation matrices with shape (B, 3, 3). + """ + norm = torch.norm(theta + 1e-8, p = 2, dim = 1) + angle = torch.unsqueeze(norm, -1) + normalized = torch.div(theta, angle) + angle = angle * 0.5 + v_cos = torch.cos(angle) + v_sin = torch.sin(angle) + quat = torch.cat([v_cos, v_sin * normalized], dim = 1) + return quat_to_rotmat(quat) + +def quat_to_rotmat(quat: torch.Tensor) -> torch.Tensor: + """ + Convert quaternion representation to rotation matrix. + Args: + quat (torch.Tensor) of shape (B, 4); 4 <===> (w, x, y, z). + Returns: + torch.Tensor: Corresponding rotation matrices with shape (B, 3, 3). + """ + norm_quat = quat + norm_quat = norm_quat/norm_quat.norm(p=2, dim=1, keepdim=True) + w, x, y, z = norm_quat[:,0], norm_quat[:,1], norm_quat[:,2], norm_quat[:,3] + + B = quat.size(0) + + w2, x2, y2, z2 = w.pow(2), x.pow(2), y.pow(2), z.pow(2) + wx, wy, wz = w*x, w*y, w*z + xy, xz, yz = x*y, x*z, y*z + + rotMat = torch.stack([w2 + x2 - y2 - z2, 2*xy - 2*wz, 2*wy + 2*xz, + 2*wz + 2*xy, w2 - x2 + y2 - z2, 2*yz - 2*wx, + 2*xz - 2*wy, 2*wx + 2*yz, w2 - x2 - y2 + z2], dim=1).view(B, 3, 3) + return rotMat + + +def rot6d_to_rotmat(x: torch.Tensor) -> torch.Tensor: + """ + Convert 6D rotation representation to 3x3 rotation matrix. + Based on Zhou et al., "On the Continuity of Rotation Representations in Neural Networks", CVPR 2019 + Args: + x (torch.Tensor): (B,6) Batch of 6-D rotation representations. + Returns: + torch.Tensor: Batch of corresponding rotation matrices with shape (B,3,3). + """ + x = x.reshape(-1,2,3).permute(0, 2, 1).contiguous() + a1 = x[:, :, 0] + a2 = x[:, :, 1] + b1 = F.normalize(a1) + b2 = F.normalize(a2 - torch.einsum('bi,bi->b', b1, a2).unsqueeze(-1) * b1) + b3 = torch.cross(b1, b2) + return torch.stack((b1, b2, b3), dim=-1) + +def perspective_projection(points: torch.Tensor, + translation: torch.Tensor, + focal_length: torch.Tensor, + camera_center: Optional[torch.Tensor] = None, + rotation: Optional[torch.Tensor] = None) -> torch.Tensor: + """ + Computes the perspective projection of a set of 3D points. + Args: + points (torch.Tensor): Tensor of shape (B, N, 3) containing the input 3D points. + translation (torch.Tensor): Tensor of shape (B, 3) containing the 3D camera translation. + focal_length (torch.Tensor): Tensor of shape (B, 2) containing the focal length in pixels. + camera_center (torch.Tensor): Tensor of shape (B, 2) containing the camera center in pixels. + rotation (torch.Tensor): Tensor of shape (B, 3, 3) containing the camera rotation. + Returns: + torch.Tensor: Tensor of shape (B, N, 2) containing the projection of the input points. + """ + batch_size = points.shape[0] + if rotation is None: + rotation = torch.eye(3, device=points.device, dtype=points.dtype).unsqueeze(0).expand(batch_size, -1, -1) + if camera_center is None: + camera_center = torch.zeros(batch_size, 2, device=points.device, dtype=points.dtype) + # Populate intrinsic camera matrix K. + K = torch.zeros([batch_size, 3, 3], device=points.device, dtype=points.dtype) + K[:,0,0] = focal_length[:,0] + K[:,1,1] = focal_length[:,1] + K[:,2,2] = 1. + K[:,:-1, -1] = camera_center + + # Transform points + points = torch.einsum('bij,bkj->bki', rotation, points) + points = points + translation.unsqueeze(1) + + # Apply perspective distortion + projected_points = points / points[:,:,-1].unsqueeze(-1) + + # Apply camera intrinsics + projected_points = torch.einsum('bij,bkj->bki', K, projected_points) + + return projected_points[:, :, :-1] \ No newline at end of file diff --git a/fourm/utils/hmr2_utils/hmr2/utils/mesh_renderer.py b/fourm/utils/hmr2_utils/hmr2/utils/mesh_renderer.py new file mode 100755 index 0000000000000000000000000000000000000000..2705fb2f187cf7ee4ab2e823bab5f6f117bd0b6f --- /dev/null +++ b/fourm/utils/hmr2_utils/hmr2/utils/mesh_renderer.py @@ -0,0 +1,155 @@ +# -------------------------------------------------------- +# Based on the 4DHumans and ProHMR code bases +# https://github.com/shubham-goel/4D-Humans +# https://github.com/nkolot/ProHMR +# -------------------------------------------------------- + +import os +if 'PYOPENGL_PLATFORM' not in os.environ: + os.environ['PYOPENGL_PLATFORM'] = 'egl' +import torch +from torchvision.utils import make_grid +import numpy as np +import pyrender +import trimesh +import cv2 +import torch.nn.functional as F + +from .render_openpose import render_openpose + +def create_raymond_lights(): + import pyrender + thetas = np.pi * np.array([1.0 / 6.0, 1.0 / 6.0, 1.0 / 6.0]) + phis = np.pi * np.array([0.0, 2.0 / 3.0, 4.0 / 3.0]) + + nodes = [] + + for phi, theta in zip(phis, thetas): + xp = np.sin(theta) * np.cos(phi) + yp = np.sin(theta) * np.sin(phi) + zp = np.cos(theta) + + z = np.array([xp, yp, zp]) + z = z / np.linalg.norm(z) + x = np.array([-z[1], z[0], 0.0]) + if np.linalg.norm(x) == 0: + x = np.array([1.0, 0.0, 0.0]) + x = x / np.linalg.norm(x) + y = np.cross(z, x) + + matrix = np.eye(4) + matrix[:3,:3] = np.c_[x,y,z] + nodes.append(pyrender.Node( + light=pyrender.DirectionalLight(color=np.ones(3), intensity=1.0), + matrix=matrix + )) + + return nodes + +class MeshRenderer: + + def __init__(self, cfg, faces=None): + self.cfg = cfg + self.focal_length = cfg.EXTRA.FOCAL_LENGTH + self.img_res = cfg.MODEL.IMAGE_SIZE + self.renderer = pyrender.OffscreenRenderer(viewport_width=self.img_res, + viewport_height=self.img_res, + point_size=1.0) + + self.camera_center = [self.img_res // 2, self.img_res // 2] + self.faces = faces + + def visualize(self, vertices, camera_translation, images, focal_length=None, nrow=3, padding=2): + images_np = np.transpose(images, (0,2,3,1)) + rend_imgs = [] + for i in range(vertices.shape[0]): + fl = self.focal_length + rend_img = torch.from_numpy(np.transpose(self.__call__(vertices[i], camera_translation[i], images_np[i], focal_length=fl, side_view=False), (2,0,1))).float() + rend_img_side = torch.from_numpy(np.transpose(self.__call__(vertices[i], camera_translation[i], images_np[i], focal_length=fl, side_view=True), (2,0,1))).float() + rend_imgs.append(torch.from_numpy(images[i])) + rend_imgs.append(rend_img) + rend_imgs.append(rend_img_side) + rend_imgs = make_grid(rend_imgs, nrow=nrow, padding=padding) + return rend_imgs + + def visualize_tensorboard(self, vertices, camera_translation, images, pred_keypoints, gt_keypoints, focal_length=None, nrow=5, padding=2): + images_np = np.transpose(images, (0,2,3,1)) + rend_imgs = [] + pred_keypoints = np.concatenate((pred_keypoints, np.ones_like(pred_keypoints)[:, :, [0]]), axis=-1) + pred_keypoints = self.img_res * (pred_keypoints + 0.5) + gt_keypoints[:, :, :-1] = self.img_res * (gt_keypoints[:, :, :-1] + 0.5) + keypoint_matches = [(1, 12), (2, 8), (3, 7), (4, 6), (5, 9), (6, 10), (7, 11), (8, 14), (9, 2), (10, 1), (11, 0), (12, 3), (13, 4), (14, 5)] + for i in range(vertices.shape[0]): + fl = self.focal_length + rend_img = torch.from_numpy(np.transpose(self.__call__(vertices[i], camera_translation[i], images_np[i], focal_length=fl, side_view=False), (2,0,1))).float() + rend_img_side = torch.from_numpy(np.transpose(self.__call__(vertices[i], camera_translation[i], images_np[i], focal_length=fl, side_view=True), (2,0,1))).float() + body_keypoints = pred_keypoints[i, :25] + extra_keypoints = pred_keypoints[i, -19:] + for pair in keypoint_matches: + body_keypoints[pair[0], :] = extra_keypoints[pair[1], :] + pred_keypoints_img = render_openpose(255 * images_np[i].copy(), body_keypoints) / 255 + body_keypoints = gt_keypoints[i, :25] + extra_keypoints = gt_keypoints[i, -19:] + for pair in keypoint_matches: + if extra_keypoints[pair[1], -1] > 0 and body_keypoints[pair[0], -1] == 0: + body_keypoints[pair[0], :] = extra_keypoints[pair[1], :] + gt_keypoints_img = render_openpose(255*images_np[i].copy(), body_keypoints) / 255 + rend_imgs.append(torch.from_numpy(images[i])) + rend_imgs.append(rend_img) + rend_imgs.append(rend_img_side) + rend_imgs.append(torch.from_numpy(pred_keypoints_img).permute(2,0,1)) + rend_imgs.append(torch.from_numpy(gt_keypoints_img).permute(2,0,1)) + rend_imgs = make_grid(rend_imgs, nrow=nrow, padding=padding) + return rend_imgs + + def __call__(self, vertices, camera_translation, image, focal_length=5000, text=None, resize=None, side_view=False, baseColorFactor=(1.0, 1.0, 0.9, 1.0), rot_angle=90): + renderer = pyrender.OffscreenRenderer(viewport_width=image.shape[1], + viewport_height=image.shape[0], + point_size=1.0) + material = pyrender.MetallicRoughnessMaterial( + metallicFactor=0.0, + alphaMode='OPAQUE', + baseColorFactor=baseColorFactor) + + camera_translation[0] *= -1. + + mesh = trimesh.Trimesh(vertices.copy(), self.faces.copy()) + if side_view: + rot = trimesh.transformations.rotation_matrix( + np.radians(rot_angle), [0, 1, 0]) + mesh.apply_transform(rot) + rot = trimesh.transformations.rotation_matrix( + np.radians(180), [1, 0, 0]) + mesh.apply_transform(rot) + mesh = pyrender.Mesh.from_trimesh(mesh, material=material) + + scene = pyrender.Scene(bg_color=[0.0, 0.0, 0.0, 0.0], + ambient_light=(0.3, 0.3, 0.3)) + scene.add(mesh, 'mesh') + + camera_pose = np.eye(4) + camera_pose[:3, 3] = camera_translation + camera_center = [image.shape[1] / 2., image.shape[0] / 2.] + camera = pyrender.IntrinsicsCamera(fx=focal_length, fy=focal_length, + cx=camera_center[0], cy=camera_center[1]) + scene.add(camera, pose=camera_pose) + + + light_nodes = create_raymond_lights() + for node in light_nodes: + scene.add_node(node) + + color, rend_depth = renderer.render(scene, flags=pyrender.RenderFlags.RGBA) + color = color.astype(np.float32) / 255.0 + valid_mask = (color[:, :, -1] > 0)[:, :, np.newaxis] + if not side_view: + output_img = (color[:, :, :3] * valid_mask + + (1 - valid_mask) * image) + else: + output_img = color[:, :, :3] + if resize is not None: + output_img = cv2.resize(output_img, resize) + + output_img = output_img.astype(np.float32) + renderer.delete() + return output_img diff --git a/fourm/utils/hmr2_utils/hmr2/utils/render_openpose.py b/fourm/utils/hmr2_utils/hmr2/utils/render_openpose.py new file mode 100755 index 0000000000000000000000000000000000000000..92ff74ff4af0cd5291881d5eefd7709670aa0a0a --- /dev/null +++ b/fourm/utils/hmr2_utils/hmr2/utils/render_openpose.py @@ -0,0 +1,155 @@ +# -------------------------------------------------------- +# Based on the 4DHumans and ProHMR code bases +# https://github.com/shubham-goel/4D-Humans +# https://github.com/nkolot/ProHMR +# -------------------------------------------------------- + +""" +Render OpenPose keypoints. +Code was ported to Python from the official C++ implementation https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/utilities/keypoint.cpp +""" +import cv2 +import math +import numpy as np +from typing import List, Tuple + +def get_keypoints_rectangle(keypoints: np.array, threshold: float) -> Tuple[float, float, float]: + """ + Compute rectangle enclosing keypoints above the threshold. + Args: + keypoints (np.array): Keypoint array of shape (N, 3). + threshold (float): Confidence visualization threshold. + Returns: + Tuple[float, float, float]: Rectangle width, height and area. + """ + valid_ind = keypoints[:, -1] > threshold + if valid_ind.sum() > 0: + valid_keypoints = keypoints[valid_ind][:, :-1] + max_x = valid_keypoints[:,0].max() + max_y = valid_keypoints[:,1].max() + min_x = valid_keypoints[:,0].min() + min_y = valid_keypoints[:,1].min() + width = max_x - min_x + height = max_y - min_y + area = width * height + return width, height, area + else: + return 0,0,0 + +def render_keypoints(img: np.array, + keypoints: np.array, + pairs: List, + colors: List, + thickness_circle_ratio: float, + thickness_line_ratio_wrt_circle: float, + pose_scales: List, + threshold: float = 0.1) -> np.array: + """ + Render keypoints on input image. + Args: + img (np.array): Input image of shape (H, W, 3) with pixel values in the [0,255] range. + keypoints (np.array): Keypoint array of shape (N, 3). + pairs (List): List of keypoint pairs per limb. + colors: (List): List of colors per keypoint. + thickness_circle_ratio (float): Circle thickness ratio. + thickness_line_ratio_wrt_circle (float): Line thickness ratio wrt the circle. + pose_scales (List): List of pose scales. + threshold (float): Only visualize keypoints with confidence above the threshold. + Returns: + (np.array): Image of shape (H, W, 3) with keypoints drawn on top of the original image. + """ + img_orig = img.copy() + width, height = img.shape[1], img.shape[2] + area = width * height + + lineType = 8 + shift = 0 + numberColors = len(colors) + thresholdRectangle = 0.1 + + person_width, person_height, person_area = get_keypoints_rectangle(keypoints, thresholdRectangle) + if person_area > 0: + ratioAreas = min(1, max(person_width / width, person_height / height)) + thicknessRatio = np.maximum(np.round(math.sqrt(area) * thickness_circle_ratio * ratioAreas), 2) + thicknessCircle = np.maximum(1, thicknessRatio if ratioAreas > 0.05 else -np.ones_like(thicknessRatio)) + thicknessLine = np.maximum(1, np.round(thicknessRatio * thickness_line_ratio_wrt_circle)) + radius = thicknessRatio / 2 + + img = np.ascontiguousarray(img.copy()) + for i, pair in enumerate(pairs): + index1, index2 = pair + if keypoints[index1, -1] > threshold and keypoints[index2, -1] > threshold: + thicknessLineScaled = int(round(min(thicknessLine[index1], thicknessLine[index2]) * pose_scales[0])) + colorIndex = index2 + color = colors[colorIndex % numberColors] + keypoint1 = keypoints[index1, :-1].astype(np.int) + keypoint2 = keypoints[index2, :-1].astype(np.int) + cv2.line(img, tuple(keypoint1.tolist()), tuple(keypoint2.tolist()), tuple(color.tolist()), thicknessLineScaled, lineType, shift) + for part in range(len(keypoints)): + faceIndex = part + if keypoints[faceIndex, -1] > threshold: + radiusScaled = int(round(radius[faceIndex] * pose_scales[0])) + thicknessCircleScaled = int(round(thicknessCircle[faceIndex] * pose_scales[0])) + colorIndex = part + color = colors[colorIndex % numberColors] + center = keypoints[faceIndex, :-1].astype(np.int) + cv2.circle(img, tuple(center.tolist()), radiusScaled, tuple(color.tolist()), thicknessCircleScaled, lineType, shift) + return img + +def render_body_keypoints(img: np.array, + body_keypoints: np.array) -> np.array: + """ + Render OpenPose body keypoints on input image. + Args: + img (np.array): Input image of shape (H, W, 3) with pixel values in the [0,255] range. + body_keypoints (np.array): Keypoint array of shape (N, 3); 3 <====> (x, y, confidence). + Returns: + (np.array): Image of shape (H, W, 3) with keypoints drawn on top of the original image. + """ + + thickness_circle_ratio = 1./75. * np.ones(body_keypoints.shape[0]) + thickness_line_ratio_wrt_circle = 0.75 + pairs = [] + pairs = [1,8,1,2,1,5,2,3,3,4,5,6,6,7,8,9,9,10,10,11,8,12,12,13,13,14,1,0,0,15,15,17,0,16,16,18,14,19,19,20,14,21,11,22,22,23,11,24] + pairs = np.array(pairs).reshape(-1,2) + colors = [255., 0., 85., + 255., 0., 0., + 255., 85., 0., + 255., 170., 0., + 255., 255., 0., + 170., 255., 0., + 85., 255., 0., + 0., 255., 0., + 255., 0., 0., + 0., 255., 85., + 0., 255., 170., + 0., 255., 255., + 0., 170., 255., + 0., 85., 255., + 0., 0., 255., + 255., 0., 170., + 170., 0., 255., + 255., 0., 255., + 85., 0., 255., + 0., 0., 255., + 0., 0., 255., + 0., 0., 255., + 0., 255., 255., + 0., 255., 255., + 0., 255., 255.] + colors = np.array(colors).reshape(-1,3) + pose_scales = [1] + return render_keypoints(img, body_keypoints, pairs, colors, thickness_circle_ratio, thickness_line_ratio_wrt_circle, pose_scales, 0.1) + +def render_openpose(img: np.array, + body_keypoints: np.array) -> np.array: + """ + Render keypoints in the OpenPose format on input image. + Args: + img (np.array): Input image of shape (H, W, 3) with pixel values in the [0,255] range. + body_keypoints (np.array): Keypoint array of shape (N, 3); 3 <====> (x, y, confidence). + Returns: + (np.array): Image of shape (H, W, 3) with keypoints drawn on top of the original image. + """ + img = render_body_keypoints(img, body_keypoints) + return img diff --git a/fourm/utils/hmr2_utils/hmr2/utils/renderer.py b/fourm/utils/hmr2_utils/hmr2/utils/renderer.py new file mode 100755 index 0000000000000000000000000000000000000000..0f07a9801a791c0257d1508f4f6387be8d70f59e --- /dev/null +++ b/fourm/utils/hmr2_utils/hmr2/utils/renderer.py @@ -0,0 +1,402 @@ +# -------------------------------------------------------- +# Based on the 4DHumans and ProHMR code bases +# https://github.com/shubham-goel/4D-Humans +# https://github.com/nkolot/ProHMR +# -------------------------------------------------------- + +import os +# if 'PYOPENGL_PLATFORM' not in os.environ: +# os.environ['PYOPENGL_PLATFORM'] = 'egl' +os.environ['PYOPENGL_PLATFORM'] = 'osmesa' + +import torch +import numpy as np +import pyrender +import trimesh +import cv2 +from yacs.config import CfgNode +from typing import List, Optional + +def cam_crop_to_full(cam_bbox, box_center, box_size, img_size, focal_length=5000.): + # Convert cam_bbox to full image + img_w, img_h = img_size[:, 0], img_size[:, 1] + cx, cy, b = box_center[:, 0], box_center[:, 1], box_size + w_2, h_2 = img_w / 2., img_h / 2. + bs = b * cam_bbox[:, 0] + 1e-9 + tz = 2 * focal_length / bs + tx = (2 * (cx - w_2) / bs) + cam_bbox[:, 1] + ty = (2 * (cy - h_2) / bs) + cam_bbox[:, 2] + full_cam = torch.stack([tx, ty, tz], dim=-1) + return full_cam + +def get_light_poses(n_lights=5, elevation=np.pi / 3, dist=12): + # get lights in a circle around origin at elevation + thetas = elevation * np.ones(n_lights) + phis = 2 * np.pi * np.arange(n_lights) / n_lights + poses = [] + trans = make_translation(torch.tensor([0, 0, dist])) + for phi, theta in zip(phis, thetas): + rot = make_rotation(rx=-theta, ry=phi, order="xyz") + poses.append((rot @ trans).numpy()) + return poses + +def make_translation(t): + return make_4x4_pose(torch.eye(3), t) + +def make_rotation(rx=0, ry=0, rz=0, order="xyz"): + Rx = rotx(rx) + Ry = roty(ry) + Rz = rotz(rz) + if order == "xyz": + R = Rz @ Ry @ Rx + elif order == "xzy": + R = Ry @ Rz @ Rx + elif order == "yxz": + R = Rz @ Rx @ Ry + elif order == "yzx": + R = Rx @ Rz @ Ry + elif order == "zyx": + R = Rx @ Ry @ Rz + elif order == "zxy": + R = Ry @ Rx @ Rz + return make_4x4_pose(R, torch.zeros(3)) + +def make_4x4_pose(R, t): + """ + :param R (*, 3, 3) + :param t (*, 3) + return (*, 4, 4) + """ + dims = R.shape[:-2] + pose_3x4 = torch.cat([R, t.view(*dims, 3, 1)], dim=-1) + bottom = ( + torch.tensor([0, 0, 0, 1], device=R.device) + .reshape(*(1,) * len(dims), 1, 4) + .expand(*dims, 1, 4) + ) + return torch.cat([pose_3x4, bottom], dim=-2) + + +def rotx(theta): + return torch.tensor( + [ + [1, 0, 0], + [0, np.cos(theta), -np.sin(theta)], + [0, np.sin(theta), np.cos(theta)], + ], + dtype=torch.float32, + ) + + +def roty(theta): + return torch.tensor( + [ + [np.cos(theta), 0, np.sin(theta)], + [0, 1, 0], + [-np.sin(theta), 0, np.cos(theta)], + ], + dtype=torch.float32, + ) + + +def rotz(theta): + return torch.tensor( + [ + [np.cos(theta), -np.sin(theta), 0], + [np.sin(theta), np.cos(theta), 0], + [0, 0, 1], + ], + dtype=torch.float32, + ) + + +def create_raymond_lights() -> List[pyrender.Node]: + """ + Return raymond light nodes for the scene. + """ + thetas = np.pi * np.array([1.0 / 6.0, 1.0 / 6.0, 1.0 / 6.0]) + phis = np.pi * np.array([0.0, 2.0 / 3.0, 4.0 / 3.0]) + + nodes = [] + + for phi, theta in zip(phis, thetas): + xp = np.sin(theta) * np.cos(phi) + yp = np.sin(theta) * np.sin(phi) + zp = np.cos(theta) + + z = np.array([xp, yp, zp]) + z = z / np.linalg.norm(z) + x = np.array([-z[1], z[0], 0.0]) + if np.linalg.norm(x) == 0: + x = np.array([1.0, 0.0, 0.0]) + x = x / np.linalg.norm(x) + y = np.cross(z, x) + + matrix = np.eye(4) + matrix[:3,:3] = np.c_[x,y,z] + nodes.append(pyrender.Node( + light=pyrender.DirectionalLight(color=np.ones(3), intensity=1.0), + matrix=matrix + )) + + return nodes + +class Renderer: + + def __init__(self, cfg: CfgNode, faces: np.array): + """ + Wrapper around the pyrender renderer to render SMPL meshes. + Args: + cfg (CfgNode): Model config file. + faces (np.array): Array of shape (F, 3) containing the mesh faces. + """ + self.cfg = cfg + self.focal_length = cfg.EXTRA.FOCAL_LENGTH + self.img_res = cfg.MODEL.IMAGE_SIZE + + self.camera_center = [self.img_res // 2, self.img_res // 2] + self.faces = faces + + def __call__(self, + vertices: np.array, + camera_translation: np.array, + image: torch.Tensor, + full_frame: bool = False, + imgname: Optional[str] = None, + side_view=False, rot_angle=90, + mesh_base_color=(1.0, 1.0, 0.9), + scene_bg_color=(0,0,0), + return_rgba=False, + ) -> np.array: + """ + Render meshes on input image + Args: + vertices (np.array): Array of shape (V, 3) containing the mesh vertices. + camera_translation (np.array): Array of shape (3,) with the camera translation. + image (torch.Tensor): Tensor of shape (3, H, W) containing the image crop with normalized pixel values. + full_frame (bool): If True, then render on the full image. + imgname (Optional[str]): Contains the original image filenamee. Used only if full_frame == True. + """ + if full_frame: + image = cv2.imread(imgname).astype(np.float32)[:, :, ::-1] / 255. + else: + image = image.clone() * torch.tensor(self.cfg.MODEL.IMAGE_STD, device=image.device).reshape(3,1,1) + image = image + torch.tensor(self.cfg.MODEL.IMAGE_MEAN, device=image.device).reshape(3,1,1) + image = image.permute(1, 2, 0).cpu().numpy() + + renderer = pyrender.OffscreenRenderer(viewport_width=image.shape[1], + viewport_height=image.shape[0], + point_size=1.0) + material = pyrender.MetallicRoughnessMaterial( + metallicFactor=0.0, + alphaMode='OPAQUE', + baseColorFactor=(*mesh_base_color, 1.0)) + + camera_translation[0] *= -1. + + mesh = trimesh.Trimesh(vertices.copy(), self.faces.copy()) + if side_view: + rot = trimesh.transformations.rotation_matrix( + np.radians(rot_angle), [0, 1, 0]) + mesh.apply_transform(rot) + rot = trimesh.transformations.rotation_matrix( + np.radians(180), [1, 0, 0]) + mesh.apply_transform(rot) + mesh = pyrender.Mesh.from_trimesh(mesh, material=material) + + scene = pyrender.Scene(bg_color=[*scene_bg_color, 0.0], + ambient_light=(0.3, 0.3, 0.3)) + scene.add(mesh, 'mesh') + + camera_pose = np.eye(4) + camera_pose[:3, 3] = camera_translation + camera_center = [image.shape[1] / 2., image.shape[0] / 2.] + camera = pyrender.IntrinsicsCamera(fx=self.focal_length, fy=self.focal_length, + cx=camera_center[0], cy=camera_center[1], zfar=1e12) + scene.add(camera, pose=camera_pose) + + + light_nodes = create_raymond_lights() + for node in light_nodes: + scene.add_node(node) + + color, rend_depth = renderer.render(scene, flags=pyrender.RenderFlags.RGBA) + color = color.astype(np.float32) / 255.0 + renderer.delete() + + if return_rgba: + return color + + valid_mask = (color[:, :, -1])[:, :, np.newaxis] + if not side_view: + output_img = (color[:, :, :3] * valid_mask + (1 - valid_mask) * image) + else: + output_img = color[:, :, :3] + + output_img = output_img.astype(np.float32) + return output_img + + def vertices_to_trimesh(self, vertices, camera_translation, mesh_base_color=(1.0, 1.0, 0.9), + rot_axis=[1,0,0], rot_angle=0,): + # material = pyrender.MetallicRoughnessMaterial( + # metallicFactor=0.0, + # alphaMode='OPAQUE', + # baseColorFactor=(*mesh_base_color, 1.0)) + vertex_colors = np.array([(*mesh_base_color, 1.0)] * vertices.shape[0]) + # print(vertices.shape, camera_translation.shape) + mesh = trimesh.Trimesh(vertices.copy() + camera_translation, self.faces.copy(), vertex_colors=vertex_colors) + # mesh = trimesh.Trimesh(vertices.copy(), self.faces.copy()) + + rot = trimesh.transformations.rotation_matrix( + np.radians(rot_angle), rot_axis) + mesh.apply_transform(rot) + + rot = trimesh.transformations.rotation_matrix( + np.radians(180), [1, 0, 0]) + mesh.apply_transform(rot) + return mesh + + def render_rgba( + self, + vertices: np.array, + cam_t = None, + rot=None, + rot_axis=[1,0,0], + rot_angle=0, + camera_z=3, + # camera_translation: np.array, + mesh_base_color=(1.0, 1.0, 0.9), + scene_bg_color=(0,0,0), + render_res=[256, 256], + ): + + renderer = pyrender.OffscreenRenderer(viewport_width=render_res[0], + viewport_height=render_res[1], + point_size=1.0) + # material = pyrender.MetallicRoughnessMaterial( + # metallicFactor=0.0, + # alphaMode='OPAQUE', + # baseColorFactor=(*mesh_base_color, 1.0)) + + if cam_t is not None: + camera_translation = cam_t.copy() + # camera_translation[0] *= -1. + else: + camera_translation = np.array([0, 0, camera_z * self.focal_length/render_res[1]]) + + mesh = self.vertices_to_trimesh(vertices, camera_translation, mesh_base_color, rot_axis, rot_angle) + mesh = pyrender.Mesh.from_trimesh(mesh) + # mesh = pyrender.Mesh.from_trimesh(mesh, material=material) + + scene = pyrender.Scene(bg_color=[*scene_bg_color, 0.0], + ambient_light=(0.3, 0.3, 0.3)) + scene.add(mesh, 'mesh') + + camera_pose = np.eye(4) + # camera_pose[:3, 3] = camera_translation + camera_center = [render_res[0] / 2., render_res[1] / 2.] + camera = pyrender.IntrinsicsCamera(fx=self.focal_length, fy=self.focal_length, + cx=camera_center[0], cy=camera_center[1], zfar=1e12) + + # Create camera node and add it to pyRender scene + camera_node = pyrender.Node(camera=camera, matrix=camera_pose) + scene.add_node(camera_node) + self.add_point_lighting(scene, camera_node) + self.add_lighting(scene, camera_node) + + light_nodes = create_raymond_lights() + for node in light_nodes: + scene.add_node(node) + + color, rend_depth = renderer.render(scene, flags=pyrender.RenderFlags.RGBA) + color = color.astype(np.float32) / 255.0 + renderer.delete() + + return color + + def render_rgba_multiple( + self, + vertices: List[np.array], + cam_t: List[np.array], + rot_axis=[1,0,0], + rot_angle=0, + mesh_base_color=(1.0, 1.0, 0.9), + scene_bg_color=(0,0,0), + render_res=[256, 256], + focal_length=None, + ): + + renderer = pyrender.OffscreenRenderer(viewport_width=render_res[0], + viewport_height=render_res[1], + point_size=1.0) + # material = pyrender.MetallicRoughnessMaterial( + # metallicFactor=0.0, + # alphaMode='OPAQUE', + # baseColorFactor=(*mesh_base_color, 1.0)) + + mesh_list = [pyrender.Mesh.from_trimesh(self.vertices_to_trimesh(vvv, ttt.copy(), mesh_base_color, rot_axis, rot_angle)) for vvv,ttt in zip(vertices, cam_t)] + + scene = pyrender.Scene(bg_color=[*scene_bg_color, 0.0], + ambient_light=(0.3, 0.3, 0.3)) + for i,mesh in enumerate(mesh_list): + scene.add(mesh, f'mesh_{i}') + + camera_pose = np.eye(4) + # camera_pose[:3, 3] = camera_translation + camera_center = [render_res[0] / 2., render_res[1] / 2.] + focal_length = focal_length if focal_length is not None else self.focal_length + camera = pyrender.IntrinsicsCamera(fx=focal_length, fy=focal_length, + cx=camera_center[0], cy=camera_center[1], zfar=1e12) + + # Create camera node and add it to pyRender scene + camera_node = pyrender.Node(camera=camera, matrix=camera_pose) + scene.add_node(camera_node) + self.add_point_lighting(scene, camera_node) + self.add_lighting(scene, camera_node) + + light_nodes = create_raymond_lights() + for node in light_nodes: + scene.add_node(node) + + color, rend_depth = renderer.render(scene, flags=pyrender.RenderFlags.RGBA) + color = color.astype(np.float32) / 255.0 + renderer.delete() + + return color + + def add_lighting(self, scene, cam_node, color=np.ones(3), intensity=1.0): + # from phalp.visualize.py_renderer import get_light_poses + light_poses = get_light_poses() + light_poses.append(np.eye(4)) + cam_pose = scene.get_pose(cam_node) + for i, pose in enumerate(light_poses): + matrix = cam_pose @ pose + node = pyrender.Node( + name=f"light-{i:02d}", + light=pyrender.DirectionalLight(color=color, intensity=intensity), + matrix=matrix, + ) + if scene.has_node(node): + continue + scene.add_node(node) + + def add_point_lighting(self, scene, cam_node, color=np.ones(3), intensity=1.0): + # from phalp.visualize.py_renderer import get_light_poses + light_poses = get_light_poses(dist=0.5) + light_poses.append(np.eye(4)) + cam_pose = scene.get_pose(cam_node) + for i, pose in enumerate(light_poses): + matrix = cam_pose @ pose + # node = pyrender.Node( + # name=f"light-{i:02d}", + # light=pyrender.DirectionalLight(color=color, intensity=intensity), + # matrix=matrix, + # ) + node = pyrender.Node( + name=f"plight-{i:02d}", + light=pyrender.PointLight(color=color, intensity=intensity), + matrix=matrix, + ) + if scene.has_node(node): + continue + scene.add_node(node) diff --git a/fourm/utils/hmr2_utils/hmr2/utils/skeleton_renderer.py b/fourm/utils/hmr2_utils/hmr2/utils/skeleton_renderer.py new file mode 100755 index 0000000000000000000000000000000000000000..2ecc20d216165849c290e081e6e53a885a091e5f --- /dev/null +++ b/fourm/utils/hmr2_utils/hmr2/utils/skeleton_renderer.py @@ -0,0 +1,128 @@ +# -------------------------------------------------------- +# Based on the 4DHumans and ProHMR code bases +# https://github.com/shubham-goel/4D-Humans +# https://github.com/nkolot/ProHMR +# -------------------------------------------------------- + +import torch +import numpy as np +import trimesh +from typing import Optional +from yacs.config import CfgNode + +from .geometry import perspective_projection +from .render_openpose import render_openpose + +class SkeletonRenderer: + + def __init__(self, cfg: CfgNode): + """ + Object used to render 3D keypoints. Faster for use during training. + Args: + cfg (CfgNode): Model config file. + """ + self.cfg = cfg + + def __call__(self, + pred_keypoints_3d: torch.Tensor, + gt_keypoints_3d: torch.Tensor, + gt_keypoints_2d: torch.Tensor, + images: Optional[np.array] = None, + camera_translation: Optional[torch.Tensor] = None) -> np.array: + """ + Render batch of 3D keypoints. + Args: + pred_keypoints_3d (torch.Tensor): Tensor of shape (B, S, N, 3) containing a batch of predicted 3D keypoints, with S samples per image. + gt_keypoints_3d (torch.Tensor): Tensor of shape (B, N, 4) containing corresponding ground truth 3D keypoints; last value is the confidence. + gt_keypoints_2d (torch.Tensor): Tensor of shape (B, N, 3) containing corresponding ground truth 2D keypoints. + images (torch.Tensor): Tensor of shape (B, H, W, 3) containing images with values in the [0,255] range. + camera_translation (torch.Tensor): Tensor of shape (B, 3) containing the camera translation. + Returns: + np.array : Image with the following layout. Each row contains the a) input image, + b) image with gt 2D keypoints, + c) image with projected gt 3D keypoints, + d_1, ... , d_S) image with projected predicted 3D keypoints, + e) gt 3D keypoints rendered from a side view, + f_1, ... , f_S) predicted 3D keypoints frorm a side view + """ + batch_size = pred_keypoints_3d.shape[0] +# num_samples = pred_keypoints_3d.shape[1] + pred_keypoints_3d = pred_keypoints_3d.clone().cpu().float() + gt_keypoints_3d = gt_keypoints_3d.clone().cpu().float() + gt_keypoints_3d[:, :, :-1] = gt_keypoints_3d[:, :, :-1] - gt_keypoints_3d[:, [25+14], :-1] + pred_keypoints_3d[:, [25+14]] + gt_keypoints_2d = gt_keypoints_2d.clone().cpu().float().numpy() + gt_keypoints_2d[:, :, :-1] = self.cfg.MODEL.IMAGE_SIZE * (gt_keypoints_2d[:, :, :-1] + 1.0) / 2.0 + + openpose_indices = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + gt_indices = [12, 8, 7, 6, 9, 10, 11, 14, 2, 1, 0, 3, 4, 5] + gt_indices = [25 + i for i in gt_indices] + keypoints_to_render = torch.ones(batch_size, gt_keypoints_3d.shape[1], 1) + rotation = torch.eye(3).unsqueeze(0) + if camera_translation is None: + camera_translation = torch.tensor([0.0, 0.0, 2 * self.cfg.EXTRA.FOCAL_LENGTH / (0.8 * self.cfg.MODEL.IMAGE_SIZE)]).unsqueeze(0).repeat(batch_size, 1) + else: + camera_translation = camera_translation.cpu() + + if images is None: + images = np.zeros((batch_size, self.cfg.MODEL.IMAGE_SIZE, self.cfg.MODEL.IMAGE_SIZE, 3)) + focal_length = torch.tensor([self.cfg.EXTRA.FOCAL_LENGTH, self.cfg.EXTRA.FOCAL_LENGTH]).reshape(1, 2) + camera_center = torch.tensor([self.cfg.MODEL.IMAGE_SIZE, self.cfg.MODEL.IMAGE_SIZE], dtype=torch.float).reshape(1, 2) / 2. + gt_keypoints_3d_proj = perspective_projection(gt_keypoints_3d[:, :, :-1], rotation=rotation.repeat(batch_size, 1, 1), translation=camera_translation[:, :], focal_length=focal_length.repeat(batch_size, 1), camera_center=camera_center.repeat(batch_size, 1)) + pred_keypoints_3d_proj = perspective_projection(pred_keypoints_3d.reshape(batch_size, -1, 3), rotation=rotation.repeat(batch_size, 1, 1), translation=camera_translation.reshape(batch_size, -1), focal_length=focal_length.repeat(batch_size, 1), camera_center=camera_center.repeat(batch_size, 1)).reshape(batch_size, -1, 2) + gt_keypoints_3d_proj = torch.cat([gt_keypoints_3d_proj, gt_keypoints_3d[:, :, [-1]]], dim=-1).cpu().numpy() + pred_keypoints_3d_proj = torch.cat([pred_keypoints_3d_proj, keypoints_to_render.reshape(batch_size, -1, 1)], dim=-1).cpu().numpy() + rows = [] + # Rotate keypoints to visualize side view + R = torch.tensor(trimesh.transformations.rotation_matrix(np.radians(90), [0, 1, 0])[:3, :3]).float() + gt_keypoints_3d_side = gt_keypoints_3d.clone() + gt_keypoints_3d_side[:, :, :-1] = torch.einsum('bni,ij->bnj', gt_keypoints_3d_side[:, :, :-1], R) + pred_keypoints_3d_side = pred_keypoints_3d.clone() + pred_keypoints_3d_side = torch.einsum('bni,ij->bnj', pred_keypoints_3d_side, R) + gt_keypoints_3d_proj_side = perspective_projection(gt_keypoints_3d_side[:, :, :-1], rotation=rotation.repeat(batch_size, 1, 1), translation=camera_translation[:, :], focal_length=focal_length.repeat(batch_size, 1), camera_center=camera_center.repeat(batch_size, 1)) + pred_keypoints_3d_proj_side = perspective_projection(pred_keypoints_3d_side.reshape(batch_size, -1, 3), rotation=rotation.repeat(batch_size, 1, 1), translation=camera_translation.reshape(batch_size, -1), focal_length=focal_length.repeat(batch_size, 1), camera_center=camera_center.repeat(batch_size, 1)).reshape(batch_size, -1, 2) + gt_keypoints_3d_proj_side = torch.cat([gt_keypoints_3d_proj_side, gt_keypoints_3d_side[:, :, [-1]]], dim=-1).cpu().numpy() + pred_keypoints_3d_proj_side = torch.cat([pred_keypoints_3d_proj_side, keypoints_to_render.reshape(batch_size, -1, 1)], dim=-1).cpu().numpy() + for i in range(batch_size): + img = images[i] + side_img = np.zeros((self.cfg.MODEL.IMAGE_SIZE, self.cfg.MODEL.IMAGE_SIZE, 3)) + # gt 2D keypoints + body_keypoints_2d = gt_keypoints_2d[i, :25].copy() + for op, gt in zip(openpose_indices, gt_indices): + if gt_keypoints_2d[i, gt, -1] > body_keypoints_2d[op, -1]: + body_keypoints_2d[op] = gt_keypoints_2d[i, gt] + gt_keypoints_img = render_openpose(img, body_keypoints_2d) / 255. + # gt 3D keypoints + body_keypoints_3d_proj = gt_keypoints_3d_proj[i, :25].copy() + for op, gt in zip(openpose_indices, gt_indices): + if gt_keypoints_3d_proj[i, gt, -1] > body_keypoints_3d_proj[op, -1]: + body_keypoints_3d_proj[op] = gt_keypoints_3d_proj[i, gt] + gt_keypoints_3d_proj_img = render_openpose(img, body_keypoints_3d_proj) / 255. + # gt 3D keypoints from the side + body_keypoints_3d_proj = gt_keypoints_3d_proj_side[i, :25].copy() + for op, gt in zip(openpose_indices, gt_indices): + if gt_keypoints_3d_proj_side[i, gt, -1] > body_keypoints_3d_proj[op, -1]: + body_keypoints_3d_proj[op] = gt_keypoints_3d_proj_side[i, gt] + gt_keypoints_3d_proj_img_side = render_openpose(side_img, body_keypoints_3d_proj) / 255. + # pred 3D keypoints + pred_keypoints_3d_proj_imgs = [] + body_keypoints_3d_proj = pred_keypoints_3d_proj[i, :25].copy() + for op, gt in zip(openpose_indices, gt_indices): + if pred_keypoints_3d_proj[i, gt, -1] >= body_keypoints_3d_proj[op, -1]: + body_keypoints_3d_proj[op] = pred_keypoints_3d_proj[i, gt] + pred_keypoints_3d_proj_imgs.append(render_openpose(img, body_keypoints_3d_proj) / 255.) + pred_keypoints_3d_proj_img = np.concatenate(pred_keypoints_3d_proj_imgs, axis=1) + # gt 3D keypoints from the side + pred_keypoints_3d_proj_imgs_side = [] + body_keypoints_3d_proj = pred_keypoints_3d_proj_side[i, :25].copy() + for op, gt in zip(openpose_indices, gt_indices): + if pred_keypoints_3d_proj_side[i, gt, -1] >= body_keypoints_3d_proj[op, -1]: + body_keypoints_3d_proj[op] = pred_keypoints_3d_proj_side[i, gt] + pred_keypoints_3d_proj_imgs_side.append(render_openpose(side_img, body_keypoints_3d_proj) / 255.) + pred_keypoints_3d_proj_img_side = np.concatenate(pred_keypoints_3d_proj_imgs_side, axis=1) + rows.append(np.concatenate((gt_keypoints_img, gt_keypoints_3d_proj_img, pred_keypoints_3d_proj_img, gt_keypoints_3d_proj_img_side, pred_keypoints_3d_proj_img_side), axis=1)) + # Concatenate images + img = np.concatenate(rows, axis=0) + img[:, ::self.cfg.MODEL.IMAGE_SIZE, :] = 1.0 + img[::self.cfg.MODEL.IMAGE_SIZE, :, :] = 1.0 + img[:, (1+1+1)*self.cfg.MODEL.IMAGE_SIZE, :] = 0.5 + return img diff --git a/fourm/utils/logger.py b/fourm/utils/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..35859ab33323e4587f5297becc86b1421dc0c0ff --- /dev/null +++ b/fourm/utils/logger.py @@ -0,0 +1,228 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -------------------------------------------------------- +# Based on DETR code base +# https://github.com/facebookresearch/detr +# -------------------------------------------------------- +import datetime +import logging +import time +from collections import defaultdict, deque + +import torch +import torch.distributed as dist + +try: + import wandb +except: + pass + +from .dist import is_dist_avail_and_initialized + + +class SmoothedValue(object): + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=20, fmt=None): + if fmt is None: + fmt = "{median:.4f} ({global_avg:.4f})" + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + self.fmt = fmt + + def update(self, value, n=1): + self.deque.append(value) + self.count += n + self.total += value * n + + def synchronize_between_processes(self): + """ + Warning: does not synchronize the deque! + """ + if not is_dist_avail_and_initialized(): + return + t = torch.tensor([self.count, self.total], dtype=torch.float64, device='cuda') + dist.barrier() + dist.all_reduce(t) + t = t.tolist() + self.count = int(t[0]) + self.total = t[1] + + @property + def median(self): + d = torch.tensor(list(self.deque)) + return d.median().item() + + @property + def avg(self): + d = torch.tensor(list(self.deque), dtype=torch.float32) + return d.mean().item() + + @property + def global_avg(self): + return self.total / self.count + + @property + def max(self): + return max(self.deque) + + @property + def value(self): + return self.deque[-1] + + def __str__(self): + return self.fmt.format( + median=self.median, + avg=self.avg, + global_avg=self.global_avg, + max=self.max, + value=self.value) + + +class MetricLogger(object): + def __init__(self, delimiter="\t"): + self.meters = defaultdict(SmoothedValue) + self.delimiter = delimiter + + def update(self, **kwargs): + for k, v in kwargs.items(): + if v is None: + continue + if isinstance(v, torch.Tensor): + v = v.item() + assert isinstance(v, (float, int)) + self.meters[k].update(v) + + def __getattr__(self, attr): + if attr in self.meters: + return self.meters[attr] + if attr in self.__dict__: + return self.__dict__[attr] + raise AttributeError("'{}' object has no attribute '{}'".format( + type(self).__name__, attr)) + + def __str__(self): + loss_str = [] + for name, meter in self.meters.items(): + loss_str.append( + "{}: {}".format(name, str(meter)) + ) + return self.delimiter.join(loss_str) + + def synchronize_between_processes(self): + for meter in self.meters.values(): + meter.synchronize_between_processes() + + def add_meter(self, name, meter): + self.meters[name] = meter + + def log_every(self, iterable, print_freq, iter_len=None, header=None): + iter_len = iter_len if iter_len is not None else len(iterable) + i = 0 + if not header: + header = '' + start_time = time.time() + end = time.time() + iter_time = SmoothedValue(fmt='{avg:.4f}') + data_time = SmoothedValue(fmt='{avg:.4f}') + space_fmt = ':' + str(len(str(iter_len))) + 'd' + log_msg = [ + header, + '[{0' + space_fmt + '}/{1}]', + 'eta: {eta}', + '{meters}', + 'time: {time}', + 'data: {data}' + ] + if torch.cuda.is_available(): + log_msg.append('max mem: {memory:.0f}') + log_msg = self.delimiter.join(log_msg) + MB = 1024.0 * 1024.0 + for obj in iterable: + data_time.update(time.time() - end) + yield obj + iter_time.update(time.time() - end) + if i % print_freq == 0 or i == iter_len - 1: + if iter_len > 0: + eta_seconds = iter_time.global_avg * (iter_len - i) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + else: + eta_string = '?' + if torch.cuda.is_available(): + print(log_msg.format( + i, iter_len if iter_len > 0 else '?', eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time), + memory=torch.cuda.max_memory_allocated() / MB)) + else: + print(log_msg.format( + i, iter_len if iter_len > 0 else '?', eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time))) + i += 1 + end = time.time() + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + time_per_iter_str = '{:.4f}'.format(total_time / iter_len) if iter_len > 0 else '?' + print('{} Total time: {} ({} s / it)'.format( + header, total_time_str, time_per_iter_str)) + + +class WandbLogger(object): + def __init__(self, args): + wandb.init( + config=args, + entity=args.wandb_entity, + project=args.wandb_project, + group=getattr(args, 'wandb_group', None), + name=getattr(args, 'wandb_run_name', None), + tags=getattr(args, 'wandb_tags', None), + mode=getattr(args, 'wandb_mode', 'online'), + ) + + @staticmethod + def wandb_safe_log(*args, **kwargs): + try: + wandb.log(*args, **kwargs) + except (wandb.CommError, BrokenPipeError): + logging.error('wandb logging failed, skipping...') + + def set_step(self, step=None): + if step is not None: + self.step = step + else: + self.step += 1 + + def update(self, metrics): + log_dict = dict() + for k, v in metrics.items(): + if v is None: + continue + if isinstance(v, torch.Tensor): + v = v.item() + log_dict[k] = v + + self.wandb_safe_log(log_dict, step=self.step) + + def flush(self): + pass + + def finish(self): + try: + wandb.finish() + except (wandb.CommError, BrokenPipeError): + logging.error('wandb failed to finish') \ No newline at end of file diff --git a/fourm/utils/misc.py b/fourm/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..d4fde24d442b057b9f8986736b2992f934fb81ef --- /dev/null +++ b/fourm/utils/misc.py @@ -0,0 +1,58 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib + +import collections.abc +from itertools import repeat +import torchvision.transforms.functional as TF + +from fourm.utils.data_constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD + + +def denormalize(img, mean=IMAGENET_DEFAULT_MEAN, std=IMAGENET_DEFAULT_STD): + """ + Denormalizes an image. + + Args: + img (torch.Tensor): Image to denormalize. + mean (tuple): Mean to use for denormalization. + std (tuple): Standard deviation to use for denormalization. + """ + return TF.normalize( + img.clone(), + mean= [-m/s for m, s in zip(mean, std)], + std= [1/s for s in std] + ) + + +def generate_uint15_hash(seed_str): + """Generates a hash of the seed string as an unsigned int15 integer""" + return int(hashlib.sha256(seed_str.encode('utf-8')).hexdigest(), 16) % (2**15) + + +# From PyTorch internals +def _ntuple(n): + def parse(x): + if isinstance(x, collections.abc.Iterable): + return x + return tuple(repeat(x, n)) + + return parse + + +to_1tuple = _ntuple(1) +to_2tuple = _ntuple(2) +to_3tuple = _ntuple(3) +to_4tuple = _ntuple(4) +to_ntuple = _ntuple diff --git a/fourm/utils/native_scaler.py b/fourm/utils/native_scaler.py new file mode 100644 index 0000000000000000000000000000000000000000..b22ff5bcc154aad6652331eeafb7fae7ca2a1d69 --- /dev/null +++ b/fourm/utils/native_scaler.py @@ -0,0 +1,65 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -------------------------------------------------------- +# Based on timm code base +# https://github.com/rwightman/pytorch-image-models/tree/master/timm +# -------------------------------------------------------- + +import torch + +class NativeScalerWithGradNormCount: + state_dict_key = "amp_scaler" + + def __init__(self, enabled=True): + self._scaler = torch.cuda.amp.GradScaler(enabled=enabled) + + def __call__(self, loss, optimizer, clip_grad=None, skip_grad=None, parameters=None, create_graph=False, update_grad=True, compute_grad_norm=True): + self._scaler.scale(loss).backward(create_graph=create_graph) + if update_grad: + if clip_grad is not None: + assert parameters is not None + self._scaler.unscale_(optimizer) # unscale the gradients of optimizer's assigned params in-place + norm = torch.nn.utils.clip_grad_norm_(parameters, clip_grad) + elif skip_grad is not None: + self._scaler.unscale_(optimizer) + norm = get_grad_norm_(parameters) + if norm >= skip_grad: + self._scaler.update() + return norm + else: + self._scaler.unscale_(optimizer) + norm = get_grad_norm_(parameters) if compute_grad_norm else None + self._scaler.step(optimizer) + self._scaler.update() + else: + norm = None + return norm + + def state_dict(self): + return self._scaler.state_dict() + + def load_state_dict(self, state_dict): + self._scaler.load_state_dict(state_dict) + + +def get_grad_norm_(parameters, norm_type: float = 2.0) -> torch.Tensor: + if isinstance(parameters, torch.Tensor): + parameters = [parameters] + parameters = [p for p in parameters if p.grad is not None] + norm_type = float(norm_type) + if len(parameters) == 0: + return torch.tensor(0.) + device = parameters[0].grad.device + total_norm = torch.norm(torch.stack([torch.norm(p.grad.detach(), norm_type).to(device) for p in parameters]), norm_type) + return total_norm \ No newline at end of file diff --git a/fourm/utils/optim_factory.py b/fourm/utils/optim_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..b9c479aa36d477707855abfcdc4ea2cf85ad404b --- /dev/null +++ b/fourm/utils/optim_factory.py @@ -0,0 +1,245 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -------------------------------------------------------- +# Based on BEiT, timm, DINO, DeiT code base +# https://github.com/microsoft/unilm/tree/master/beit +# https://github.com/rwightman/pytorch-image-models/tree/master/timm +# https://github.com/facebookresearch/deit +# https://github.com/facebookresearch/dino +# -------------------------------------------------------- +import json + +import torch +from torch import optim as optim + + +def get_num_layer_for_vit(var_name, num_max_layer): + if var_name in ("cls_token", "mask_token", "pos_embed", "global_tokens"): + return 0 + elif var_name.startswith("patch_embed"): + return 0 + elif var_name.startswith("input_adapters") or var_name.startswith("encoder_embeddings"): + return 0 + elif var_name.startswith("rel_pos_bias"): + return num_max_layer - 1 + elif var_name.startswith("blocks") or (var_name.startswith("encoder.") and not var_name.startswith("encoder_norm")): + layer_id = int(var_name.split('.')[1]) + return layer_id + 1 + else: + return num_max_layer - 1 + + +def get_num_layer_for_beit(var_name, num_max_layer): + if "embed" in var_name: + return 0 + elif var_name in ( + "cls_token", "mask_token", "pos_embed", "language_pos_embed", + "word_embeddings.weight", "vision_cls_token", "vision_pos_embed" + ): + return 0 + elif var_name.startswith("patch_embed"): + return 0 + elif var_name.startswith("rel_pos_bias"): + return num_max_layer - 1 + elif "layers." in var_name: + layer_id = int(var_name.split('layers.')[1].split('.')[0]) + return layer_id + 1 + else: + return num_max_layer - 1 + + +def get_num_layer_for_fm(var_name, num_enc_layers, num_dec_layers, last_layer_mod_emb=False): + """Layers go from 0 to (num_enc + num_dec + 1) + where 0 is the encoder embedding and (num_enc + num_dec + 1) is the projection following the decoder + """ + if var_name.startswith("encoder_embeddings"): + return 0 + elif var_name.startswith("encoder."): + layer_id = int(var_name.split('.')[1]) + return layer_id + 1 + elif var_name in ("encoder_norm", "decoder_proj_context", "mask_token"): + return num_enc_layers + elif not last_layer_mod_emb and var_name.startswith("decoder_embeddings.") and "mod_emb" in var_name: + return num_enc_layers + elif var_name.startswith("decoder."): + layer_id = int(var_name.split('.')[1]) + return num_enc_layers + layer_id + 1 + else: + return num_enc_layers + num_dec_layers + 1 + + +class LayerDecayValueAssigner(object): + def __init__(self, values, is_beit3=False): + self.values = values + self.is_beit3 = is_beit3 + + def get_scale(self, layer_id): + return self.values[layer_id] + + def get_layer_id(self, var_name): + if self.is_beit3: + return get_num_layer_for_beit(var_name, len(self.values)) + else: + return get_num_layer_for_vit(var_name, len(self.values)) + +class LayerDecayValueAssignerForFourM(object): + def __init__(self, values, num_enc_layers, num_dec_layers, last_layer_mod_emb=False): + self.values = values + self.num_enc_layers = num_enc_layers + self.num_dec_layers = num_dec_layers + self.last_layer_mod_emb = last_layer_mod_emb + assert len(values) == num_enc_layers + num_dec_layers + 2 + + def get_scale(self, layer_id): + return self.values[layer_id] + + def get_layer_id(self, var_name): + return get_num_layer_for_fm(var_name, num_enc_layers=self.num_enc_layers, num_dec_layers=self.num_dec_layers, last_layer_mod_emb=self.last_layer_mod_emb) + + +def get_parameter_groups( + model, weight_decay=1e-5, skip_list=(), get_num_layer=None, get_layer_scale=None, + decoder_decay=None, decoder_list=(), no_lr_scale_list=[]): + parameter_group_names = {} + parameter_group_vars = {} + + for name, param in model.named_parameters(): + # Remove wrapped module to be compatible with FSDP + name = name.replace("_fsdp_wrapped_module.", "") + + if not param.requires_grad: + continue # frozen weights + + # Assign weight decay values + # Only norm and bias terms should have no decay + # Previously, this checked if (param.shape) == 1 which is incompatible with FSDP which flattens all params + if "norm." in name or ".norm" in name or name.endswith(".bias") or name.endswith(".lookup_table_weight") or name.endswith(".gamma") or name in skip_list: + group_name = "no_decay" + this_weight_decay = 0. + elif decoder_decay is not None and (name.startswith("decoder.") or name in decoder_list): + group_name = "decoder_decay" + this_weight_decay = decoder_decay + else: + group_name = "decay" + this_weight_decay = weight_decay + + # Assign layer ID for LR scaling + skip_scale = False + if get_num_layer is not None: + layer_id = get_num_layer(name) + group_name = "layer_%d_%s" % (layer_id, group_name) + if name in no_lr_scale_list: + skip_scale = True + group_name = f'{group_name}_no_lr_scale' + else: + layer_id = None + + if group_name not in parameter_group_names: + if get_layer_scale is not None and not skip_scale: + scale = get_layer_scale(layer_id) + else: + scale = 1. + + parameter_group_names[group_name] = { + "weight_decay": this_weight_decay, + "params": [], + "lr_scale": scale + } + parameter_group_vars[group_name] = { + "weight_decay": this_weight_decay, + "params": [], + "lr_scale": scale + } + + parameter_group_vars[group_name]["params"].append(param) + parameter_group_names[group_name]["params"].append(name) + print("Param groups = %s" % json.dumps(parameter_group_names, indent=2)) + return list(parameter_group_vars.values()) + + +def create_optimizer(args, model, get_num_layer=None, get_layer_scale=None, filter_bias_and_bn=True, skip_list=None): + """ + Model can either be a single nn.Module, or a dictionary with {'model': model, 'balancer': balancer}. + """ + opt_lower = args.opt.lower() + weight_decay = args.weight_decay + try: + decoder_decay = args.decoder_decay + except: + decoder_decay = None + try: + no_lr_scale_list = args.no_lr_scale_list.split('-') + except: + no_lr_scale_list = [] + + def get_parameters(m): + if weight_decay and filter_bias_and_bn: + skip = {} + if skip_list is not None: + skip = skip_list + elif hasattr(m, 'no_weight_decay'): + skip = m.no_weight_decay() + decoder={} + if hasattr(m, 'decoder_weight_decay'): + decoder = m.decoder_weight_decay() + parameters = get_parameter_groups(m, weight_decay, skip, get_num_layer, get_layer_scale, decoder_decay, decoder, no_lr_scale_list) + wd = 0. + else: + parameters = m.parameters() + wd = weight_decay + return parameters, wd + + if isinstance(model, torch.nn.Module): + parameters, weight_decay = get_parameters(model) + elif isinstance(model, dict): + print("WARNING: Weight decay assignment is skipped. All layers are assigned a weight decay value." ) + parameters = [ + { + "params": [p for n, p in model['model'].named_parameters() + if p.requires_grad], + "lr_scale": 1., + }, + { + "params": [p for n, p in model['balancer'].named_parameters() + if p.requires_grad], + "lr_scale": args.balancer_lr_scale, + }, + ] + + opt_args = dict(lr=args.lr, weight_decay=weight_decay) + if hasattr(args, 'opt_eps') and args.opt_eps is not None: + opt_args['eps'] = args.opt_eps + if hasattr(args, 'opt_betas') and args.opt_betas is not None: + opt_args['betas'] = args.opt_betas + + print("optimizer settings:", opt_args) + + opt_split = opt_lower.split('_') + opt_lower = opt_split[-1] + + if opt_lower == 'sgd' or opt_lower == 'nesterov': + opt_args.pop('eps', None) + optimizer = optim.SGD(parameters, momentum=args.momentum, nesterov=True, **opt_args) + elif opt_lower == 'momentum': + opt_args.pop('eps', None) + optimizer = optim.SGD(parameters, momentum=args.momentum, nesterov=False, **opt_args) + elif opt_lower == 'adam': + optimizer = optim.Adam(parameters, **opt_args) + elif opt_lower == 'adamw': + optimizer = optim.AdamW(parameters, **opt_args) + else: + assert False and "Invalid optimizer" + raise ValueError + + return optimizer \ No newline at end of file diff --git a/fourm/utils/plotting_utils.py b/fourm/utils/plotting_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..addc5dec845c06780a17dfc2dc7cc53983604cae --- /dev/null +++ b/fourm/utils/plotting_utils.py @@ -0,0 +1,1346 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import numpy as np +import torch +import torchvision.transforms.functional as TF +from einops import rearrange +import textwrap +import matplotlib.pyplot as plt +from PIL import Image +import cv2 +from itertools import groupby + +# For visualizing CLIP feature maps +from sklearn.decomposition import PCA + +# Detectron2 for semantic segmentation visualizations +try: + from detectron2.utils.visualizer import ColorMode, Visualizer + from detectron2.data import MetadataCatalog + coco_metadata = MetadataCatalog.get("coco_2017_val_panoptic") + USE_DETECTRON = True +except Exception as e: + print(e) + print("Detectron2 can be used for semseg visualizations. Please install detectron2 to use this feature, or plotting will fall back to matplotlib.") + USE_DETECTRON = False + +from fourm.data.modality_transforms import get_transform_key, get_transform_resolution, MetadataTransform +from fourm.utils.data_constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, COCO_SEMSEG_NUM_CLASSES +from fourm.utils import denormalize, get_sentinel_to_id_mapping, merge_span_masking +from fourm.utils.generation import unbatch + +device = 'cuda' if torch.cuda.is_available() else 'cpu' + + +def tensor_to_images(tensor): + """ + Converts a (B C H W) tensor to numpy arrays. + If B = 1, the tensor is unbatched and converted to a single image. + If C = 1, the channel dimension is removed. + + Args: + tensor (torch.Tensor): Tensor to convert to images. + """ + B, C, H, W = tensor.shape + if B == 1: + img = rearrange(unbatch(tensor), "c h w -> h w c") + else: + img = rearrange(tensor, "b c h w -> b h w c") + if C == 1: + img = img[..., 0] + return img.detach().cpu().numpy() + +def pca_visualize(features, n_components=3): + """ + Visualizes a feature map using PCA. + + Args: + features (torch.Tensor): CxHxW feature map to visualize. + n_components (int): Number of PCA components to use. + """ + C, H, W = features.shape + features_flat = rearrange(features.float(), 'c h w -> (h w) c').detach().cpu().numpy() + pca = PCA(n_components=n_components) + img_pca = rearrange(pca.fit_transform(features_flat), '(h w) c -> h w c', h=H, w=W) + img_pca = (img_pca - img_pca.min()) / (img_pca.max() - img_pca.min()) + return img_pca + +def np_squeeze(array, axis=0): + """ + Squeeses a numpy array along a given axis if that axis is one-dimensional. + Otherwise, it returns the same array. + + Args: + array (numpy.ndarray): Array to squeeze. + axis (int): Axis to squeeze. + """ + if array.shape[axis] == 1: + return np.squeeze(array, axis=axis) + else: + return array + +def decode_input_rgb(mod_dict, key='rgb'): + """ + Decodes (denormalizes) an RGB image from a model dictionary. + + Args: + mod_dict (dict): Model output dictionary. + key (str): Key of the RGB modality to decode. + """ + img = denormalize(mod_dict[key]['tensor']) + return tensor_to_images(img) + +def decode_tok_rgb(mod_dict, tokenizers, key='tok_rgb', image_size=224, patch_size=16, t=25, verbose=False): + """ + Decodes a sequence of RGB tokens from a model dictionary into an RGB image. + + Args: + mod_dict (dict): Model output dictionary. + tokenizers (dict): Dictionary of tokenizers. + key (str): Key of the tokenized RGB modality to decode. + image_size (int): Size of the image. + patch_size (int): Size of the patches. + t (int): Number of timesteps to decode using the tokenizer diffusion model (if applicable). + verbose (bool): Whether to print the decoding progress. + """ + img_tok = rearrange(mod_dict[key]['tensor'], "b (nh nw) -> b nh nw", nh=image_size//patch_size, nw=image_size//patch_size) + rec = tokenizers[get_transform_key(key)].decode_tokens(img_tok, timesteps=t, image_size=image_size, verbose=verbose) + rec = denormalize(rec, (0.5, 0.5, 0.5), (0.5, 0.5, 0.5)).clamp(0, 1) + return tensor_to_images(rec) + +def decode_tok_rgb_controlnet(mod_dict, tokenizers, key='tok_rgb', image_size=224, patch_size=16, + t=25, guidance_scale=2.5, cond_scale=0.8, verbose=False): + """ + Decodes a sequence of RGB tokens from a model dictionary into an RGB image using a ControlNet. + + Args: + mod_dict (dict): Model output dictionary. + tokenizers (dict): Dictionary of tokenizers. Needs to contain the key 'controlnet'. + key (str): Key of the tokenized RGB modality to decode. + image_size (int): Size of the image. + patch_size (int): Size of the patches. + t (int): Number of timesteps to decode using the ControlNet. + guidance_scale (float): Classifier-free guidance scale. + cond_scale (float): ControlNet conditioning scale. + verbose (bool): Whether to print the decoding progress. + """ + img_tok = rearrange(mod_dict[key]['tensor'], "b (nh nw) -> b nh nw", nh=image_size//patch_size, nw=image_size//patch_size) + rec = tokenizers['controlnet'].decode_tokens( + img_tok, timesteps=t, guidance_scale=guidance_scale, cond_scale=cond_scale, verbose=verbose + ) + rec = tokenizers['controlnet'].vae_decode(rec) + rec = denormalize(rec, (0.5, 0.5, 0.5), (0.5, 0.5, 0.5)).clamp(0, 1) + return tensor_to_images(rec) + +def decode_tok_normal(mod_dict, tokenizers, key='tok_normal', image_size=224, patch_size=16, t=25, verbose=False): + """ + Decodes a sequence of surface normal tokens from a model dictionary into an RGB image. + + Args: + mod_dict (dict): Model output dictionary. + tokenizers (dict): Dictionary of tokenizers. + key (str): Key of the tokenized normal modality to decode. + image_size (int): Size of the image. + patch_size (int): Size of the patches. + t (int): Number of timesteps to decode using the tokenizer diffusion model (if applicable). + verbose (bool): Whether to print the decoding progress. + """ + img_tok = rearrange(mod_dict[key]['tensor'], "b (nh nw) -> b nh nw", nh=image_size//patch_size, nw=image_size//patch_size) + rec = tokenizers[get_transform_key(key)].decode_tokens(img_tok, timesteps=t, image_size=image_size, verbose=verbose) + rec = denormalize(rec, (0.5, 0.5, 0.5), (0.5, 0.5, 0.5)).clamp(0, 1) + return tensor_to_images(rec) + +def decode_tok_canny_edge(mod_dict, tokenizers, key='tok_canny_edge', image_size=224, patch_size=16, t=10, verbose=False): + """ + Decodes a sequence of Canny edges tokens from a model dictionary into an RGB image. + + Args: + mod_dict (dict): Model output dictionary. + tokenizers (dict): Dictionary of tokenizers. + key (str): Key of the tokenized Canny edges modality to decode. + image_size (int): Size of the image. + patch_size (int): Size of the patches. + t (int): Number of timesteps to decode using the tokenizer diffusion model (if applicable). + verbose (bool): Whether to print the decoding progress. + """ + img_tok = rearrange(mod_dict[key]['tensor'], "b (nh nw) -> b nh nw", nh=image_size//patch_size, nw=image_size//patch_size) + rec = tokenizers[get_transform_key(key)].decode_tokens(img_tok, timesteps=t, image_size=image_size, verbose=verbose) + rec = (0.5*(rec+1)).clamp(0, 1) + return tensor_to_images(rec) + +def decode_tok_sam_edge(mod_dict, tokenizers, key='tok_sam_edge', image_size=224, patch_size=16, t=10, verbose=False): + """ + Decodes a sequence of SAM edges from a model dictionary into an RGB image. + + Args: + mod_dict (dict): Model output dictionary. + tokenizers (dict): Dictionary of tokenizers. + key (str): Key of the tokenized SAM edges modality to decode. + image_size (int): Size of the image. + patch_size (int): Size of the patches. + t (int): Number of timesteps to decode using the tokenizer diffusion model (if applicable). + verbose (bool): Whether to print the decoding progress. + """ + img_tok = rearrange(mod_dict[key]['tensor'], "b (nh nw) -> b nh nw", nh=image_size//patch_size, nw=image_size//patch_size) + rec = tokenizers[get_transform_key(key)].decode_tokens(img_tok, timesteps=t, image_size=image_size, verbose=verbose) + rec = (0.5*(rec+1)).clamp(0, 1) + return tensor_to_images(rec) + +def decode_tok_depth(mod_dict, tokenizers, key='tok_depth', image_size=224, patch_size=16, t=25, verbose=False, cmap='turbo'): + """ + Decodes a sequence of depth tokens from a model dictionary into an RGB image. + + Args: + mod_dict (dict): Model output dictionary. + tokenizers (dict): Dictionary of tokenizers. + key (str): Key of the tokenized depth modality to decode. + image_size (int): Size of the image. + patch_size (int): Size of the patches. + t (int): Number of timesteps to decode using the tokenizer diffusion model (if applicable). + verbose (bool): Whether to print the decoding progress. + cmap (str): Colormap to use for the depth image. + """ + img_tok = rearrange(mod_dict[key]['tensor'], "b (nh nw) -> b nh nw", nh=image_size//patch_size, nw=image_size//patch_size) + rec = tokenizers[get_transform_key(key)].decode_tokens(img_tok, timesteps=t, image_size=image_size, verbose=verbose) + rec = rec.detach().cpu().numpy()[:,0] + + if cmap is None: + return rec + + colormap = plt.get_cmap('turbo') + imgs = [] + for img in rec: + img_norm = (img - np.min(img)) / (np.max(img) - np.min(img)) + rgb_image = colormap(img_norm)[..., :3] + imgs.append(rgb_image) + + rgb_image = np_squeeze(np.stack(imgs), axis=0) + + return rgb_image + +def decode_tok_semseg(rgb_img, mod_dict, tokenizers, key='tok_semseg', image_size=224, patch_size=16, use_detectron=True, return_logits=False): + """ + Decodes a sequence of semantic segmentation tokens from a model dictionary into an RGB image. + + Args: + rgb_img (torch.Tensor): RGB image to overlay the semantic segmentation on. + mod_dict (dict): Model output dictionary. + tokenizers (dict): Dictionary of tokenizers. + key (str): Key of the tokenized semantic segmentation modality to decode. + image_size (int): Size of the image. + patch_size (int): Size of the patches. + use_detectron (bool): Uses detectron2's visualization for the semseg output. + """ + tokens = mod_dict[key]['tensor'] + tokens = tokens.unsqueeze(0) if tokens.ndim == 1 else tokens + img_tok = rearrange(tokens, "b (nh nw) -> b nh nw", nh=image_size//patch_size, nw=image_size//patch_size) + rec = tokenizers[get_transform_key(key)].decode_tokens(img_tok).detach().cpu() + if return_logits: + return rec + semsegs = rec.argmax(1) + B, H, W = semsegs.shape + + if not use_detectron: + return semsegs if B > 1 else semsegs[0] + else: + rgb_imgs = [rgb_img] * B + imgs = [] + for rgb, semseg in zip(rgb_imgs, semsegs): + if USE_DETECTRON: + v = Visualizer(255*rgb, coco_metadata, scale=1.2, instance_mode=ColorMode.IMAGE_BW) + img = v.draw_sem_seg((semseg-1).cpu()).get_image() / 255.0 + else: + colormap = plt.get_cmap('viridis') + img = colormap(semseg.cpu())[..., :3] + imgs.append(img) + imgs = np_squeeze(np.stack(imgs), axis=0) + return imgs + +def decode_tok_clip(mod_dict, tokenizers, key='tok_clip', image_size=224, patch_size=16): + """ + Decodes a sequence of CLIP tokens from a model dictionary into an PCA representation. + + Args: + mod_dict (dict): Model output dictionary. + key (str): Key of the tokenized CLIP modality to decode. + tokenizers (dict): Dictionary of tokenizers. + image_size (int): Size of the image. + patch_size (int): Size of the patches. + """ + n_patches = image_size // patch_size + img_tok = rearrange(mod_dict[key]['tensor'], "b (nh nw) -> b nh nw", nh=n_patches, nw=n_patches) + rec = tokenizers[get_transform_key(key)].decode_tokens(img_tok) + pca_viz = [pca_visualize(feat) for feat in rec] + pca_viz = np_squeeze(np.stack(pca_viz), axis=0) + return pca_viz + +def decode_tok_dinov2(mod_dict, tokenizers, key='tok_dinov2', image_size=224, patch_size=14): + """ + Decodes a sequence of DINOv2 spatial tokens from a model dictionary into an PCA representation. + + Args: + mod_dict (dict): Model output dictionary. + key (str): Key of the tokenized CLIP modality to decode. + tokenizers (dict): Dictionary of tokenizers. + image_size (int): Size of the image. + patch_size (int): Size of the patches. + """ + patch_size = 14 + n_patches = image_size // patch_size + img_tok = rearrange(mod_dict[key]['tensor'], "b (nh nw) -> b nh nw", nh=n_patches, nw=n_patches) + rec = tokenizers[get_transform_key(key)].decode_tokens(img_tok) + pca_viz = [pca_visualize(feat) for feat in rec] + pca_viz = np_squeeze(np.stack(pca_viz), axis=0) + return pca_viz + +def decode_tok_imagebind(mod_dict, tokenizers, key='tok_imagebind', image_size=224, patch_size=14): + """ + Decodes a sequence of ImageBind spatial tokens from a model dictionary into an PCA representation. + + Args: + mod_dict (dict): Model output dictionary. + key (str): Key of the tokenized CLIP modality to decode. + tokenizers (dict): Dictionary of tokenizers. + image_size (int): Size of the image. + patch_size (int): Size of the patches. + """ + patch_size = 14 + n_patches = image_size // patch_size + img_tok = rearrange(mod_dict[key]['tensor'], "b (nh nw) -> b nh nw", nh=n_patches, nw=n_patches) + rec = tokenizers[get_transform_key(key)].decode_tokens(img_tok) + pca_viz = [pca_visualize(feat) for feat in rec] + pca_viz = np_squeeze(np.stack(pca_viz), axis=0) + return pca_viz + +def decode_tok_dinov2_global(mod_dict, tokenizers, key='tok_dinov2_global'): + """ + Decodes a sequence of DINOv2 global tokens from a model dictionary. + + Args: + mod_dict (dict): Model output dictionary. + key (str): Key of the tokenized DINOv2 global token modality to decode. + tokenizers (dict): Dictionary of tokenizers. + image_size (int): Size of the image. + patch_size (int): Size of the patches. + """ + toks = rearrange(mod_dict[key]['tensor'].long(), 'b n -> b n 1 1') + rec = tokenizers[get_transform_key(key)].decode_tokens(toks) + return rec.squeeze() + +def decode_tok_imagebind_global(mod_dict, tokenizers, key='tok_imagebind_global'): + """ + Decodes a sequence of ImageBind global tokens from a model dictionary. + + Args: + mod_dict (dict): Model output dictionary. + key (str): Key of the tokenized ImageBind global token modality to decode. + tokenizers (dict): Dictionary of tokenizers. + image_size (int): Size of the image. + patch_size (int): Size of the patches. + """ + toks = rearrange(mod_dict[key]['tensor'].long(), 'b n -> b n 1 1') + rec = tokenizers[get_transform_key(key)].decode_tokens(toks) + return rec.squeeze() + +def decode_color_palette(mod_dict, text_tokenizer, key='color_palette'): + """ + Decodes a sequence of color palettes from a model dictionary. + + Args: + mod_dict (dict): Model output dictionary. + key (str): Key of the tokenized ImageBind modality to decode. + tokenizers (dict): Dictionary of tokenizers. + image_size (int): Size of the image. + patch_size (int): Size of the patches. + """ + decoded = decode_text(mod_dict, key, text_tokenizer)[2] + all_decoded = decoded if isinstance(decoded, list) else [decoded] + all_decoded = [d.replace(' [EOS]', '') for d in all_decoded] + all_decoded = [visualize_palettes_multi(d) for d in all_decoded] + all_decoded = all_decoded[0] if len(all_decoded) == 1 else all_decoded + return all_decoded + +def decode_human_poses(mod_dict, tokenizers, text_tokenizer, key='human_poses'): + """ + Decodes human poses tokenized with text + BMLP + """ + decoded = decode_text(mod_dict, key, text_tokenizer)[2] + all_decoded = decoded if isinstance(decoded, list) else [decoded] + all_decoded = [d.replace(' [EOS]', '') for d in all_decoded] + + imgs = [] + for decoded in all_decoded: + img = np.ones((224,224,4)) + if decoded != 'none': + try: + img = visualize_human_poses(decoded, tokenizers[key], mod_dict) + except Exception as e: + print('Error in decoding human poses. Packages required for plotting may not be installed. Trace:') + print(e) + imgs.append(img) + + imgs = np_squeeze(np.stack(imgs), axis=0) + + return imgs + +metadata_transform = MetadataTransform(shuffle=False, random_trunc=False, return_chunks=False) + +def _split_metadata_string(input_string): + result = [] + current_subseq = [] + + for part in input_string.split(): + # If we encounter a "v1" and there's already a subsequence being built, + # we add it to the result and start a new one + if 'v1' in part and current_subseq: + result.append(current_subseq) + current_subseq = [] + + current_subseq.append(part) + + # Append any remaining subsequence to the result + if current_subseq: + result.append(current_subseq) + + return result + +def decode_metadata(mod_dict, text_tokenizer, key='metadata'): + """ + Decodes a sequence of metadata tokens into a dictionary of metadata. + + Args: + mod_dict (dict): Model output dictionary. + key (str): Key of the metadata modality to decode. + text_tokenizer (tokenizers.Tokenizer): Text tokenizer. + """ + decoded = decode_text(mod_dict, key, text_tokenizer)[2] + all_decoded = decoded if isinstance(decoded, list) else [decoded] + all_decoded = [d.replace(' [EOS]', '').replace(' [PAD]', '') for d in all_decoded] + + all_metadata = [] + + for decoded in all_decoded: + + parts = _split_metadata_string(decoded) + + invalid_parts = [] + metadata_dict = {} + + for part in parts: + + # Check if part has been parsed correctly + if len(part) != 2: + invalid_parts.append(str(part)) + continue + metadata_id, metadata_value = part + if (not metadata_id.startswith('v1=') or + not metadata_value.startswith('v0=') or + metadata_id not in metadata_transform.id_metadata_map): + invalid_parts.append(str(part)) + + # Parse metadata type and value + metadata_type = metadata_transform.id_metadata_map[metadata_id] + + metadata_value = int(metadata_value.split('=')[1]) + + if metadata_type in metadata_transform.image_dim_modalities: + metadata_value *= metadata_transform.image_dim_bin_size + elif metadata_type in metadata_transform.metadata_min_max_bins: + vmin, vmax, bins = metadata_transform.metadata_min_max_bins[metadata_type] + metadata_value = (vmax - vmin) * (metadata_value / bins) + vmin + + metadata_dict[metadata_type] = metadata_value + + metadata_dict = {k: metadata_dict[k] for k in metadata_transform.metadata_id_map if k in metadata_dict} + all_metadata.append(metadata_dict) + + all_metadata = all_metadata[0] if len(all_metadata) == 1 else all_metadata + + return all_metadata + +def decode_text(mod_dict, key, text_tokenizer): + """ + Decodes a text sequence from a model dictionary. + + Args: + mod_dict (dict): Model output dictionary. + key (str): Key of the text modality to decode. + text_tokenizer (tokenizers.Tokenizer): Text tokenizer. + """ + input_texts, target_texts, merged_texts = [], [], [] + + sentinel_ids = set(get_sentinel_to_id_mapping(text_tokenizer).values()) + B = mod_dict[key]['tensor'].shape[0] + + for i in range(B): + + input_seq = mod_dict[key]['tensor'][i] + input_seq = input_seq[mod_dict[key]['input_mask'][i] == 0] + input_seq = input_seq.tolist() + + target_seq = mod_dict[key]['tensor'][i] + target_seq = target_seq[mod_dict[key]['target_mask'][i] == 0] + target_seq = target_seq.tolist() + + merged_seq = merge_span_masking(input_seq, target_seq, sentinel_ids=sentinel_ids) + + input_text = text_tokenizer.decode(input_seq, skip_special_tokens=False) + target_text = text_tokenizer.decode(target_seq, skip_special_tokens=False) + merged_text = text_tokenizer.decode(merged_seq, skip_special_tokens=False) + + input_texts.append(input_text) + target_texts.append(target_text) + merged_texts.append(merged_text) + + if B == 1: + input_texts, target_texts, merged_texts = input_texts[0], target_texts[0], merged_texts[0] + + return input_texts, target_texts, merged_texts + +def decode_sam_instances(mod_dict, tokenizers, text_tokenizer, key='sam_instance', image_size=224, token_len=16): + ''' + Decodes a sequence of SAM instance tokens into the instance representation. + + Args: + mod_dict (dict): Model output dictionary. + key (str): Key of the tokenized ImageBind modality to decode. + tokenizers (dict): Dictionary of tokenizers. + text_tokenizer (tokenizers.Tokenizer): Text tokenizer. + image_size (int): Size of the image. + token_len (int): Tokenized SAM instance token length. + ''' + assert image_size == 224, 'SAM instance decoding only supports 224x224 images' + decoded = decode_text(mod_dict, key, text_tokenizer)[2] + all_decoded = decoded if isinstance(decoded, list) else [decoded] + all_decoded = [d.replace(' [EOS]', '') for d in all_decoded] + + # Generate deterministic SAM color palette + rng = np.random.default_rng(seed=0) + sam_palette = [rng.integers(0, 255, size=3) for i in range(1000)] + + def group_by_identifier(input_list, identifier): + ''' + Groups the input_list [a,b,c,a,d,d,c,..] using the identifier a, in the following format: + [[b,c], [d,d,c], ...] + ''' + return [list(group) for key, group in groupby(input_list, lambda x: x == identifier) if not key] + + def map_locations(inp, tokens=False): + ''' + Converts v0, v1, v2, v3 textual representation into int. + When tokens=True, inp is mapped to its corresponding token id. + ''' + if '=' not in inp: + return None + axis, position = inp.split("=") + try: + position = int(position) + except: + return None + if tokens: + if axis == 'v0': + return position + else: + return position + 512 + return position + + def iou(box1, box2): + ''' + Calculates iou of the input bounding boxes + ''' + # Calculate the coordinates of the intersection rectangle + x1 = max(box1[0], box2[0]) + y1 = max(box1[1], box2[1]) + x2 = min(box1[2], box2[2]) + y2 = min(box1[3], box2[3]) + + # Calculate the area of the intersection + intersection_area = max(0, x2 - x1) * max(0, y2 - y1) + + # Calculate the areas of the individual bounding boxes + area_box1 = (box1[2] - box1[0]) * (box1[3] - box1[1]) + area_box2 = (box2[2] - box2[0]) * (box2[3] - box2[1]) + + # Calculate the union area + union_area = area_box1 + area_box2 - intersection_area + + # Calculate and return the IoU + return intersection_area / union_area + + all_sam_instances = [] + + for decoded in all_decoded: + + tokens_per_sample = [] + bboxes_per_sample = [] + areas_per_sample = [] + parts = decoded.split() + + for part in group_by_identifier(parts, identifier='point'): + instances = part[2:] + # Ignore 'none' cases + if len(instances) <= 1: + continue + + for positions in group_by_identifier(part, identifier='polygon'): + # Ignore incomplete polygons + if len(positions) != token_len + 4: + continue + + bbox, tokens = positions[:4], positions[4:] + min_w, min_h, max_w, max_h = map(map_locations, bbox) + # Ignore the cases where the bounding box is prediction is in incorrect format + if None in [min_w, max_w, min_h, max_h] or (min_w >= max_w or min_h >= max_h): + continue + + bbox = np.array([min_h, min_w, max_h, max_w]) + tokens = list(map(lambda x: map_locations(x, tokens=True), tokens)) + if None in tokens: + continue + tokens = np.array(tokens) + + tokens_per_sample.append(tokens) + bboxes_per_sample.append(bbox) + areas_per_sample.append((max_w - min_w) * (max_h - min_h)) + + final_instances = np.zeros((image_size, image_size, 3), dtype=np.uint8) + if len(areas_per_sample) == 0: + return final_instances + + # Sort the instance masks by area + areas_per_sample = np.array(areas_per_sample) + sorted_idx = np.argsort(-areas_per_sample) + tokens_per_sample = np.stack(tokens_per_sample)[sorted_idx] + bboxes_per_sample = np.stack(bboxes_per_sample)[sorted_idx] + # Decoded tokens + tokens_per_sample = torch.LongTensor(tokens_per_sample).reshape(-1, 4, 4).to(device) + decoded_tokens = tokenizers[key].decode_tokens(tokens_per_sample) + instances = torch.sigmoid(decoded_tokens).squeeze(1).cpu().detach().numpy() + + # Filter and group instances + representive_masks = [] + representive_bboxes = [] + for (mask, bbox) in zip(instances, bboxes_per_sample): + # Filter out unusual masks + if (mask.max() - mask.min()) < 0.9: + continue + + # Groups the duplicated instance masks + duplicated_flag = False + for rms, rbs in zip(representive_masks, representive_bboxes): + rm, rb = rms[0], rbs[0] + sim_score = 2 * ((rm * mask).sum() + 0.01) / (rm.sum() + mask.sum() + 0.01) + box_iou = iou(rb, bbox) + # If the similarity and IoU are high, consider them as the same instance and group them + if sim_score > 0.8 and box_iou > 0.9: + # Add the mask to its corresponding group + rms.append(mask) + rbs.append(bbox) + duplicated_flag = True + break + if not duplicated_flag: + representive_masks.append([mask]) + representive_bboxes.append([bbox]) + + # Plot the instances + for i, (rms, rbs) in enumerate(zip(representive_masks, representive_bboxes)): + mask = np.mean(rms, axis=0) + bbox = np.mean(rbs, axis=0).astype(np.int32) + min_h, min_w, max_h, max_w = bbox.tolist() + mask = cv2.resize(mask, (max_w - min_w, max_h - min_h), interpolation=cv2.INTER_CUBIC) + max_w, max_h = min(max_w, final_instances.shape[1]), min(max_h, final_instances.shape[0]) + mask = mask[:max_h - min_h,:max_w - min_w] > 0.5 + final_instances[min_h:max_h, min_w:max_w, :][mask] = sam_palette[i] + + all_sam_instances.append(final_instances) + + all_sam_instances = all_sam_instances[0] if len(all_sam_instances) == 1 else np.stack(all_sam_instances) + + return all_sam_instances + +def decode_dict(mod_dict, tokenizers, text_tokenizer, image_size=224, patch_size=16, + decoding_steps=25, activate_controlnet=False, controlnet_guidance_scale=2.5, controlnet_cond_scale=0.8, + to_rgb=True, seed=None): + """ + Decodes the model output dictionary into a dictionary of images and text. + + Args: + mod_dict (dict): Model output dictionary. + tokenizers (dict): Dictionary of tokenizers. + text_tokenizer (tokenizers.Tokenizer): Text tokenizer. + image_size (int): Image size. + patch_size (int): Patch size. + decoding_steps (int): Number of diffusion decoding steps (if applicable). + activate_controlnet (bool): Whether to activate the RGB ControlNet and override the RGB detokenizer. + controlnet_guidance_scale (float): Classifier-free guidance scale for the ControlNet. + controlnet_cond_scale (float): ControlNet conditioning scale. + """ + dec_dict = {} + + for key in mod_dict: + k, res = get_transform_key(key), get_transform_resolution(key, image_size, to_tuple=False) + + if k == 'rgb': + decoded = decode_input_rgb(mod_dict, key=key) + elif k == 'tok_rgb': + if not activate_controlnet or 'controlnet' not in tokenizers: + decoded = decode_tok_rgb( + mod_dict, tokenizers, key=key, + image_size=res, patch_size=patch_size, + t=decoding_steps, verbose=False + ) + else: + decoded = decode_tok_rgb_controlnet( + mod_dict, tokenizers, key=key, + image_size=res, patch_size=patch_size, + t=decoding_steps, guidance_scale=controlnet_guidance_scale, + cond_scale=controlnet_cond_scale, verbose=False + ) + elif k == 'tok_canny_edge': + decoded = decode_tok_canny_edge( + mod_dict, tokenizers, key=key, + image_size=res, patch_size=patch_size, + t=decoding_steps, verbose=False + ) + elif k == 'tok_sam_edge': + decoded = decode_tok_sam_edge( + mod_dict, tokenizers, key=key, + image_size=res, patch_size=patch_size, + t=decoding_steps, verbose=False + ) + elif k == 'tok_normal': + decoded = decode_tok_normal( + mod_dict, tokenizers, key=key, + image_size=res, patch_size=patch_size, + t=decoding_steps, verbose=False + ) + elif k == 'tok_depth': + decoded = decode_tok_depth( + mod_dict, tokenizers, key=key, + image_size=res, patch_size=patch_size, + t=decoding_steps, verbose=False, cmap='turbo' if to_rgb else None + ) + elif k == 'tok_semseg': + decoded = decode_tok_semseg( + np.ones((res, res, 3)), mod_dict, tokenizers, key=key, + image_size=res, patch_size=patch_size, return_logits=not to_rgb + ) + elif k == 'tok_clip': + decoded = decode_tok_clip( + mod_dict, tokenizers, key=key, + image_size=res, patch_size=patch_size + ) + elif k == 'tok_dinov2': + decoded = decode_tok_dinov2( + mod_dict, tokenizers, key=key, + image_size=res, patch_size=patch_size + ) + elif k == 'tok_dinov2_global': + decoded = decode_tok_dinov2_global( + mod_dict, tokenizers, key=key + ) + elif k == 'tok_imagebind': + decoded = decode_tok_imagebind( + mod_dict, tokenizers, key=key, + image_size=res, patch_size=patch_size + ) + elif k == 'tok_imagebind_global': + decoded = decode_tok_imagebind_global( + mod_dict, tokenizers, key=key + ) + elif k == 'color_palette': + decoded = decode_color_palette( + mod_dict, text_tokenizer, key=key + ) + elif k == 'human_poses': + decoded = decode_human_poses( + mod_dict, tokenizers, text_tokenizer, key=key + ) + elif k in ['caption', 'det']: + decoded = decode_text(mod_dict, key, text_tokenizer)[2] + decoded = decoded if isinstance(decoded, list) else [decoded] + decoded = [d.replace(' [EOS]', '') for d in decoded] + elif k in ['metadata']: + decoded = decode_metadata( + mod_dict, text_tokenizer, key=key + ) + elif k == 'sam_instance': + decoded = decode_sam_instances( + mod_dict, tokenizers, text_tokenizer, + key=key, image_size=224, + ) + elif k in ['t5_caption']: + if 'ascii_tensor' in mod_dict[key]: + decoded = [] + for ascii_tensor in mod_dict[key]['ascii_tensor']: + ascii_values = ascii_tensor.flatten().tolist() + decoded_text = ''.join(chr(val) for val in ascii_values if val != 0) + decoded.append(f"T5-XXL embedding of: {decoded_text}") + decoded = decoded[0] if len(decoded) == 1 else decoded + else: + decoded = "T5-XXL embedding" + dec_dict[key] = decoded + return dec_dict + + + + +# Plotting utils + +MOD_PRINT_NAMES = { + 'rgb': 'RGB', + 'tok_rgb': 'RGB (tok)', + 'tok_normal': 'Normal (tok)', + 'tok_depth': 'Depth (tok)', + 'tok_semseg': 'Semseg (tok)', + 'tok_clip': 'CLIP (tok)', + 'tok_canny': 'Canny (tok)', + 'tok_sam': 'SAM (tok)', + 'sam_instance': 'SAM Instances (tok)', + + 'rgb@224': 'RGB@224', + 'tok_rgb@224': 'RGB@224 (tok)', + 'tok_normal@224': 'Normal@224 (tok)', + 'tok_depth@224': 'Depth@224 (tok)', + 'tok_semseg@224': 'Semseg@224 (tok)', + 'tok_clip@224': 'CLIP@224 (tok)', + + 'rgb@448': 'RGB@448', + 'tok_rgb@448': 'RGB@448 (tok)', + 'tok_normal@448': 'Normal@448 (tok)', + 'tok_depth@448': 'Depth@448 (tok)', + 'tok_semseg@448': 'Semseg@448 (tok)', + 'tok_clip@448': 'CLIP@448 (tok)', + + 'caption': 'Caption', + 'det': 'Detection', + 't5_caption': 'T5 XXL', + 'metadata': 'Metadata', + 'human_poses': 'Human poses', + 'color_palette': 'Color palette', + 'tok_dinov2': 'DINOv2 (tok)', + 'tok_dinov2_global': 'DINOv2 global (tok)', + 'tok_imagebind': 'ImageBind (tok)', + 'tok_imagebind_global': 'ImageBind global (tok)', +} + +def remove_ticks_and_labels(ax): + """ + Remove the axis ticks and labels + + Args: + ax (matplotlib.axes.Axes): Axis to remove ticks and labels from + """ + ax.set_xticks([]) + ax.set_yticks([]) + ax.set_xticklabels([]) + ax.set_yticklabels([]) + +def remove_spines(ax): + """ + Removes the spines from the given axis. + + Args: + ax (matplotlib.axes.Axes): Axis to remove spines from + """ + ax.spines['top'].set_visible(False) + ax.spines['right'].set_visible(False) + ax.spines['bottom'].set_visible(False) + ax.spines['left'].set_visible(False) + +def convert_string_to_bboxes(bboxes_str, bins=1000): + """ + Converts a string of bboxes to a list of bboxes. + + Args: + bboxes_str (str): String of bboxes + bins (int): Number of bins (default: 1000) + """ + bboxes_str = bboxes_str.split(" ") + bboxes = [] + for token in bboxes_str: + if "=" in token: + coord = token.split("=")[1] + coord = float(coord) / (bins - 1) + + if token.startswith("v0="): + bboxes.append([coord,]) + else: + bboxes[-1].append(coord) + elif len(bboxes[-1]) == 4: + bboxes[-1].append(token) + else: + bboxes[-1][4] = " ".join([bboxes[-1][4], token]) + + bboxes = [bbox for bbox in bboxes if len(bbox) == 5] + + return bboxes + +def visualize_palettes_multi(palettes): + palettes = palettes.split() + palettes = palettes[1:] + + all_colors = [] + for ii in range(len(palettes)): + all_colors.append(int(palettes[ii][3:])) + w = h = 25 + # construct palette image + o = Image.new("RGB", size=(w * len(palettes)//3, h * len(palettes)//3)) + arr = np.asarray(o).copy() + for ii in range(len(palettes)//3): + arr[:, ii * h : (ii + 1) * h, :] = all_colors[ii*3:(ii+1)*3] + final_palette = arr / 255 + + return final_palette + +BOX_COLOR = (255, 0, 0) # Red +TEXT_COLOR = (255, 255, 255) # White + +try: + from fourm.utils.hmr2_utils.hmr2.models.smpl_wrapper import SMPL + from fourm.utils.hmr2_utils.hmr2.utils.renderer import Renderer, cam_crop_to_full + import pickle as pkl + LIGHT_BLUE=(0.65098039, 0.74117647, 0.85882353) + with open('./fourm/utils/hmr2_utils/model_cfg.pkl','rb') as f: + pose_model_cfg = pkl.load(f) + # Instantiate SMPL model + smpl_cfg = {k.lower(): v for k,v in dict(pose_model_cfg.SMPL).items()} + smpl_cfg['model_path'] = './fourm/utils/hmr2_utils/data/smpl' + smpl_cfg['joint_regressor_extra'] = './fourm/utils/hmr2_utils/data/SMPL_to_J19.pkl' + smpl_cfg['mean_params'] = './fourm/utils/hmr2_utils/data/smpl_mean_params.npz' + smpl = SMPL(**smpl_cfg) + # Setup the renderer + renderer = Renderer(pose_model_cfg, faces=smpl.faces) +except Exception as e: + print(e) + print('Human pose dependencies are not installed, hence poses will not be visualized. To visualize them (optional), you can do the following: \n' \ + '1) Install via `pip install timm yacs smplx pyrender pyopengl==3.1.4` \n' \ + ' You may need to follow the pyrender install instructions: https://pyrender.readthedocs.io/en/latest/install/index.html \n' \ + '2) Download SMPL data from https://smpl.is.tue.mpg.de/. See https://github.com/shubham-goel/4D-Humans/ for an example. \n' \ + '3) Copy the required SMPL files (smpl_mean_params.npz, SMPL_to_J19.pkl, smpl/SMPL_NEUTRAL.pkl) to fourm/utils/hmr2_utils/data .') + +def visualize_human_poses(pose, poses_tokenizer, mod_dict): + full_gts = pose + full_gts = full_gts.split() + num_instances = len(full_gts) // 39 # total length of a pose instance seq is 39 + all_verts = [] + all_cam_t = [] + for inst in range(num_instances): + try: + full_gt = full_gts[inst*39:(inst+1)*39] + ##create the pose params dict + all_params = {} + all_params['bbox_xyxy'] = torch.Tensor((int(full_gt[1][3:])/999*224, int(full_gt[2][3:])/999*224, int(full_gt[3][3:])/999*224, int(full_gt[4][3:])/999*224)) + all_params["box_center"] = torch.cat(( ((all_params["bbox_xyxy"][0] + all_params["bbox_xyxy"][2]) / 2.).unsqueeze(0).unsqueeze(1) , ( (all_params["bbox_xyxy"][1] + all_params["bbox_xyxy"][3]) / 2.).unsqueeze(0).unsqueeze(1) ), dim = 1) + all_params["box_size"] = torch.max((all_params["box_center"][0,0] - all_params["bbox_xyxy"][0]) * 2 , (all_params["box_center"][0,1] - all_params["bbox_xyxy"][1]) * 2 ) + all_params["img_size"] = torch.Tensor([224., 224.]) + all_params["img_size"] = all_params["img_size"].unsqueeze(0) + all_params["focal_length"] = torch.Tensor([5000., 5000.]) + + for ii in range(len(full_gt)): + if full_gt[ii] == 'camera': + all_params['pred_cam'] = torch.Tensor([ (int(full_gt[ii+1][3:])-49.95)/49.95, (int(full_gt[ii+2][3:])-49.95)/49.95, (int(full_gt[ii+3][3:])-49.95)/49.95 ]) + break + all_params['pred_cam'] = all_params['pred_cam'].unsqueeze(0) + all_params['pred_smpl_params'] = {} + + for ii in range(len(full_gt)): + if full_gt[ii] == 'shape': + all_params['pred_smpl_params']['betas'] = torch.Tensor([ (int(full_gt[ii+1][3:])-499.5)/166.5, (int(full_gt[ii+2][3:])-499.5)/166.5, (int(full_gt[ii+3][3:])-499.5)/166.5, (int(full_gt[ii+4][3:])-499.5)/166.5, (int(full_gt[ii+5][3:])-499.5)/166.5, (int(full_gt[ii+6][3:])-499.5)/166.5, (int(full_gt[ii+7][3:])-499.5)/166.5, (int(full_gt[ii+8][3:])-499.5)/166.5, (int(full_gt[ii+9][3:])-499.5)/166.5, (int(full_gt[ii+10][3:])-499.5)/166.5 ]) + break + all_params['pred_smpl_params']['betas'] = all_params['pred_smpl_params']['betas'].unsqueeze(0) + + for ii in range(len(full_gt)): + if full_gt[ii] == 'global': + all_params['pred_smpl_params']['global_orient'] = torch.Tensor( [ [(int(full_gt[ii+1][3:])-499.5)/499.5, (int(full_gt[ii+2][3:])-499.5)/499.5, (int(full_gt[ii+3][3:])-499.5)/499.5 ] , [ (int(full_gt[ii+4][3:])-499.5)/499.5, (int(full_gt[ii+5][3:])-499.5)/499.5, (int(full_gt[ii+6][3:])-499.5)/499.5], [(int(full_gt[ii+7][3:])-499.5)/499.5, (int(full_gt[ii+8][3:])-499.5)/499.5, (int(full_gt[ii+9][3:])-499.5)/499.5 ] ] ) + break + all_params['pred_smpl_params']['global_orient'] = all_params['pred_smpl_params']['global_orient'].unsqueeze(0).unsqueeze(0) + + body_poses = torch.FloatTensor() + for ii in range(len(full_gt)): + if full_gt[ii] == 'pose': + pose_start = ii + break + + for ii in range(8): + pose_curr = ii + pose_start + 1 + if 'v1' in full_gt[pose_curr]: + poses_curr = torch.Tensor([int(full_gt[pose_curr][3:])+512]) + else: + poses_curr = torch.Tensor([int(full_gt[pose_curr][3:])]) + poses_curr = poses_curr + body_poses = torch.cat((body_poses,poses_curr), dim=0) + body_poses = body_poses.long() + + body_poses = body_poses.unsqueeze(0).unsqueeze(2).unsqueeze(2).to(device) + body_poses = poses_tokenizer.decode_tokens(body_poses).squeeze(2).squeeze().reshape(1,23,3,3).cpu() + + all_params['pred_smpl_params']['body_pose'] = body_poses + + smpl_params = (all_params['pred_smpl_params']) + smpl_output = smpl(**{k: v.float().cpu() for k,v in smpl_params.items()}, pose2rot=False) + + + for n in range(smpl_output.vertices.size(0)): + # Add all verts and cams to list + verts = smpl_output.vertices[n].detach().cpu().numpy() + + img_size = all_params["img_size"].float() + pred_cam = all_params['pred_cam'] + box_center = all_params["box_center"].float() + box_size = all_params["box_size"].float() + + scaled_focal_length = pose_model_cfg.EXTRA.FOCAL_LENGTH / pose_model_cfg.MODEL.IMAGE_SIZE * img_size.max() + pred_cam_t_full = cam_crop_to_full(pred_cam, box_center, box_size, img_size, scaled_focal_length).detach().cpu().numpy() + + + cam_t = pred_cam_t_full[n] + all_verts.append(verts) + all_cam_t.append(cam_t) + + except Exception as e: + print('Error in decoding human poses: ', end='') + print(e) + continue + + try: + input_img = denormalize(mod_dict['rgb@224']['tensor'].squeeze(), mean=(IMAGENET_DEFAULT_MEAN), std=IMAGENET_DEFAULT_STD).permute(1,2,0).cpu() + + except Exception as e: + print(e) + input_img = 1. + + if 'tok_rgb' in mod_dict: + input_img = decode_tok_rgb(mod_dict, toks, key='tok_rgb') + + # Render front view + input_img_overlay = 0.5* input_img[:,:,:3] + if len(all_verts) > 0: + misc_args = dict( + mesh_base_color=LIGHT_BLUE, + scene_bg_color=(1, 1, 1), + focal_length=scaled_focal_length, + ) + cam_view = renderer.render_rgba_multiple(all_verts, cam_t=all_cam_t, render_res=img_size[n], **misc_args) + mask = (cam_view[:,:,0]<1.).astype(int)[:,:,None] + input_img_overlay = 0.5* input_img[:,:,:3] * (1-mask) + cam_view[:,:,:3] * mask + + return input_img_overlay + +def visualize_bboxes(img, bboxes_str, color=BOX_COLOR, thickness=2): + """ + Visualizes bounding boxes on the image. + + Args: + img (np.array): Image to draw bounding boxes on. + bboxes_str (str): String containing bounding boxes in the format: + v0=1 v1=2 v2=3 v3=4 class_name ..., where + v0 is xmin, v1 is ymin, v2 is xmax, v3 is ymax + color (tuple): Color of the bounding box. + thickness (int): Thickness of the bounding box. + """ + if img is None: + img = 255 * np.ones((256,256,3), dtype=np.int32) + img = img.copy() + + bboxes_str = bboxes_str.replace('[PAD]', '') + + if len(bboxes_str.replace('[EOS]', '')) == 0: + return img + + try: + bboxes = convert_string_to_bboxes(bboxes_str.replace(' [EOS]', '')) + except: + return img + + for bbox in bboxes: + x_min, y_min, x_max, y_max, class_name = bbox + img_h, img_w = img.shape[0], img.shape[1] + x_min, x_max, y_min, y_max = int(x_min * img_w), int(x_max * img_w), int(y_min * img_h), int(y_max * img_h) + + cv2.rectangle(img, (x_min, y_min), (x_max, y_max), color=color, thickness=thickness) + + ((text_width, text_height), _) = cv2.getTextSize(class_name.rstrip(), cv2.FONT_HERSHEY_SIMPLEX, 0.35, 1) + cv2.rectangle(img, (x_min, y_min - int(1.3 * text_height)), (x_min + text_width, y_min), BOX_COLOR, -1) + cv2.putText( + img, + text=f"{class_name}", + org=(x_min, y_min - int(0.3 * text_height)), + fontFace=cv2.FONT_HERSHEY_SIMPLEX, + fontScale=0.35, + color=TEXT_COLOR, + lineType=cv2.LINE_AA, + ) + return img + + +def plot_text_in_square(ax, text, padding=0.5, fontsize=14, wrap_width=50): + """ + Plots text in a square. + + Args: + ax (matplotlib.axes.Axes): Matplotlib axis to plot on + text (str): Text to plot + padding (float): Padding around the text + fontsize (int): Font size of the text + wrap_width (int): Width of the text to wrap + """ + ax.set_xlim(0, 1) + ax.set_ylim(0, 1) + + if isinstance(text, list): + text = text[0] + + text = text.replace('[PAD]', '') + + # Wrap the text if necessary + wrapped_text = textwrap.fill(text, int(wrap_width)) + + # Add the padding + bbox_props = dict(boxstyle="square,pad=" + str(padding), facecolor="white", edgecolor="black") + + # Add the text to the plot + ax.text(0.5, 0.5, wrapped_text, ha='center', va='center', fontsize=fontsize, bbox=bbox_props) + + remove_ticks_and_labels(ax) + remove_spines(ax) + + +def text_to_pil_image(text, padding=0.5, fontsize=14, wrap_width=40, image_size=(512, 512)): + """ + Converts text to a PIL image. + + Args: + text (str): Text to convert to image + padding (float): Padding around the text + fontsize (int): Font size of the text + wrap_width (int): Width of the text to wrap + image_size (tuple): Size of the output image (width, height) + + Returns: + PIL.Image.Image: Generated image with the text + """ + fig, ax = plt.subplots(figsize=(image_size[0] / 100, image_size[1] / 100), dpi=100) + ax.set_xlim(0, 1) + ax.set_ylim(0, 1) + + if isinstance(text, list): + text = text[0] + + text = text.replace('[PAD]', '') + + # Wrap the text if necessary + wrapped_text = textwrap.fill(text, wrap_width) + + # Add the padding + bbox_props = dict(boxstyle="square,pad=" + str(padding), facecolor="white", edgecolor="black") + + # Add the text to the plot + ax.text(0.5, 0.5, wrapped_text, ha='center', va='center', fontsize=fontsize, bbox=bbox_props) + + # Remove ticks, labels, and spines + ax.set_xticks([]) + ax.set_yticks([]) + for spine in ax.spines.values(): + spine.set_visible(False) + + # Convert the plot to a PIL image + fig.canvas.draw() + image = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8) + image = image.reshape(fig.canvas.get_width_height()[::-1] + (3,)) + + plt.close(fig) + + return Image.fromarray(image) + + +def plot_modality(dec_dict, key, ax, figscale=4.0): + """ + Plots a single modality. Function name has a typo because of legacy reasons. + + Args: + dec_dict (dict): Dictionary of decoded modalities + key (str): Key of the modality to plot + ax (matplotlib.axes.Axes): Matplotlib axis to plot on + figscale (float): Scaling factor for the figure (used to scale the caption box) + """ + modality = dec_dict[key] + k = get_transform_key(key) + + if 'tok' in k or k == 'rgb' or k == 'human_poses' or k == 'color_palette': + ax.imshow(modality.clip(0,1)) + elif k == 'caption': + plot_text_in_square(ax, modality, wrap_width=max(1,int(7*figscale))) # 7*figscale turns out to make caption box fit nicely + elif k == 't5_caption': + plot_text_in_square(ax, modality, wrap_width=max(1,int(7*figscale))) # 7*figscale turns out to make caption box fit nicely + elif k == 'metadata': + modality = ',\n'.join([f'{k}: {v:.2f}' if isinstance(v, float) else f'{k}: {v}' for k, v in modality.items()]) + plot_text_in_square(ax, modality, wrap_width=max(1,int(7*figscale)), fontsize=11) + elif k == 'det': + bbox_img = visualize_bboxes(np.ones((224,224,3)), modality, thickness=2) + ax.imshow(bbox_img.clip(0,1)) + +def plot_conds_and_targets(cond_domains, target_domains, dec_dicts, save_path=None, fs_titles=15, figscale=4.0, dpi=100): + """ + Plots the conditioning and target modalities for a batch of samples. + + Args: + cond_domains (list of str): List of conditioning domains + target_domains (list of str): List of target domains + dec_dicts (list of dicts): List of dictionaries containing the decoded conditioning and target modalities + save_path (str): Path to save the figure. If None, the figure is not saved but plotted instead. + fs_titles (int): Font size of the titles + figscale (float): Scaling factor for the figure size (minimum 4.0 for good results) + dpi (float): Dots per inch for the saved figure + """ + + n_cond = len(cond_domains) + n_target = len(target_domains) + n_samples = len(dec_dicts) + ncols = n_samples + 1 if n_cond > 0 else n_samples + nrows = max(n_cond, n_target) + + fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(ncols*figscale, nrows*figscale), facecolor='white') + + if nrows == 1 and ncols == 1: + ax = np.array([[ax]]) + elif nrows == 1: + ax = np.expand_dims(ax, axis=0) + elif ncols == 1: + ax = np.expand_dims(ax, axis=1) + + for cond_idx, cond_domain in enumerate(cond_domains): + axi = ax[cond_idx, 0] + plot_modality(dec_dicts[0], key=cond_domain, ax=axi) + axi.set_title(f'Conditioning: {MOD_PRINT_NAMES[cond_domain]}', fontsize=fs_titles) + + # Remove spines that are not needed + if n_cond > 0: + for i in range(n_cond, nrows, 1): + remove_spines(ax[i, 0]) + + offset = 0 if n_cond == 0 else 1 + + for sample_idx, dec_dict in enumerate(dec_dicts): + for target_idx, target_domain in enumerate(target_domains): + axi = ax[target_idx, sample_idx+offset] + plot_modality(dec_dict, key=target_domain, ax=axi) + axi.set_title(f'{sample_idx+1}.{target_idx+1}: {MOD_PRINT_NAMES[target_domain]}', fontsize=fs_titles) + + # Remove spines that are not needed + for i in range(n_target, nrows, 1): + remove_spines(ax[i, sample_idx+offset]) + + for ax in fig.axes: + remove_ticks_and_labels(ax) + + plt.tight_layout() + if save_path is not None: + os.makedirs(os.path.dirname(save_path), exist_ok=True) + plt.savefig(save_path, bbox_inches='tight', dpi=dpi) #, pil_kwargs={'quality': 30}) + plt.close() + else: + plt.show() + + +def save_conds_and_targets(cond_domains, target_domains, dec_dicts, save_dir, sample_idx, suffix=None, vis_det=False): + """ + Saves the conditioning and target modalities for a batch of samples. + + Args: + cond_domains (list of str): List of conditioning domains + target_domains (list of str): List of target domains + dec_dicts (list of dicts): List of dictionaries containing the decoded conditioning and target modalities + save_dir (str): Path to save the modalities + sample_idx (int): Unique index of the dataset sample + suffix (str): Suffix to append to the saved file names + vis_det (bool): Whether to visualize detection + """ + for variant_idx, dec_dict in enumerate(dec_dicts): + + for domain in cond_domains + target_domains: + if variant_idx != 0 and domain in cond_domains: + continue + + variant_suffix = f'_{variant_idx}' if domain in target_domains else '' + if suffix is not None: + variant_suffix += f'_{suffix}' + + domain_save_dir = os.path.join(save_dir, 'conds' if domain in cond_domains else 'targets', domain) + os.makedirs(domain_save_dir, exist_ok=True) + + if 'tok' in domain or domain in ['rgb', 'human_poses', 'color_palette']: + img = Image.fromarray((255 * dec_dict[domain]).astype(np.uint8)) + if domain in ['tok_clip', 'tok_dinov2', 'tok_imagebind']: + img = img.resize((224,224), resample=Image.NEAREST) + save_path = os.path.join(domain_save_dir, f'{sample_idx:06d}{variant_suffix}.png') + img.save(save_path) + + elif domain in ['caption', 'det', 'metadata']: + if vis_det: + save_path = os.path.join(domain_save_dir, f'{sample_idx:06d}{variant_suffix}.png') + bbox_img = visualize_bboxes(np.ones((512,512,3)), dec_dict[domain], thickness=2) + bbox_img = Image.fromarray((255 * bbox_img.clip(0,1)).astype(np.uint8)) + bbox_img.save(save_path) + else: + # Save caption as text file + save_path = os.path.join(domain_save_dir, f'{sample_idx:06d}{variant_suffix}.txt') + with open(save_path, 'w') as f: + f.write(dec_dict[domain]) + + +def plot_images_with_captions(images, captions, save_path=None, dpi=100, wrap_length=40, figscale=4.0): + """ + Plots images with their corresponding captions. + + Parameters: + - images (torch.Tensor): A tensor of shape Bx3xHxW with images. + - captions (list): A list of B captions. + """ + assert len(images) == len(captions), "Number of images must match number of captions!" + + B = len(images) + sqrt_B = int(B**0.5) + + # Determine the number of rows and columns for subplots + nrows = sqrt_B + ncols = (B + nrows - 1) // nrows + + fig, axarr = plt.subplots(nrows=nrows, ncols=ncols, figsize=(figscale*ncols, figscale*nrows)) + + axarr = np.array([axarr]) if nrows == 1 and ncols == 1 else axarr.ravel() + + for i, ax in enumerate(axarr): + if i < B: + # Convert tensor image to numpy + image_np = images[i].permute(1, 2, 0).cpu().float().numpy() + ax.imshow(image_np) + + # Place caption below the image + caption_wrapped = textwrap.fill(captions[i], width=wrap_length) + ax.text(0.5, -0.1, caption_wrapped, ha='center', va='top', transform=ax.transAxes, wrap=True) + + ax.axis("off") + else: + ax.axis("off") # Hide any additional subplots + + plt.subplots_adjust(hspace=0.6) + plt.tight_layout() + if save_path is not None: + os.makedirs(os.path.dirname(save_path), exist_ok=True) + plt.savefig(save_path, bbox_inches='tight', dpi=dpi) + plt.close() + else: + plt.show() \ No newline at end of file diff --git a/fourm/utils/pos_embed.py b/fourm/utils/pos_embed.py new file mode 100644 index 0000000000000000000000000000000000000000..a5a4981c4d0ada733fc597d36b745de7f4598767 --- /dev/null +++ b/fourm/utils/pos_embed.py @@ -0,0 +1,86 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -------------------------------------------------------- +# Based on UniLM / BEiT code base +# https://github.com/microsoft/unilm/tree/master/beit +# -------------------------------------------------------- +import re + +import torch + + +def interpolate_pos_embed_vit(model, checkpoint_model): + if 'pos_embed' in checkpoint_model: + pos_embed_checkpoint = checkpoint_model['pos_embed'] + embedding_size = pos_embed_checkpoint.shape[-1] + num_patches = model.patch_embed.num_patches + num_extra_tokens = model.pos_embed.shape[-2] - num_patches + # height (== width) for the checkpoint position embedding + orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5) + # height (== width) for the new position embedding + new_size = int(num_patches ** 0.5) + # class_token and dist_token are kept unchanged + if orig_size != new_size: + print("Position interpolate from %dx%d to %dx%d" % (orig_size, orig_size, new_size, new_size)) + extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens] + # only the position tokens are interpolated + pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:] + pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2) + pos_tokens = torch.nn.functional.interpolate( + pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False) + pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2) + new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1) + checkpoint_model['pos_embed'] = new_pos_embed + + +def interpolate_pos_embed_multimae(model, checkpoint_model): + pattern = "input_adapters\.(.*)\.pos_emb" + matched_keys = [k for k in checkpoint_model if bool(re.match(pattern, k))] + + for key in matched_keys: + domain = re.match(pattern, key).group(1) # group(0) is entire matched regex + if getattr(model.input_adapters, domain, None) is not None: + pos_embed_checkpoint = checkpoint_model[key] + _, _, orig_H, orig_W = pos_embed_checkpoint.shape + _, _, new_H, new_W = getattr(model.input_adapters, domain).pos_emb.shape + if (orig_H != new_H) or (orig_W != new_W): + print(f"Key {key}: Position interpolate from {orig_H}x{orig_W} to {new_H}x{new_W}") + pos_embed_checkpoint = torch.nn.functional.interpolate( + pos_embed_checkpoint, size=(new_H, new_W), mode='bicubic', align_corners=False) + checkpoint_model[key] = pos_embed_checkpoint + + + +def interpolate_rgb_pos_emb_fm(model, checkpoint_model): + if 'encoder_embeddings.rgb.pos_emb' in checkpoint_model: + pos_embed_checkpoint = checkpoint_model['encoder_embeddings.rgb.pos_emb'] + embedding_size = pos_embed_checkpoint.shape[-1] + num_patches = model.encoder_embeddings.rgb.num_patches + num_extra_tokens = 0 + # height (== width) for the checkpoint position embedding + orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5) + # height (== width) for the new position embedding + new_size = int(num_patches ** 0.5) + # class_token and dist_token are kept unchanged + if orig_size != new_size: + print("Position interpolate from %dx%d to %dx%d" % (orig_size, orig_size, new_size, new_size)) + extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens] + # only the position tokens are interpolated + pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:] + pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2) + pos_tokens = torch.nn.functional.interpolate( + pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False) + pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2) + new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1) + checkpoint_model['encoder_embeddings.rgb.pos_emb'] = new_pos_embed diff --git a/fourm/utils/run_name.py b/fourm/utils/run_name.py new file mode 100644 index 0000000000000000000000000000000000000000..270a3905f0f4c23e79497d0c8e1fef417035eeef --- /dev/null +++ b/fourm/utils/run_name.py @@ -0,0 +1,27 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +def setup_run_name(args): + if args.run_name == 'auto': + # This returns the config name after removing the first two parent dirs and extension + args.run_name = args.config_path.partition('cfgs/')[2].partition('/')[2].replace(".yaml", "") + + if "wandb_run_name" in args and args.wandb_run_name == 'auto': + # Wandb omits the current parent dir (pretrain, finetune, etc...) as it is part of the wandb project + args.wandb_run_name = args.run_name.partition('/')[2] + + if "output_dir" in args and 'auto' in args.output_dir: + args.output_dir = args.output_dir.replace('auto', args.run_name) + + if "s3_save_dir" in args and 'auto' in args.s3_save_dir: + args.s3_save_dir = args.s3_save_dir.replace('auto', args.run_name) \ No newline at end of file diff --git a/fourm/utils/s3_utils.py b/fourm/utils/s3_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e338119f98980264c68b362c8057e21a0a6ef268 --- /dev/null +++ b/fourm/utils/s3_utils.py @@ -0,0 +1,112 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import io +import os +import re +import boto3 +import webdataset as wds + +from boto3.s3.transfer import TransferConfig +from webdataset.handlers import reraise_exception + + +def setup_s3_args(args): + if not args.s3_data_endpoint: + args.s3_data_endpoint = args.s3_endpoint + + +def save_on_s3(filename, s3_path, s3_endpoint): + + s3_client = boto3.client( + service_name='s3', + aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], + aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'], + endpoint_url=s3_endpoint + ) + + _, bucket, key, _ = re.split("s3://(.*?)/(.*)$", s3_path) + + s3_client.upload_file(filename, bucket, key) + + +def download_from_s3(s3_path, s3_endpoint, filename, multipart_threshold_mb=512, multipart_chunksize_mb=512): + + MB = 1024 ** 2 + transfer_config = TransferConfig( + multipart_threshold=multipart_threshold_mb * MB, + multipart_chunksize=multipart_chunksize_mb * MB, + max_io_queue=1000) + + + s3_client = boto3.client( + service_name='s3', + aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], + aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'], + endpoint_url=s3_endpoint, + ) + + _, bucket, key, _ = re.split("s3://(.*?)/(.*)$", s3_path) + + s3_client.download_file(bucket, key, filename, Config=transfer_config) + + + +def override_wds_s3_tar_loading(s3_data_endpoint, s3_multipart_threshold_mb, s3_multipart_chunksize_mb, s3_max_io_queue): + + # When loading from S3 using boto3, hijack webdatasets tar loading + MB = 1024 ** 2 + transfer_config = TransferConfig( + multipart_threshold=s3_multipart_threshold_mb * MB, + multipart_chunksize=s3_multipart_chunksize_mb * MB, + max_io_queue=s3_max_io_queue) + + s3_client = boto3.client( + service_name='s3', + aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], + aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'], + endpoint_url=s3_data_endpoint, + ) + + def get_bytes_io(path): + byte_io = io.BytesIO() + _, bucket, key, _ = re.split("s3://(.*?)/(.*)$", path) + s3_client.download_fileobj(bucket, key, byte_io, Config=transfer_config) + byte_io.seek(0) + return byte_io + + def gopen_with_s3(url, mode="rb", bufsize=8192, **kw): + """gopen from webdataset, but with s3 support""" + if url.startswith("s3://"): + return get_bytes_io(url) + else: + return wds.gopen.gopen(url, mode, bufsize, **kw) + + def url_opener(data, handler=reraise_exception, **kw): + for sample in data: + url = sample["url"] + try: + stream = gopen_with_s3(url, **kw) + # stream = get_bytes_io(url) + sample.update(stream=stream) + yield sample + except Exception as exn: + exn.args = exn.args + (url,) + if handler(exn): + continue + else: + break + + wds.tariterators.url_opener = url_opener + + diff --git a/fourm/utils/scheduler.py b/fourm/utils/scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..1112b6fe7689898cb1af47a710beaa4f971b83fc --- /dev/null +++ b/fourm/utils/scheduler.py @@ -0,0 +1,83 @@ + +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -------------------------------------------------------- +# Based on DINO code base +# https://github.com/facebookresearch/dino +# -------------------------------------------------------- +import numpy as np +import math + +def cosine_scheduler(base_value, final_value, epochs, niter_per_ep, warmup_epochs=0, + start_warmup_value=0, warmup_steps=-1): + warmup_schedule = np.array([]) + warmup_iters = warmup_epochs * niter_per_ep + if warmup_steps > 0: + warmup_iters = warmup_steps + print("Set warmup steps = %d" % warmup_iters) + if warmup_epochs > 0 or warmup_steps > 0: + warmup_schedule = np.linspace(start_warmup_value, base_value, warmup_iters) + + iters = np.arange(epochs * niter_per_ep - warmup_iters) + schedule = np.array( + [final_value + 0.5 * (base_value - final_value) * (1 + math.cos(math.pi * i / (len(iters)))) for i in iters]) + + schedule = np.concatenate((warmup_schedule, schedule)) + + assert len(schedule) == epochs * niter_per_ep + return schedule + + +def constant_scheduler(base_value, epochs, niter_per_ep): + return base_value * np.ones(epochs*niter_per_ep) + + +def inverse_sqrt_scheduler(base_value, final_value, epochs, niter_per_ep, warmup_epochs=0, + start_warmup_value=0, warmup_steps=-1, + cooldown_epochs=0, cooldown_steps=-1, + timescale=10_000): + + warmup_iters = warmup_epochs * niter_per_ep + if warmup_steps > 0: + warmup_iters = warmup_steps + print("Set warmup steps = %d" % warmup_iters) + + cooldown_iters = cooldown_epochs * niter_per_ep + if cooldown_steps > 0: + cooldown_iters = cooldown_steps + print("Set cooldown steps = %d" % cooldown_iters) + + # Warmup schedule + if warmup_epochs > 0 or warmup_steps > 0: + warmup_schedule = np.linspace(start_warmup_value, base_value, warmup_iters) + else: + warmup_schedule = np.array([]) + + # Inverse square-root LR schedule + iters = np.arange(epochs * niter_per_ep - warmup_iters - cooldown_iters) + if base_value == final_value: + schedule = base_value * np.ones(len(iters)) + else: + schedule = base_value / np.sqrt((iters + timescale) / timescale) + + # Cooldown schedule + if cooldown_epochs > 0 or cooldown_steps > 0: + cooldown_schedule = np.linspace(schedule[-1], final_value, cooldown_iters) + else: + cooldown_schedule = np.array([]) + + schedule = np.concatenate((warmup_schedule, schedule, cooldown_schedule)) + + assert len(schedule) == epochs * niter_per_ep + return schedule \ No newline at end of file diff --git a/fourm/utils/seeds.py b/fourm/utils/seeds.py new file mode 100644 index 0000000000000000000000000000000000000000..8e18f02983c00c998f7b4828a158d4cac6e2b1cf --- /dev/null +++ b/fourm/utils/seeds.py @@ -0,0 +1,32 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib + +def generate_seed(*seeds): + # Create a hash object using the SHA-256 algorithm + hash_object = hashlib.sha256() + + # Combine all seeds into a single string + combined_seeds = ''.join(str(seed) for seed in seeds) + + # Update the hash object with the combined seeds + hash_object.update(combined_seeds.encode('utf-8')) + + # Get the hexadecimal digest of the hash object + hex_digest = hash_object.hexdigest() + + # Convert the hexadecimal digest to a 32-bit integer + seed_int = int(hex_digest, 16) % 2**32 + + return seed_int \ No newline at end of file diff --git a/fourm/utils/timm/auto_augment.py b/fourm/utils/timm/auto_augment.py new file mode 100644 index 0000000000000000000000000000000000000000..35eb70332ca1c47e549626a3e5cd4310370abe1f --- /dev/null +++ b/fourm/utils/timm/auto_augment.py @@ -0,0 +1,835 @@ +# -------------------------------------------------------- +# Based on the timm code base +# https://github.com/huggingface/pytorch-image-models +# -------------------------------------------------------- + +""" AutoAugment, RandAugment, and AugMix for PyTorch + +This code implements the searched ImageNet policies with various tweaks and improvements and +does not include any of the search code. + +AA and RA Implementation adapted from: + https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/autoaugment.py + +AugMix adapted from: + https://github.com/google-research/augmix + +Papers: + AutoAugment: Learning Augmentation Policies from Data - https://arxiv.org/abs/1805.09501 + Learning Data Augmentation Strategies for Object Detection - https://arxiv.org/abs/1906.11172 + RandAugment: Practical automated data augmentation... - https://arxiv.org/abs/1909.13719 + AugMix: A Simple Data Processing Method to Improve Robustness and Uncertainty - https://arxiv.org/abs/1912.02781 + +Hacked together by / Copyright 2020 Ross Wightman +""" +import math +import random +import re + +import numpy as np +import PIL +from PIL import Image, ImageEnhance, ImageOps + +_PIL_VER = tuple([int(x) for x in PIL.__version__.split('.')[:2]]) + +_FILL = (128, 128, 128) + +_LEVEL_DENOM = 10. # denominator for conversion from 'Mx' magnitude scale to fractional aug level for op arguments + +_HPARAMS_DEFAULT = dict( + translate_const=250, + img_mean=_FILL, +) + +_RANDOM_INTERPOLATION = (Image.BILINEAR, Image.BICUBIC) + + +def _interpolation(kwargs): + interpolation = kwargs.pop('resample', Image.BILINEAR) + if isinstance(interpolation, (list, tuple)): + return random.choice(interpolation) + else: + return interpolation + + +def _check_args_tf(kwargs): + if 'fillcolor' in kwargs and _PIL_VER < (5, 0): + kwargs.pop('fillcolor') + kwargs['resample'] = _interpolation(kwargs) + + +def shear_x(img, factor, **kwargs): + _check_args_tf(kwargs) + return img.transform(img.size, Image.AFFINE, (1, factor, 0, 0, 1, 0), **kwargs) + + +def shear_y(img, factor, **kwargs): + _check_args_tf(kwargs) + return img.transform(img.size, Image.AFFINE, (1, 0, 0, factor, 1, 0), **kwargs) + + +def translate_x_rel(img, pct, **kwargs): + pixels = pct * img.size[0] + _check_args_tf(kwargs) + return img.transform(img.size, Image.AFFINE, (1, 0, pixels, 0, 1, 0), **kwargs) + + +def translate_y_rel(img, pct, **kwargs): + pixels = pct * img.size[1] + _check_args_tf(kwargs) + return img.transform(img.size, Image.AFFINE, (1, 0, 0, 0, 1, pixels), **kwargs) + + +def translate_x_abs(img, pixels, **kwargs): + _check_args_tf(kwargs) + return img.transform(img.size, Image.AFFINE, (1, 0, pixels, 0, 1, 0), **kwargs) + + +def translate_y_abs(img, pixels, **kwargs): + _check_args_tf(kwargs) + return img.transform(img.size, Image.AFFINE, (1, 0, 0, 0, 1, pixels), **kwargs) + + +def rotate(img, degrees, **kwargs): + _check_args_tf(kwargs) + if _PIL_VER >= (5, 2): + return img.rotate(degrees, **kwargs) + elif _PIL_VER >= (5, 0): + w, h = img.size + post_trans = (0, 0) + rotn_center = (w / 2.0, h / 2.0) + angle = -math.radians(degrees) + matrix = [ + round(math.cos(angle), 15), + round(math.sin(angle), 15), + 0.0, + round(-math.sin(angle), 15), + round(math.cos(angle), 15), + 0.0, + ] + + def transform(x, y, matrix): + (a, b, c, d, e, f) = matrix + return a * x + b * y + c, d * x + e * y + f + + matrix[2], matrix[5] = transform( + -rotn_center[0] - post_trans[0], -rotn_center[1] - post_trans[1], matrix + ) + matrix[2] += rotn_center[0] + matrix[5] += rotn_center[1] + return img.transform(img.size, Image.AFFINE, matrix, **kwargs) + else: + return img.rotate(degrees, resample=kwargs['resample']) + + +def auto_contrast(img, **__): + return ImageOps.autocontrast(img) + + +def invert(img, **__): + return ImageOps.invert(img) + + +def equalize(img, **__): + return ImageOps.equalize(img) + + +def solarize(img, thresh, **__): + return ImageOps.solarize(img, thresh) + + +def solarize_add(img, add, thresh=128, **__): + lut = [] + for i in range(256): + if i < thresh: + lut.append(min(255, i + add)) + else: + lut.append(i) + if img.mode in ("L", "RGB"): + if img.mode == "RGB" and len(lut) == 256: + lut = lut + lut + lut + return img.point(lut) + else: + return img + + +def posterize(img, bits_to_keep, **__): + if bits_to_keep >= 8: + return img + return ImageOps.posterize(img, bits_to_keep) + + +def contrast(img, factor, **__): + return ImageEnhance.Contrast(img).enhance(factor) + + +def color(img, factor, **__): + return ImageEnhance.Color(img).enhance(factor) + + +def brightness(img, factor, **__): + return ImageEnhance.Brightness(img).enhance(factor) + + +def sharpness(img, factor, **__): + return ImageEnhance.Sharpness(img).enhance(factor) + + +def _randomly_negate(v): + """With 50% prob, negate the value""" + return -v if random.random() > 0.5 else v + + +def _rotate_level_to_arg(level, _hparams): + # range [-30, 30] + level = (level / _LEVEL_DENOM) * 30. + level = _randomly_negate(level) + return level, + + +def _enhance_level_to_arg(level, _hparams): + # range [0.1, 1.9] + return (level / _LEVEL_DENOM) * 1.8 + 0.1, + + +def _enhance_increasing_level_to_arg(level, _hparams): + # the 'no change' level is 1.0, moving away from that towards 0. or 2.0 increases the enhancement blend + # range [0.1, 1.9] if level <= _LEVEL_DENOM + level = (level / _LEVEL_DENOM) * .9 + level = max(0.1, 1.0 + _randomly_negate(level)) # keep it >= 0.1 + return level, + + +def _shear_level_to_arg(level, _hparams): + # range [-0.3, 0.3] + level = (level / _LEVEL_DENOM) * 0.3 + level = _randomly_negate(level) + return level, + + +def _translate_abs_level_to_arg(level, hparams): + translate_const = hparams['translate_const'] + level = (level / _LEVEL_DENOM) * float(translate_const) + level = _randomly_negate(level) + return level, + + +def _translate_rel_level_to_arg(level, hparams): + # default range [-0.45, 0.45] + translate_pct = hparams.get('translate_pct', 0.45) + level = (level / _LEVEL_DENOM) * translate_pct + level = _randomly_negate(level) + return level, + + +def _posterize_level_to_arg(level, _hparams): + # As per Tensorflow TPU EfficientNet impl + # range [0, 4], 'keep 0 up to 4 MSB of original image' + # intensity/severity of augmentation decreases with level + return int((level / _LEVEL_DENOM) * 4), + + +def _posterize_increasing_level_to_arg(level, hparams): + # As per Tensorflow models research and UDA impl + # range [4, 0], 'keep 4 down to 0 MSB of original image', + # intensity/severity of augmentation increases with level + return 4 - _posterize_level_to_arg(level, hparams)[0], + + +def _posterize_original_level_to_arg(level, _hparams): + # As per original AutoAugment paper description + # range [4, 8], 'keep 4 up to 8 MSB of image' + # intensity/severity of augmentation decreases with level + return int((level / _LEVEL_DENOM) * 4) + 4, + + +def _solarize_level_to_arg(level, _hparams): + # range [0, 256] + # intensity/severity of augmentation decreases with level + return int((level / _LEVEL_DENOM) * 256), + + +def _solarize_increasing_level_to_arg(level, _hparams): + # range [0, 256] + # intensity/severity of augmentation increases with level + return 256 - _solarize_level_to_arg(level, _hparams)[0], + + +def _solarize_add_level_to_arg(level, _hparams): + # range [0, 110] + return int((level / _LEVEL_DENOM) * 110), + + +LEVEL_TO_ARG = { + 'AutoContrast': None, + 'Equalize': None, + 'Invert': None, + 'Rotate': _rotate_level_to_arg, + # There are several variations of the posterize level scaling in various Tensorflow/Google repositories/papers + 'Posterize': _posterize_level_to_arg, + 'PosterizeIncreasing': _posterize_increasing_level_to_arg, + 'PosterizeOriginal': _posterize_original_level_to_arg, + 'Solarize': _solarize_level_to_arg, + 'SolarizeIncreasing': _solarize_increasing_level_to_arg, + 'SolarizeAdd': _solarize_add_level_to_arg, + 'Color': _enhance_level_to_arg, + 'ColorIncreasing': _enhance_increasing_level_to_arg, + 'Contrast': _enhance_level_to_arg, + 'ContrastIncreasing': _enhance_increasing_level_to_arg, + 'Brightness': _enhance_level_to_arg, + 'BrightnessIncreasing': _enhance_increasing_level_to_arg, + 'Sharpness': _enhance_level_to_arg, + 'SharpnessIncreasing': _enhance_increasing_level_to_arg, + 'ShearX': _shear_level_to_arg, + 'ShearY': _shear_level_to_arg, + 'TranslateX': _translate_abs_level_to_arg, + 'TranslateY': _translate_abs_level_to_arg, + 'TranslateXRel': _translate_rel_level_to_arg, + 'TranslateYRel': _translate_rel_level_to_arg, +} + +NAME_TO_OP = { + 'AutoContrast': auto_contrast, + 'Equalize': equalize, + 'Invert': invert, + 'Rotate': rotate, + 'Posterize': posterize, + 'PosterizeIncreasing': posterize, + 'PosterizeOriginal': posterize, + 'Solarize': solarize, + 'SolarizeIncreasing': solarize, + 'SolarizeAdd': solarize_add, + 'Color': color, + 'ColorIncreasing': color, + 'Contrast': contrast, + 'ContrastIncreasing': contrast, + 'Brightness': brightness, + 'BrightnessIncreasing': brightness, + 'Sharpness': sharpness, + 'SharpnessIncreasing': sharpness, + 'ShearX': shear_x, + 'ShearY': shear_y, + 'TranslateX': translate_x_abs, + 'TranslateY': translate_y_abs, + 'TranslateXRel': translate_x_rel, + 'TranslateYRel': translate_y_rel, +} + + +class AugmentOp: + + def __init__(self, name, prob=0.5, magnitude=10, hparams=None): + hparams = hparams or _HPARAMS_DEFAULT + self.aug_fn = NAME_TO_OP[name] + self.level_fn = LEVEL_TO_ARG[name] + self.prob = prob + self.magnitude = magnitude + self.hparams = hparams.copy() + self.kwargs = dict( + fillcolor=hparams['img_mean'] if 'img_mean' in hparams else _FILL, + resample=hparams['interpolation'] if 'interpolation' in hparams else _RANDOM_INTERPOLATION, + ) + + # If magnitude_std is > 0, we introduce some randomness + # in the usually fixed policy and sample magnitude from a normal distribution + # with mean `magnitude` and std-dev of `magnitude_std`. + # NOTE This is my own hack, being tested, not in papers or reference impls. + # If magnitude_std is inf, we sample magnitude from a uniform distribution + self.magnitude_std = self.hparams.get('magnitude_std', 0) + self.magnitude_max = self.hparams.get('magnitude_max', None) + + def __call__(self, img): + if self.prob < 1.0 and random.random() > self.prob: + return img + magnitude = self.magnitude + if self.magnitude_std > 0: + # magnitude randomization enabled + if self.magnitude_std == float('inf'): + magnitude = random.uniform(0, magnitude) + elif self.magnitude_std > 0: + magnitude = random.gauss(magnitude, self.magnitude_std) + # default upper_bound for the timm RA impl is _LEVEL_DENOM (10) + # setting magnitude_max overrides this to allow M > 10 (behaviour closer to Google TF RA impl) + upper_bound = self.magnitude_max or _LEVEL_DENOM + magnitude = max(0., min(magnitude, upper_bound)) + level_args = self.level_fn(magnitude, self.hparams) if self.level_fn is not None else tuple() + return self.aug_fn(img, *level_args, **self.kwargs) + + +def auto_augment_policy_v0(hparams): + # ImageNet v0 policy from TPU EfficientNet impl, cannot find a paper reference. + policy = [ + [('Equalize', 0.8, 1), ('ShearY', 0.8, 4)], + [('Color', 0.4, 9), ('Equalize', 0.6, 3)], + [('Color', 0.4, 1), ('Rotate', 0.6, 8)], + [('Solarize', 0.8, 3), ('Equalize', 0.4, 7)], + [('Solarize', 0.4, 2), ('Solarize', 0.6, 2)], + [('Color', 0.2, 0), ('Equalize', 0.8, 8)], + [('Equalize', 0.4, 8), ('SolarizeAdd', 0.8, 3)], + [('ShearX', 0.2, 9), ('Rotate', 0.6, 8)], + [('Color', 0.6, 1), ('Equalize', 1.0, 2)], + [('Invert', 0.4, 9), ('Rotate', 0.6, 0)], + [('Equalize', 1.0, 9), ('ShearY', 0.6, 3)], + [('Color', 0.4, 7), ('Equalize', 0.6, 0)], + [('Posterize', 0.4, 6), ('AutoContrast', 0.4, 7)], + [('Solarize', 0.6, 8), ('Color', 0.6, 9)], + [('Solarize', 0.2, 4), ('Rotate', 0.8, 9)], + [('Rotate', 1.0, 7), ('TranslateYRel', 0.8, 9)], + [('ShearX', 0.0, 0), ('Solarize', 0.8, 4)], + [('ShearY', 0.8, 0), ('Color', 0.6, 4)], + [('Color', 1.0, 0), ('Rotate', 0.6, 2)], + [('Equalize', 0.8, 4), ('Equalize', 0.0, 8)], + [('Equalize', 1.0, 4), ('AutoContrast', 0.6, 2)], + [('ShearY', 0.4, 7), ('SolarizeAdd', 0.6, 7)], + [('Posterize', 0.8, 2), ('Solarize', 0.6, 10)], # This results in black image with Tpu posterize + [('Solarize', 0.6, 8), ('Equalize', 0.6, 1)], + [('Color', 0.8, 6), ('Rotate', 0.4, 5)], + ] + pc = [[AugmentOp(*a, hparams=hparams) for a in sp] for sp in policy] + return pc + + +def auto_augment_policy_v0r(hparams): + # ImageNet v0 policy from TPU EfficientNet impl, with variation of Posterize used + # in Google research implementation (number of bits discarded increases with magnitude) + policy = [ + [('Equalize', 0.8, 1), ('ShearY', 0.8, 4)], + [('Color', 0.4, 9), ('Equalize', 0.6, 3)], + [('Color', 0.4, 1), ('Rotate', 0.6, 8)], + [('Solarize', 0.8, 3), ('Equalize', 0.4, 7)], + [('Solarize', 0.4, 2), ('Solarize', 0.6, 2)], + [('Color', 0.2, 0), ('Equalize', 0.8, 8)], + [('Equalize', 0.4, 8), ('SolarizeAdd', 0.8, 3)], + [('ShearX', 0.2, 9), ('Rotate', 0.6, 8)], + [('Color', 0.6, 1), ('Equalize', 1.0, 2)], + [('Invert', 0.4, 9), ('Rotate', 0.6, 0)], + [('Equalize', 1.0, 9), ('ShearY', 0.6, 3)], + [('Color', 0.4, 7), ('Equalize', 0.6, 0)], + [('PosterizeIncreasing', 0.4, 6), ('AutoContrast', 0.4, 7)], + [('Solarize', 0.6, 8), ('Color', 0.6, 9)], + [('Solarize', 0.2, 4), ('Rotate', 0.8, 9)], + [('Rotate', 1.0, 7), ('TranslateYRel', 0.8, 9)], + [('ShearX', 0.0, 0), ('Solarize', 0.8, 4)], + [('ShearY', 0.8, 0), ('Color', 0.6, 4)], + [('Color', 1.0, 0), ('Rotate', 0.6, 2)], + [('Equalize', 0.8, 4), ('Equalize', 0.0, 8)], + [('Equalize', 1.0, 4), ('AutoContrast', 0.6, 2)], + [('ShearY', 0.4, 7), ('SolarizeAdd', 0.6, 7)], + [('PosterizeIncreasing', 0.8, 2), ('Solarize', 0.6, 10)], + [('Solarize', 0.6, 8), ('Equalize', 0.6, 1)], + [('Color', 0.8, 6), ('Rotate', 0.4, 5)], + ] + pc = [[AugmentOp(*a, hparams=hparams) for a in sp] for sp in policy] + return pc + + +def auto_augment_policy_original(hparams): + # ImageNet policy from https://arxiv.org/abs/1805.09501 + policy = [ + [('PosterizeOriginal', 0.4, 8), ('Rotate', 0.6, 9)], + [('Solarize', 0.6, 5), ('AutoContrast', 0.6, 5)], + [('Equalize', 0.8, 8), ('Equalize', 0.6, 3)], + [('PosterizeOriginal', 0.6, 7), ('PosterizeOriginal', 0.6, 6)], + [('Equalize', 0.4, 7), ('Solarize', 0.2, 4)], + [('Equalize', 0.4, 4), ('Rotate', 0.8, 8)], + [('Solarize', 0.6, 3), ('Equalize', 0.6, 7)], + [('PosterizeOriginal', 0.8, 5), ('Equalize', 1.0, 2)], + [('Rotate', 0.2, 3), ('Solarize', 0.6, 8)], + [('Equalize', 0.6, 8), ('PosterizeOriginal', 0.4, 6)], + [('Rotate', 0.8, 8), ('Color', 0.4, 0)], + [('Rotate', 0.4, 9), ('Equalize', 0.6, 2)], + [('Equalize', 0.0, 7), ('Equalize', 0.8, 8)], + [('Invert', 0.6, 4), ('Equalize', 1.0, 8)], + [('Color', 0.6, 4), ('Contrast', 1.0, 8)], + [('Rotate', 0.8, 8), ('Color', 1.0, 2)], + [('Color', 0.8, 8), ('Solarize', 0.8, 7)], + [('Sharpness', 0.4, 7), ('Invert', 0.6, 8)], + [('ShearX', 0.6, 5), ('Equalize', 1.0, 9)], + [('Color', 0.4, 0), ('Equalize', 0.6, 3)], + [('Equalize', 0.4, 7), ('Solarize', 0.2, 4)], + [('Solarize', 0.6, 5), ('AutoContrast', 0.6, 5)], + [('Invert', 0.6, 4), ('Equalize', 1.0, 8)], + [('Color', 0.6, 4), ('Contrast', 1.0, 8)], + [('Equalize', 0.8, 8), ('Equalize', 0.6, 3)], + ] + pc = [[AugmentOp(*a, hparams=hparams) for a in sp] for sp in policy] + return pc + + +def auto_augment_policy_originalr(hparams): + # ImageNet policy from https://arxiv.org/abs/1805.09501 with research posterize variation + policy = [ + [('PosterizeIncreasing', 0.4, 8), ('Rotate', 0.6, 9)], + [('Solarize', 0.6, 5), ('AutoContrast', 0.6, 5)], + [('Equalize', 0.8, 8), ('Equalize', 0.6, 3)], + [('PosterizeIncreasing', 0.6, 7), ('PosterizeIncreasing', 0.6, 6)], + [('Equalize', 0.4, 7), ('Solarize', 0.2, 4)], + [('Equalize', 0.4, 4), ('Rotate', 0.8, 8)], + [('Solarize', 0.6, 3), ('Equalize', 0.6, 7)], + [('PosterizeIncreasing', 0.8, 5), ('Equalize', 1.0, 2)], + [('Rotate', 0.2, 3), ('Solarize', 0.6, 8)], + [('Equalize', 0.6, 8), ('PosterizeIncreasing', 0.4, 6)], + [('Rotate', 0.8, 8), ('Color', 0.4, 0)], + [('Rotate', 0.4, 9), ('Equalize', 0.6, 2)], + [('Equalize', 0.0, 7), ('Equalize', 0.8, 8)], + [('Invert', 0.6, 4), ('Equalize', 1.0, 8)], + [('Color', 0.6, 4), ('Contrast', 1.0, 8)], + [('Rotate', 0.8, 8), ('Color', 1.0, 2)], + [('Color', 0.8, 8), ('Solarize', 0.8, 7)], + [('Sharpness', 0.4, 7), ('Invert', 0.6, 8)], + [('ShearX', 0.6, 5), ('Equalize', 1.0, 9)], + [('Color', 0.4, 0), ('Equalize', 0.6, 3)], + [('Equalize', 0.4, 7), ('Solarize', 0.2, 4)], + [('Solarize', 0.6, 5), ('AutoContrast', 0.6, 5)], + [('Invert', 0.6, 4), ('Equalize', 1.0, 8)], + [('Color', 0.6, 4), ('Contrast', 1.0, 8)], + [('Equalize', 0.8, 8), ('Equalize', 0.6, 3)], + ] + pc = [[AugmentOp(*a, hparams=hparams) for a in sp] for sp in policy] + return pc + + +def auto_augment_policy(name='v0', hparams=None): + hparams = hparams or _HPARAMS_DEFAULT + if name == 'original': + return auto_augment_policy_original(hparams) + elif name == 'originalr': + return auto_augment_policy_originalr(hparams) + elif name == 'v0': + return auto_augment_policy_v0(hparams) + elif name == 'v0r': + return auto_augment_policy_v0r(hparams) + else: + assert False, 'Unknown AA policy (%s)' % name + + +class AutoAugment: + + def __init__(self, policy): + self.policy = policy + + def __call__(self, img): + sub_policy = random.choice(self.policy) + for op in sub_policy: + img = op(img) + return img + + +def auto_augment_transform(config_str, hparams): + """ + Create a AutoAugment transform + + :param config_str: String defining configuration of auto augmentation. Consists of multiple sections separated by + dashes ('-'). The first section defines the AutoAugment policy (one of 'v0', 'v0r', 'original', 'originalr'). + The remaining sections, not order sepecific determine + 'mstd' - float std deviation of magnitude noise applied + Ex 'original-mstd0.5' results in AutoAugment with original policy, magnitude_std 0.5 + + :param hparams: Other hparams (kwargs) for the AutoAugmentation scheme + + :return: A PyTorch compatible Transform + """ + config = config_str.split('-') + policy_name = config[0] + config = config[1:] + for c in config: + cs = re.split(r'(\d.*)', c) + if len(cs) < 2: + continue + key, val = cs[:2] + if key == 'mstd': + # noise param injected via hparams for now + hparams.setdefault('magnitude_std', float(val)) + else: + assert False, 'Unknown AutoAugment config section' + aa_policy = auto_augment_policy(policy_name, hparams=hparams) + return AutoAugment(aa_policy) + + +_RAND_TRANSFORMS = [ + 'AutoContrast', + 'Equalize', + 'Invert', + 'Rotate', + 'Posterize', + 'Solarize', + 'SolarizeAdd', + 'Color', + 'Contrast', + 'Brightness', + 'Sharpness', + 'ShearX', + 'ShearY', + 'TranslateXRel', + 'TranslateYRel', + # 'Cutout' # NOTE I've implement this as random erasing separately +] + +_RAND_INCREASING_TRANSFORMS = [ + 'AutoContrast', + 'Equalize', + 'Invert', + 'Rotate', + 'PosterizeIncreasing', + 'SolarizeIncreasing', + 'SolarizeAdd', + 'ColorIncreasing', + 'ContrastIncreasing', + 'BrightnessIncreasing', + 'SharpnessIncreasing', + 'ShearX', + 'ShearY', + 'TranslateXRel', + 'TranslateYRel', + # 'Cutout' # NOTE I've implement this as random erasing separately +] + +# These experimental weights are based loosely on the relative improvements mentioned in paper. +# They may not result in increased performance, but could likely be tuned to so. +_RAND_CHOICE_WEIGHTS_0 = { + 'Rotate': 0.3, + 'ShearX': 0.2, + 'ShearY': 0.2, + 'TranslateXRel': 0.1, + 'TranslateYRel': 0.1, + 'Color': .025, + 'Sharpness': 0.025, + 'AutoContrast': 0.025, + 'Solarize': .005, + 'SolarizeAdd': .005, + 'Contrast': .005, + 'Brightness': .005, + 'Equalize': .005, + 'Posterize': 0, + 'Invert': 0, +} + + +def _select_rand_weights(weight_idx=0, transforms=None): + transforms = transforms or _RAND_TRANSFORMS + assert weight_idx == 0 # only one set of weights currently + rand_weights = _RAND_CHOICE_WEIGHTS_0 + probs = [rand_weights[k] for k in transforms] + probs /= np.sum(probs) + return probs + + +def rand_augment_ops(magnitude=10, hparams=None, transforms=None): + hparams = hparams or _HPARAMS_DEFAULT + transforms = transforms or _RAND_TRANSFORMS + return [AugmentOp( + name, prob=0.5, magnitude=magnitude, hparams=hparams) for name in transforms] + + +class RandAugment: + def __init__(self, ops, num_layers=2, choice_weights=None): + self.ops = ops + self.num_layers = num_layers + self.choice_weights = choice_weights + + def __call__(self, img): + # no replacement when using weighted choice + ops = np.random.choice( + self.ops, self.num_layers, replace=self.choice_weights is None, p=self.choice_weights) + for op in ops: + img = op(img) + return img + + +def rand_augment_transform(config_str, hparams): + """ + Create a RandAugment transform + + :param config_str: String defining configuration of random augmentation. Consists of multiple sections separated by + dashes ('-'). The first section defines the specific variant of rand augment (currently only 'rand'). The remaining + sections, not order sepecific determine + 'm' - integer magnitude of rand augment + 'n' - integer num layers (number of transform ops selected per image) + 'w' - integer probabiliy weight index (index of a set of weights to influence choice of op) + 'mstd' - float std deviation of magnitude noise applied, or uniform sampling if infinity (or > 100) + 'mmax' - set upper bound for magnitude to something other than default of _LEVEL_DENOM (10) + 'inc' - integer (bool), use augmentations that increase in severity with magnitude (default: 0) + Ex 'rand-m9-n3-mstd0.5' results in RandAugment with magnitude 9, num_layers 3, magnitude_std 0.5 + 'rand-mstd1-w0' results in magnitude_std 1.0, weights 0, default magnitude of 10 and num_layers 2 + + :param hparams: Other hparams (kwargs) for the RandAugmentation scheme + + :return: A PyTorch compatible Transform + """ + magnitude = _LEVEL_DENOM # default to _LEVEL_DENOM for magnitude (currently 10) + num_layers = 2 # default to 2 ops per image + weight_idx = None # default to no probability weights for op choice + transforms = _RAND_TRANSFORMS + config = config_str.split('-') + assert config[0] == 'rand' + config = config[1:] + for c in config: + cs = re.split(r'(\d.*)', c) + if len(cs) < 2: + continue + key, val = cs[:2] + if key == 'mstd': + # noise param / randomization of magnitude values + mstd = float(val) + if mstd > 100: + # use uniform sampling in 0 to magnitude if mstd is > 100 + mstd = float('inf') + hparams.setdefault('magnitude_std', mstd) + elif key == 'mmax': + # clip magnitude between [0, mmax] instead of default [0, _LEVEL_DENOM] + hparams.setdefault('magnitude_max', int(val)) + elif key == 'inc': + if bool(val): + transforms = _RAND_INCREASING_TRANSFORMS + elif key == 'm': + magnitude = int(val) + elif key == 'n': + num_layers = int(val) + elif key == 'w': + weight_idx = int(val) + else: + assert False, 'Unknown RandAugment config section' + ra_ops = rand_augment_ops(magnitude=magnitude, hparams=hparams, transforms=transforms) + choice_weights = None if weight_idx is None else _select_rand_weights(weight_idx) + return RandAugment(ra_ops, num_layers, choice_weights=choice_weights) + + +_AUGMIX_TRANSFORMS = [ + 'AutoContrast', + 'ColorIncreasing', # not in paper + 'ContrastIncreasing', # not in paper + 'BrightnessIncreasing', # not in paper + 'SharpnessIncreasing', # not in paper + 'Equalize', + 'Rotate', + 'PosterizeIncreasing', + 'SolarizeIncreasing', + 'ShearX', + 'ShearY', + 'TranslateXRel', + 'TranslateYRel', +] + + +def augmix_ops(magnitude=10, hparams=None, transforms=None): + hparams = hparams or _HPARAMS_DEFAULT + transforms = transforms or _AUGMIX_TRANSFORMS + return [AugmentOp( + name, prob=1.0, magnitude=magnitude, hparams=hparams) for name in transforms] + + +class AugMixAugment: + """ AugMix Transform + Adapted and improved from impl here: https://github.com/google-research/augmix/blob/master/imagenet.py + From paper: 'AugMix: A Simple Data Processing Method to Improve Robustness and Uncertainty - + https://arxiv.org/abs/1912.02781 + """ + + def __init__(self, ops, alpha=1., width=3, depth=-1, blended=False): + self.ops = ops + self.alpha = alpha + self.width = width + self.depth = depth + self.blended = blended # blended mode is faster but not well tested + + def _calc_blended_weights(self, ws, m): + ws = ws * m + cump = 1. + rws = [] + for w in ws[::-1]: + alpha = w / cump + cump *= (1 - alpha) + rws.append(alpha) + return np.array(rws[::-1], dtype=np.float32) + + def _apply_blended(self, img, mixing_weights, m): + # This is my first crack and implementing a slightly faster mixed augmentation. Instead + # of accumulating the mix for each chain in a Numpy array and then blending with original, + # it recomputes the blending coefficients and applies one PIL image blend per chain. + # TODO the results appear in the right ballpark but they differ by more than rounding. + img_orig = img.copy() + ws = self._calc_blended_weights(mixing_weights, m) + for w in ws: + depth = self.depth if self.depth > 0 else np.random.randint(1, 4) + ops = np.random.choice(self.ops, depth, replace=True) + img_aug = img_orig # no ops are in-place, deep copy not necessary + for op in ops: + img_aug = op(img_aug) + img = Image.blend(img, img_aug, w) + return img + + def _apply_basic(self, img, mixing_weights, m): + # This is a literal adaptation of the paper/official implementation without normalizations and + # PIL <-> Numpy conversions between every op. It is still quite CPU compute heavy compared to the + # typical augmentation transforms, could use a GPU / Kornia implementation. + img_shape = img.size[0], img.size[1], len(img.getbands()) + mixed = np.zeros(img_shape, dtype=np.float32) + for mw in mixing_weights: + depth = self.depth if self.depth > 0 else np.random.randint(1, 4) + ops = np.random.choice(self.ops, depth, replace=True) + img_aug = img # no ops are in-place, deep copy not necessary + for op in ops: + img_aug = op(img_aug) + mixed += mw * np.asarray(img_aug, dtype=np.float32) + np.clip(mixed, 0, 255., out=mixed) + mixed = Image.fromarray(mixed.astype(np.uint8)) + return Image.blend(img, mixed, m) + + def __call__(self, img): + mixing_weights = np.float32(np.random.dirichlet([self.alpha] * self.width)) + m = np.float32(np.random.beta(self.alpha, self.alpha)) + if self.blended: + mixed = self._apply_blended(img, mixing_weights, m) + else: + mixed = self._apply_basic(img, mixing_weights, m) + return mixed + + +def augment_and_mix_transform(config_str, hparams): + """ Create AugMix PyTorch transform + + :param config_str: String defining configuration of random augmentation. Consists of multiple sections separated by + dashes ('-'). The first section defines the specific variant of rand augment (currently only 'rand'). The remaining + sections, not order sepecific determine + 'm' - integer magnitude (severity) of augmentation mix (default: 3) + 'w' - integer width of augmentation chain (default: 3) + 'd' - integer depth of augmentation chain (-1 is random [1, 3], default: -1) + 'b' - integer (bool), blend each branch of chain into end result without a final blend, less CPU (default: 0) + 'mstd' - float std deviation of magnitude noise applied (default: 0) + Ex 'augmix-m5-w4-d2' results in AugMix with severity 5, chain width 4, chain depth 2 + + :param hparams: Other hparams (kwargs) for the Augmentation transforms + + :return: A PyTorch compatible Transform + """ + magnitude = 3 + width = 3 + depth = -1 + alpha = 1. + blended = False + config = config_str.split('-') + assert config[0] == 'augmix' + config = config[1:] + for c in config: + cs = re.split(r'(\d.*)', c) + if len(cs) < 2: + continue + key, val = cs[:2] + if key == 'mstd': + # noise param injected via hparams for now + hparams.setdefault('magnitude_std', float(val)) + elif key == 'm': + magnitude = int(val) + elif key == 'w': + width = int(val) + elif key == 'd': + depth = int(val) + elif key == 'a': + alpha = float(val) + elif key == 'b': + blended = bool(val) + else: + assert False, 'Unknown AugMix config section' + hparams.setdefault('magnitude_std', float('inf')) # default to uniform sampling (if not set via mstd arg) + ops = augmix_ops(magnitude=magnitude, hparams=hparams) + return AugMixAugment(ops, alpha=alpha, width=width, depth=depth, blended=blended) diff --git a/fourm/utils/timm/cross_entropy.py b/fourm/utils/timm/cross_entropy.py new file mode 100644 index 0000000000000000000000000000000000000000..5d3f17583f0259ac7b468c88a386d517f730dcf3 --- /dev/null +++ b/fourm/utils/timm/cross_entropy.py @@ -0,0 +1,42 @@ +# -------------------------------------------------------- +# Based on the timm code base +# https://github.com/huggingface/pytorch-image-models +# -------------------------------------------------------- + +""" Cross Entropy w/ smoothing or soft targets + +Hacked together by / Copyright 2021 Ross Wightman +""" + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class LabelSmoothingCrossEntropy(nn.Module): + """ NLL loss with label smoothing. + """ + + def __init__(self, smoothing=0.1): + super(LabelSmoothingCrossEntropy, self).__init__() + assert smoothing < 1.0 + self.smoothing = smoothing + self.confidence = 1. - smoothing + + def forward(self, x: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + logprobs = F.log_softmax(x, dim=-1) + nll_loss = -logprobs.gather(dim=-1, index=target.unsqueeze(1)) + nll_loss = nll_loss.squeeze(1) + smooth_loss = -logprobs.mean(dim=-1) + loss = self.confidence * nll_loss + self.smoothing * smooth_loss + return loss.mean() + + +class SoftTargetCrossEntropy(nn.Module): + + def __init__(self): + super(SoftTargetCrossEntropy, self).__init__() + + def forward(self, x: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + loss = torch.sum(-target * F.log_softmax(x, dim=-1), dim=-1) + return loss.mean() diff --git a/fourm/utils/timm/metrics.py b/fourm/utils/timm/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..0d1ee0d4155bd79c03000f22bbddd76347a4d284 --- /dev/null +++ b/fourm/utils/timm/metrics.py @@ -0,0 +1,44 @@ +# -------------------------------------------------------- +# Based on the timm code base +# https://github.com/huggingface/pytorch-image-models +# -------------------------------------------------------- +""" Eval metrics and related + +Hacked together by / Copyright 2020 Ross Wightman +""" + + +class AverageMeter: + """Computes and stores the average and current value""" + + def __init__(self): + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + +def accuracy(output, target, topk=(1,)): + """Computes the accuracy over the k top predictions for the specified values of k""" + maxk = min(max(topk), output.size()[1]) + batch_size = target.size(0) + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.reshape(1, -1).expand_as(pred)) + return [correct[:min(k, maxk)].reshape(-1).float().sum(0) * 100. / batch_size for k in topk] + + +def cls_map(output, target): + # batch_size = target.size(0) + # idx_axes = torch.arange(batch_size) + scores, preds = output.softmax(dim=-1).topk(1, 1, True, True) + return scores, preds diff --git a/fourm/utils/timm/mixup.py b/fourm/utils/timm/mixup.py new file mode 100644 index 0000000000000000000000000000000000000000..0ad9e7f4c4ace60ffb484b7f2ca81b5366110364 --- /dev/null +++ b/fourm/utils/timm/mixup.py @@ -0,0 +1,320 @@ +# -------------------------------------------------------- +# Based on the timm code base +# https://github.com/huggingface/pytorch-image-models +# -------------------------------------------------------- +""" Mixup and Cutmix + +Papers: +mixup: Beyond Empirical Risk Minimization (https://arxiv.org/abs/1710.09412) + +CutMix: Regularization Strategy to Train Strong Classifiers with Localizable Features (https://arxiv.org/abs/1905.04899) + +Code Reference: +CutMix: https://github.com/clovaai/CutMix-PyTorch + +Hacked together by / Copyright 2020 Ross Wightman +""" +import numpy as np +import torch + + +def one_hot(x, num_classes, on_value=1., off_value=0., device='cuda'): + x = x.long().view(-1, 1) + return torch.full((x.size()[0], num_classes), off_value, device=device).scatter_(1, x, on_value) + + +def mixup_target(target, num_classes, lam=1., smoothing=0.0, device='cuda'): + off_value = smoothing / num_classes + on_value = 1. - smoothing + off_value + y1 = one_hot(target, num_classes, on_value=on_value, off_value=off_value, device=device) + y2 = one_hot(target.flip(0), num_classes, on_value=on_value, off_value=off_value, device=device) + return y1 * lam + y2 * (1. - lam) + + +def rand_bbox(img_shape, lam, margin=0., count=None): + """ Standard CutMix bounding-box + Generates a random square bbox based on lambda value. This impl includes + support for enforcing a border margin as percent of bbox dimensions. + + Args: + img_shape (tuple): Image shape as tuple + lam (float): Cutmix lambda value + margin (float): Percentage of bbox dimension to enforce as margin (reduce amount of box outside image) + count (int): Number of bbox to generate + """ + ratio = np.sqrt(1 - lam) + img_h, img_w = img_shape[-2:] + cut_h, cut_w = int(img_h * ratio), int(img_w * ratio) + margin_y, margin_x = int(margin * cut_h), int(margin * cut_w) + cy = np.random.randint(0 + margin_y, img_h - margin_y, size=count) + cx = np.random.randint(0 + margin_x, img_w - margin_x, size=count) + yl = np.clip(cy - cut_h // 2, 0, img_h) + yh = np.clip(cy + cut_h // 2, 0, img_h) + xl = np.clip(cx - cut_w // 2, 0, img_w) + xh = np.clip(cx + cut_w // 2, 0, img_w) + return yl, yh, xl, xh + + +def rand_bbox_minmax(img_shape, minmax, count=None): + """ Min-Max CutMix bounding-box + Inspired by Darknet cutmix impl, generates a random rectangular bbox + based on min/max percent values applied to each dimension of the input image. + + Typical defaults for minmax are usually in the .2-.3 for min and .8-.9 range for max. + + Args: + img_shape (tuple): Image shape as tuple + minmax (tuple or list): Min and max bbox ratios (as percent of image size) + count (int): Number of bbox to generate + """ + assert len(minmax) == 2 + img_h, img_w = img_shape[-2:] + cut_h = np.random.randint(int(img_h * minmax[0]), int(img_h * minmax[1]), size=count) + cut_w = np.random.randint(int(img_w * minmax[0]), int(img_w * minmax[1]), size=count) + yl = np.random.randint(0, img_h - cut_h, size=count) + xl = np.random.randint(0, img_w - cut_w, size=count) + yu = yl + cut_h + xu = xl + cut_w + return yl, yu, xl, xu + + +def cutmix_bbox_and_lam(img_shape, lam, ratio_minmax=None, correct_lam=True, count=None): + """ Generate bbox and apply lambda correction. + """ + if ratio_minmax is not None: + yl, yu, xl, xu = rand_bbox_minmax(img_shape, ratio_minmax, count=count) + else: + yl, yu, xl, xu = rand_bbox(img_shape, lam, count=count) + if correct_lam or ratio_minmax is not None: + bbox_area = (yu - yl) * (xu - xl) + lam = 1. - bbox_area / float(img_shape[-2] * img_shape[-1]) + return (yl, yu, xl, xu), lam + + +class Mixup: + """ Mixup/Cutmix that applies different params to each element or whole batch + + Args: + mixup_alpha (float): mixup alpha value, mixup is active if > 0. + cutmix_alpha (float): cutmix alpha value, cutmix is active if > 0. + cutmix_minmax (List[float]): cutmix min/max image ratio, cutmix is active and uses this vs alpha if not None. + prob (float): probability of applying mixup or cutmix per batch or element + switch_prob (float): probability of switching to cutmix instead of mixup when both are active + mode (str): how to apply mixup/cutmix params (per 'batch', 'pair' (pair of elements), 'elem' (element) + correct_lam (bool): apply lambda correction when cutmix bbox clipped by image borders + label_smoothing (float): apply label smoothing to the mixed target tensor + num_classes (int): number of classes for target + """ + + def __init__(self, mixup_alpha=1., cutmix_alpha=0., cutmix_minmax=None, prob=1.0, switch_prob=0.5, + mode='batch', correct_lam=True, label_smoothing=0.1, num_classes=1000): + self.mixup_alpha = mixup_alpha + self.cutmix_alpha = cutmix_alpha + self.cutmix_minmax = cutmix_minmax + if self.cutmix_minmax is not None: + assert len(self.cutmix_minmax) == 2 + # force cutmix alpha == 1.0 when minmax active to keep logic simple & safe + self.cutmix_alpha = 1.0 + self.mix_prob = prob + self.switch_prob = switch_prob + self.label_smoothing = label_smoothing + self.num_classes = num_classes + self.mode = mode + self.correct_lam = correct_lam # correct lambda based on clipped area for cutmix + self.mixup_enabled = True # set to false to disable mixing (intended tp be set by train loop) + + def _params_per_elem(self, batch_size): + lam = np.ones(batch_size, dtype=np.float32) + use_cutmix = np.zeros(batch_size, dtype=np.bool) + if self.mixup_enabled: + if self.mixup_alpha > 0. and self.cutmix_alpha > 0.: + use_cutmix = np.random.rand(batch_size) < self.switch_prob + lam_mix = np.where( + use_cutmix, + np.random.beta(self.cutmix_alpha, self.cutmix_alpha, size=batch_size), + np.random.beta(self.mixup_alpha, self.mixup_alpha, size=batch_size)) + elif self.mixup_alpha > 0.: + lam_mix = np.random.beta(self.mixup_alpha, self.mixup_alpha, size=batch_size) + elif self.cutmix_alpha > 0.: + use_cutmix = np.ones(batch_size, dtype=np.bool) + lam_mix = np.random.beta(self.cutmix_alpha, self.cutmix_alpha, size=batch_size) + else: + assert False, "One of mixup_alpha > 0., cutmix_alpha > 0., cutmix_minmax not None should be true." + lam = np.where(np.random.rand(batch_size) < self.mix_prob, lam_mix.astype(np.float32), lam) + return lam, use_cutmix + + def _params_per_batch(self): + lam = 1. + use_cutmix = False + if self.mixup_enabled and np.random.rand() < self.mix_prob: + if self.mixup_alpha > 0. and self.cutmix_alpha > 0.: + use_cutmix = np.random.rand() < self.switch_prob + lam_mix = np.random.beta(self.cutmix_alpha, self.cutmix_alpha) if use_cutmix else \ + np.random.beta(self.mixup_alpha, self.mixup_alpha) + elif self.mixup_alpha > 0.: + lam_mix = np.random.beta(self.mixup_alpha, self.mixup_alpha) + elif self.cutmix_alpha > 0.: + use_cutmix = True + lam_mix = np.random.beta(self.cutmix_alpha, self.cutmix_alpha) + else: + assert False, "One of mixup_alpha > 0., cutmix_alpha > 0., cutmix_minmax not None should be true." + lam = float(lam_mix) + return lam, use_cutmix + + def _mix_elem(self, x): + batch_size = len(x) + lam_batch, use_cutmix = self._params_per_elem(batch_size) + x_orig = x.clone() # need to keep an unmodified original for mixing source + for i in range(batch_size): + j = batch_size - i - 1 + lam = lam_batch[i] + if lam != 1.: + if use_cutmix[i]: + (yl, yh, xl, xh), lam = cutmix_bbox_and_lam( + x[i].shape, lam, ratio_minmax=self.cutmix_minmax, correct_lam=self.correct_lam) + x[i][:, yl:yh, xl:xh] = x_orig[j][:, yl:yh, xl:xh] + lam_batch[i] = lam + else: + x[i] = x[i] * lam + x_orig[j] * (1 - lam) + return torch.tensor(lam_batch, device=x.device, dtype=x.dtype).unsqueeze(1) + + def _mix_pair(self, x): + batch_size = len(x) + lam_batch, use_cutmix = self._params_per_elem(batch_size // 2) + x_orig = x.clone() # need to keep an unmodified original for mixing source + for i in range(batch_size // 2): + j = batch_size - i - 1 + lam = lam_batch[i] + if lam != 1.: + if use_cutmix[i]: + (yl, yh, xl, xh), lam = cutmix_bbox_and_lam( + x[i].shape, lam, ratio_minmax=self.cutmix_minmax, correct_lam=self.correct_lam) + x[i][:, yl:yh, xl:xh] = x_orig[j][:, yl:yh, xl:xh] + x[j][:, yl:yh, xl:xh] = x_orig[i][:, yl:yh, xl:xh] + lam_batch[i] = lam + else: + x[i] = x[i] * lam + x_orig[j] * (1 - lam) + x[j] = x[j] * lam + x_orig[i] * (1 - lam) + lam_batch = np.concatenate((lam_batch, lam_batch[::-1])) + return torch.tensor(lam_batch, device=x.device, dtype=x.dtype).unsqueeze(1) + + def _mix_batch(self, x): + lam, use_cutmix = self._params_per_batch() + if lam == 1.: + return 1. + if use_cutmix: + (yl, yh, xl, xh), lam = cutmix_bbox_and_lam( + x.shape, lam, ratio_minmax=self.cutmix_minmax, correct_lam=self.correct_lam) + x[:, :, yl:yh, xl:xh] = x.flip(0)[:, :, yl:yh, xl:xh] + else: + x_flipped = x.flip(0).mul_(1. - lam) + x.mul_(lam).add_(x_flipped) + return lam + + def __call__(self, x, target): + assert len(x) % 2 == 0, 'Batch size should be even when using this' + if self.mode == 'elem': + lam = self._mix_elem(x) + elif self.mode == 'pair': + lam = self._mix_pair(x) + else: + lam = self._mix_batch(x) + target = mixup_target(target, self.num_classes, lam, self.label_smoothing, x.device) + return x, target + + +class FastCollateMixup(Mixup): + """ Fast Collate w/ Mixup/Cutmix that applies different params to each element or whole batch + + A Mixup impl that's performed while collating the batches. + """ + + def _mix_elem_collate(self, output, batch, half=False): + batch_size = len(batch) + num_elem = batch_size // 2 if half else batch_size + assert len(output) == num_elem + lam_batch, use_cutmix = self._params_per_elem(num_elem) + for i in range(num_elem): + j = batch_size - i - 1 + lam = lam_batch[i] + mixed = batch[i][0] + if lam != 1.: + if use_cutmix[i]: + if not half: + mixed = mixed.copy() + (yl, yh, xl, xh), lam = cutmix_bbox_and_lam( + output.shape, lam, ratio_minmax=self.cutmix_minmax, correct_lam=self.correct_lam) + mixed[:, yl:yh, xl:xh] = batch[j][0][:, yl:yh, xl:xh] + lam_batch[i] = lam + else: + mixed = mixed.astype(np.float32) * lam + batch[j][0].astype(np.float32) * (1 - lam) + np.rint(mixed, out=mixed) + output[i] += torch.from_numpy(mixed.astype(np.uint8)) + if half: + lam_batch = np.concatenate((lam_batch, np.ones(num_elem))) + return torch.tensor(lam_batch).unsqueeze(1) + + def _mix_pair_collate(self, output, batch): + batch_size = len(batch) + lam_batch, use_cutmix = self._params_per_elem(batch_size // 2) + for i in range(batch_size // 2): + j = batch_size - i - 1 + lam = lam_batch[i] + mixed_i = batch[i][0] + mixed_j = batch[j][0] + assert 0 <= lam <= 1.0 + if lam < 1.: + if use_cutmix[i]: + (yl, yh, xl, xh), lam = cutmix_bbox_and_lam( + output.shape, lam, ratio_minmax=self.cutmix_minmax, correct_lam=self.correct_lam) + patch_i = mixed_i[:, yl:yh, xl:xh].copy() + mixed_i[:, yl:yh, xl:xh] = mixed_j[:, yl:yh, xl:xh] + mixed_j[:, yl:yh, xl:xh] = patch_i + lam_batch[i] = lam + else: + mixed_temp = mixed_i.astype(np.float32) * lam + mixed_j.astype(np.float32) * (1 - lam) + mixed_j = mixed_j.astype(np.float32) * lam + mixed_i.astype(np.float32) * (1 - lam) + mixed_i = mixed_temp + np.rint(mixed_j, out=mixed_j) + np.rint(mixed_i, out=mixed_i) + output[i] += torch.from_numpy(mixed_i.astype(np.uint8)) + output[j] += torch.from_numpy(mixed_j.astype(np.uint8)) + lam_batch = np.concatenate((lam_batch, lam_batch[::-1])) + return torch.tensor(lam_batch).unsqueeze(1) + + def _mix_batch_collate(self, output, batch): + batch_size = len(batch) + lam, use_cutmix = self._params_per_batch() + if use_cutmix: + (yl, yh, xl, xh), lam = cutmix_bbox_and_lam( + output.shape, lam, ratio_minmax=self.cutmix_minmax, correct_lam=self.correct_lam) + for i in range(batch_size): + j = batch_size - i - 1 + mixed = batch[i][0] + if lam != 1.: + if use_cutmix: + mixed = mixed.copy() # don't want to modify the original while iterating + mixed[:, yl:yh, xl:xh] = batch[j][0][:, yl:yh, xl:xh] + else: + mixed = mixed.astype(np.float32) * lam + batch[j][0].astype(np.float32) * (1 - lam) + np.rint(mixed, out=mixed) + output[i] += torch.from_numpy(mixed.astype(np.uint8)) + return lam + + def __call__(self, batch, _=None): + batch_size = len(batch) + assert batch_size % 2 == 0, 'Batch size should be even when using this' + half = 'half' in self.mode + if half: + batch_size //= 2 + output = torch.zeros((batch_size, *batch[0][0].shape), dtype=torch.uint8) + if self.mode == 'elem' or self.mode == 'half': + lam = self._mix_elem_collate(output, batch, half=half) + elif self.mode == 'pair': + lam = self._mix_pair_collate(output, batch) + else: + lam = self._mix_batch_collate(output, batch) + target = torch.tensor([b[1] for b in batch], dtype=torch.int64) + target = mixup_target(target, self.num_classes, lam, self.label_smoothing, device='cpu') + target = target[:batch_size] + return output, target diff --git a/fourm/utils/timm/model.py b/fourm/utils/timm/model.py new file mode 100644 index 0000000000000000000000000000000000000000..454d2a45d0576cc24d2d52bf16e7be444b585feb --- /dev/null +++ b/fourm/utils/timm/model.py @@ -0,0 +1,277 @@ +# -------------------------------------------------------- +# Based on the timm code base +# https://github.com/huggingface/pytorch-image-models +# -------------------------------------------------------- +""" Model / state_dict utils + +Hacked together by / Copyright 2020 Ross Wightman +""" +import fnmatch + +import torch +from torchvision.ops.misc import FrozenBatchNorm2d + +from .model_ema import ModelEma + + +def unwrap_model(model): + if isinstance(model, ModelEma): + return unwrap_model(model.ema) + else: + return model.module if hasattr(model, 'module') else model + + +def get_state_dict(model, unwrap_fn=unwrap_model): + return unwrap_fn(model).state_dict() + + +def avg_sq_ch_mean(model, input, output): + """ calculate average channel square mean of output activations + """ + return torch.mean(output.mean(axis=[0, 2, 3]) ** 2).item() + + +def avg_ch_var(model, input, output): + """ calculate average channel variance of output activations + """ + return torch.mean(output.var(axis=[0, 2, 3])).item() + + +def avg_ch_var_residual(model, input, output): + """ calculate average channel variance of output activations + """ + return torch.mean(output.var(axis=[0, 2, 3])).item() + + +class ActivationStatsHook: + """Iterates through each of `model`'s modules and matches modules using unix pattern + matching based on `hook_fn_locs` and registers `hook_fn` to the module if there is + a match. + + Arguments: + model (nn.Module): model from which we will extract the activation stats + hook_fn_locs (List[str]): List of `hook_fn` locations based on Unix type string + matching with the name of model's modules. + hook_fns (List[Callable]): List of hook functions to be registered at every + module in `layer_names`. + + Inspiration from https://docs.fast.ai/callback.hook.html. + + Refer to https://gist.github.com/amaarora/6e56942fcb46e67ba203f3009b30d950 for an example + on how to plot Signal Propogation Plots using `ActivationStatsHook`. + """ + + def __init__(self, model, hook_fn_locs, hook_fns): + self.model = model + self.hook_fn_locs = hook_fn_locs + self.hook_fns = hook_fns + if len(hook_fn_locs) != len(hook_fns): + raise ValueError("Please provide `hook_fns` for each `hook_fn_locs`, \ + their lengths are different.") + self.stats = dict((hook_fn.__name__, []) for hook_fn in hook_fns) + for hook_fn_loc, hook_fn in zip(hook_fn_locs, hook_fns): + self.register_hook(hook_fn_loc, hook_fn) + + def _create_hook(self, hook_fn): + def append_activation_stats(module, input, output): + out = hook_fn(module, input, output) + self.stats[hook_fn.__name__].append(out) + + return append_activation_stats + + def register_hook(self, hook_fn_loc, hook_fn): + for name, module in self.model.named_modules(): + if not fnmatch.fnmatch(name, hook_fn_loc): + continue + module.register_forward_hook(self._create_hook(hook_fn)) + + +def extract_spp_stats( + model, + hook_fn_locs, + hook_fns, + input_shape=[8, 3, 224, 224]): + """Extract average square channel mean and variance of activations during + forward pass to plot Signal Propogation Plots (SPP). + + Paper: https://arxiv.org/abs/2101.08692 + + Example Usage: https://gist.github.com/amaarora/6e56942fcb46e67ba203f3009b30d950 + """ + x = torch.normal(0., 1., input_shape) + hook = ActivationStatsHook(model, hook_fn_locs=hook_fn_locs, hook_fns=hook_fns) + _ = model(x) + return hook.stats + + +def freeze_batch_norm_2d(module): + """ + Converts all `BatchNorm2d` and `SyncBatchNorm` layers of provided module into `FrozenBatchNorm2d`. If `module` is + itself an instance of either `BatchNorm2d` or `SyncBatchNorm`, it is converted into `FrozenBatchNorm2d` and + returned. Otherwise, the module is walked recursively and submodules are converted in place. + + Args: + module (torch.nn.Module): Any PyTorch module. + + Returns: + torch.nn.Module: Resulting module + + Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762 + """ + res = module + if isinstance(module, (torch.nn.modules.batchnorm.BatchNorm2d, torch.nn.modules.batchnorm.SyncBatchNorm)): + res = FrozenBatchNorm2d(module.num_features) + res.num_features = module.num_features + res.affine = module.affine + if module.affine: + res.weight.data = module.weight.data.clone().detach() + res.bias.data = module.bias.data.clone().detach() + res.running_mean.data = module.running_mean.data + res.running_var.data = module.running_var.data + res.eps = module.eps + else: + for name, child in module.named_children(): + new_child = freeze_batch_norm_2d(child) + if new_child is not child: + res.add_module(name, new_child) + return res + + +def unfreeze_batch_norm_2d(module): + """ + Converts all `FrozenBatchNorm2d` layers of provided module into `BatchNorm2d`. If `module` is itself and instance + of `FrozenBatchNorm2d`, it is converted into `BatchNorm2d` and returned. Otherwise, the module is walked + recursively and submodules are converted in place. + + Args: + module (torch.nn.Module): Any PyTorch module. + + Returns: + torch.nn.Module: Resulting module + + Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762 + """ + res = module + if isinstance(module, FrozenBatchNorm2d): + res = torch.nn.BatchNorm2d(module.num_features) + if module.affine: + res.weight.data = module.weight.data.clone().detach() + res.bias.data = module.bias.data.clone().detach() + res.running_mean.data = module.running_mean.data + res.running_var.data = module.running_var.data + res.eps = module.eps + else: + for name, child in module.named_children(): + new_child = unfreeze_batch_norm_2d(child) + if new_child is not child: + res.add_module(name, new_child) + return res + + +def _freeze_unfreeze(root_module, submodules=[], include_bn_running_stats=True, mode='freeze'): + """ + Freeze or unfreeze parameters of the specified modules and those of all their hierarchical descendants. This is + done in place. + Args: + root_module (nn.Module, optional): Root module relative to which the `submodules` are referenced. + submodules (list[str]): List of modules for which the parameters will be (un)frozen. They are to be provided as + named modules relative to the root module (accessible via `root_module.named_modules()`). An empty list + means that the whole root module will be (un)frozen. Defaults to [] + include_bn_running_stats (bool): Whether to also (un)freeze the running statistics of batch norm 2d layers. + Defaults to `True`. + mode (bool): Whether to freeze ("freeze") or unfreeze ("unfreeze"). Defaults to `"freeze"`. + """ + assert mode in ["freeze", "unfreeze"], '`mode` must be one of "freeze" or "unfreeze"' + + if isinstance(root_module, (torch.nn.modules.batchnorm.BatchNorm2d, torch.nn.modules.batchnorm.SyncBatchNorm)): + # Raise assertion here because we can't convert it in place + raise AssertionError( + "You have provided a batch norm layer as the `root module`. Please use " + "`timm.utils.model.freeze_batch_norm_2d` or `timm.utils.model.unfreeze_batch_norm_2d` instead.") + + if isinstance(submodules, str): + submodules = [submodules] + + named_modules = submodules + submodules = [root_module.get_submodule(m) for m in submodules] + + if not len(submodules): + named_modules, submodules = list(zip(*root_module.named_children())) + + for n, m in zip(named_modules, submodules): + # (Un)freeze parameters + for p in m.parameters(): + p.requires_grad = False if mode == 'freeze' else True + if include_bn_running_stats: + # Helper to add submodule specified as a named_module + def _add_submodule(module, name, submodule): + split = name.rsplit('.', 1) + if len(split) > 1: + module.get_submodule(split[0]).add_module(split[1], submodule) + else: + module.add_module(name, submodule) + + # Freeze batch norm + if mode == 'freeze': + res = freeze_batch_norm_2d(m) + # It's possible that `m` is a type of BatchNorm in itself, in which case `unfreeze_batch_norm_2d` won't + # convert it in place, but will return the converted result. In this case `res` holds the converted + # result and we may try to re-assign the named module + if isinstance(m, (torch.nn.modules.batchnorm.BatchNorm2d, torch.nn.modules.batchnorm.SyncBatchNorm)): + _add_submodule(root_module, n, res) + # Unfreeze batch norm + else: + res = unfreeze_batch_norm_2d(m) + # Ditto. See note above in mode == 'freeze' branch + if isinstance(m, FrozenBatchNorm2d): + _add_submodule(root_module, n, res) + + +def freeze(root_module, submodules=[], include_bn_running_stats=True): + """ + Freeze parameters of the specified modules and those of all their hierarchical descendants. This is done in place. + Args: + root_module (nn.Module): Root module relative to which `submodules` are referenced. + submodules (list[str]): List of modules for which the parameters will be frozen. They are to be provided as + named modules relative to the root module (accessible via `root_module.named_modules()`). An empty list + means that the whole root module will be frozen. Defaults to `[]`. + include_bn_running_stats (bool): Whether to also freeze the running statistics of `BatchNorm2d` and + `SyncBatchNorm` layers. These will be converted to `FrozenBatchNorm2d` in place. Hint: During fine tuning, + it's good practice to freeze batch norm stats. And note that these are different to the affine parameters + which are just normal PyTorch parameters. Defaults to `True`. + + Hint: If you want to freeze batch norm ONLY, use `timm.utils.model.freeze_batch_norm_2d`. + + Examples:: + + >>> model = timm.create_model('resnet18') + >>> # Freeze up to and including layer2 + >>> submodules = [n for n, _ in model.named_children()] + >>> print(submodules) + ['conv1', 'bn1', 'act1', 'maxpool', 'layer1', 'layer2', 'layer3', 'layer4', 'global_pool', 'fc'] + >>> freeze(model, submodules[:submodules.index('layer2') + 1]) + >>> # Check for yourself that it works as expected + >>> print(model.layer2[0].conv1.weight.requires_grad) + False + >>> print(model.layer3[0].conv1.weight.requires_grad) + True + >>> # Unfreeze + >>> unfreeze(model) + """ + _freeze_unfreeze(root_module, submodules, include_bn_running_stats=include_bn_running_stats, mode="freeze") + + +def unfreeze(root_module, submodules=[], include_bn_running_stats=True): + """ + Unfreeze parameters of the specified modules and those of all their hierarchical descendants. This is done in place. + Args: + root_module (nn.Module): Root module relative to which `submodules` are referenced. + submodules (list[str]): List of submodules for which the parameters will be (un)frozen. They are to be provided + as named modules relative to the root module (accessible via `root_module.named_modules()`). An empty + list means that the whole root module will be unfrozen. Defaults to `[]`. + include_bn_running_stats (bool): Whether to also unfreeze the running statistics of `FrozenBatchNorm2d` layers. + These will be converted to `BatchNorm2d` in place. Defaults to `True`. + + See example in docstring for `freeze`. + """ + _freeze_unfreeze(root_module, submodules, include_bn_running_stats=include_bn_running_stats, mode="unfreeze") diff --git a/fourm/utils/timm/model_builder.py b/fourm/utils/timm/model_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..37342663a5823d82bc6f15897cafdc23bc0c0f08 --- /dev/null +++ b/fourm/utils/timm/model_builder.py @@ -0,0 +1,74 @@ +# -------------------------------------------------------- +# Based on the timm code base +# https://github.com/huggingface/pytorch-image-models +# -------------------------------------------------------- +from .registry import is_model_in_modules, model_entrypoint + + +def split_model_name(model_name): + model_split = model_name.split(':', 1) + if len(model_split) == 1: + return '', model_split[0] + else: + source_name, model_name = model_split + assert source_name in ('timm', 'hf_hub') + return source_name, model_name + + +def safe_model_name(model_name, remove_source=True): + def make_safe(name): + return ''.join(c if c.isalnum() else '_' for c in name).rstrip('_') + + if remove_source: + model_name = split_model_name(model_name)[-1] + return make_safe(model_name) + + +def create_model( + model_name, + pretrained=False, + checkpoint_path='', + scriptable=None, + exportable=None, + no_jit=None, + **kwargs): + """Create a model + + Args: + model_name (str): name of model to instantiate + pretrained (bool): load pretrained ImageNet-1k weights if true + checkpoint_path (str): path of checkpoint to load after model is initialized + scriptable (bool): set layer config so that model is jit scriptable (not working for all models yet) + exportable (bool): set layer config so that model is traceable / ONNX exportable (not fully impl/obeyed yet) + no_jit (bool): set layer config so that model doesn't utilize jit scripted layers (so far activations only) + + Keyword Args: + drop_rate (float): dropout rate for training (default: 0.0) + global_pool (str): global pool type (default: 'avg') + **: other kwargs are model specific + """ + source_name, model_name = split_model_name(model_name) + + # Only EfficientNet and MobileNetV3 models have support for batchnorm params or drop_connect_rate passed as args + is_efficientnet = is_model_in_modules(model_name, ['efficientnet', 'mobilenetv3']) + if not is_efficientnet: + kwargs.pop('bn_tf', None) + kwargs.pop('bn_momentum', None) + kwargs.pop('bn_eps', None) + + # handle backwards compat with drop_connect -> drop_path change + drop_connect_rate = kwargs.pop('drop_connect_rate', None) + if drop_connect_rate is not None and kwargs.get('drop_path_rate', None) is None: + print("WARNING: 'drop_connect' as an argument is deprecated, please use 'drop_path'." + " Setting drop_path to %f." % drop_connect_rate) + kwargs['drop_path_rate'] = drop_connect_rate + + # Parameters that aren't supported by all models or are intended to only override model defaults if set + # should default to None in command line args/cfg. Remove them if they are present and not set so that + # non-supporting models don't break and default args remain in effect. + kwargs = {k: v for k, v in kwargs.items()} + + create_fn = model_entrypoint(model_name) + model = create_fn(**kwargs) + + return model diff --git a/fourm/utils/timm/model_ema.py b/fourm/utils/timm/model_ema.py new file mode 100644 index 0000000000000000000000000000000000000000..509f764643ed68db7e947b1f29c8822c9857d785 --- /dev/null +++ b/fourm/utils/timm/model_ema.py @@ -0,0 +1,149 @@ +# -------------------------------------------------------- +# Based on the timm code base +# https://github.com/huggingface/pytorch-image-models +# -------------------------------------------------------- +""" Exponential Moving Average (EMA) of model updates + +Hacked together by / Copyright 2020 Ross Wightman +""" +from collections import OrderedDict +from copy import deepcopy + +import torch +import torch.nn as nn + + +class ModelEma: + """ Model Exponential Moving Average (DEPRECATED) + + Keep a moving average of everything in the model state_dict (parameters and buffers). + This version is deprecated, it does not work with scripted models. Will be removed eventually. + + This is intended to allow functionality like + https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage + + A smoothed version of the weights is necessary for some training schemes to perform well. + E.g. Google's hyper-params for training MNASNet, MobileNet-V3, EfficientNet, etc that use + RMSprop with a short 2.4-3 epoch decay period and slow LR decay rate of .96-.99 requires EMA + smoothing of weights to match results. Pay attention to the decay constant you are using + relative to your update count per epoch. + + To keep EMA from using GPU resources, set device='cpu'. This will save a bit of memory but + disable validation of the EMA weights. Validation will have to be done manually in a separate + process, or after the training stops converging. + + This class is sensitive where it is initialized in the sequence of model init, + GPU assignment and distributed training wrappers. + """ + + def __init__(self, model, decay=0.9999, device='', resume=''): + # make a copy of the model for accumulating moving average of weights + self.ema = deepcopy(model) + self.ema.eval() + self.decay = decay + self.device = device # perform ema on different device from model if set + if device: + self.ema.to(device=device) + self.ema_has_module = hasattr(self.ema, 'module') + if resume: + self._load_checkpoint(resume) + for p in self.ema.parameters(): + p.requires_grad_(False) + + def _load_checkpoint(self, checkpoint_path): + checkpoint = torch.load(checkpoint_path, map_location='cpu') + assert isinstance(checkpoint, dict) + if 'state_dict_ema' in checkpoint: + new_state_dict = OrderedDict() + for k, v in checkpoint['state_dict_ema'].items(): + # ema model may have been wrapped by DataParallel, and need module prefix + if self.ema_has_module: + name = 'module.' + k if not k.startswith('module') else k + else: + name = k + new_state_dict[name] = v + self.ema.load_state_dict(new_state_dict) + print("Loaded state_dict_ema") + else: + print("Failed to find state_dict_ema, starting from loaded model weights") + + def update(self, model): + # correct a mismatch in state dict keys + needs_module = hasattr(model, 'module') and not self.ema_has_module + with torch.no_grad(): + msd = model.state_dict() + for k, ema_v in self.ema.state_dict().items(): + if needs_module: + k = 'module.' + k + model_v = msd[k].detach() + if self.device: + model_v = model_v.to(device=self.device) + ema_v.copy_(ema_v * self.decay + (1. - self.decay) * model_v) + + +class ModelEmaV2(nn.Module): + """ Model Exponential Moving Average V2 + + Keep a moving average of everything in the model state_dict (parameters and buffers). + V2 of this module is simpler, it does not match params/buffers based on name but simply + iterates in order. It works with torchscript (JIT of full model). + + This is intended to allow functionality like + https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage + + A smoothed version of the weights is necessary for some training schemes to perform well. + E.g. Google's hyper-params for training MNASNet, MobileNet-V3, EfficientNet, etc that use + RMSprop with a short 2.4-3 epoch decay period and slow LR decay rate of .96-.99 requires EMA + smoothing of weights to match results. Pay attention to the decay constant you are using + relative to your update count per epoch. + + To keep EMA from using GPU resources, set device='cpu'. This will save a bit of memory but + disable validation of the EMA weights. Validation will have to be done manually in a separate + process, or after the training stops converging. + + This class is sensitive where it is initialized in the sequence of model init, + GPU assignment and distributed training wrappers. + """ + + def __init__(self, model, decay=0.9999, device=None, resume=''): + super(ModelEmaV2, self).__init__() + # make a copy of the model for accumulating moving average of weights + self.module = deepcopy(model) + self.module.eval() + self.decay = decay + self.device = device # perform ema on different device from model if set + if device: + self.module.to(device=device) + self.ema_has_module = hasattr(self.module, 'module') + if resume: + self._load_checkpoint(resume) + + def _update(self, model, update_fn): + with torch.no_grad(): + for ema_v, model_v in zip(self.module.state_dict().values(), model.state_dict().values()): + if self.device: + model_v = model_v.to(device=self.device) + ema_v.copy_(update_fn(ema_v, model_v)) + + def update(self, model): + self._update(model, update_fn=lambda e, m: self.decay * e + (1. - self.decay) * m) + + def set(self, model): + self._update(model, update_fn=lambda e, m: m) + + def _load_checkpoint(self, checkpoint_path): + checkpoint = torch.load(checkpoint_path, map_location='cpu') + assert isinstance(checkpoint, dict) + if 'state_dict_ema' in checkpoint: + new_state_dict = OrderedDict() + for k, v in checkpoint['state_dict_ema'].items(): + # ema model may have been wrapped by DataParallel, and need module prefix + if self.ema_has_module: + name = 'module.' + k if not k.startswith('module') else k + else: + name = k + new_state_dict[name] = v + self.module.load_state_dict(new_state_dict) + print("Loaded state_dict_ema") + else: + print("Failed to find state_dict_ema, starting from loaded model weights") \ No newline at end of file diff --git a/fourm/utils/timm/random_erasing.py b/fourm/utils/timm/random_erasing.py new file mode 100644 index 0000000000000000000000000000000000000000..41670dc3b396dc6b16dc3240a1d033dc7d925cc2 --- /dev/null +++ b/fourm/utils/timm/random_erasing.py @@ -0,0 +1,102 @@ +# -------------------------------------------------------- +# Based on the timm code base +# https://github.com/huggingface/pytorch-image-models +# -------------------------------------------------------- +""" Random Erasing (Cutout) + +Originally inspired by impl at https://github.com/zhunzhong07/Random-Erasing, Apache 2.0 +Copyright Zhun Zhong & Liang Zheng + +Hacked together by / Copyright 2020 Ross Wightman +""" +import math +import random + +import torch + + +def _get_pixels(per_pixel, rand_color, patch_size, dtype=torch.float32, device='cuda'): + # NOTE I've seen CUDA illegal memory access errors being caused by the normal_() + # paths, flip the order so normal is run on CPU if this becomes a problem + # Issue has been fixed in master https://github.com/pytorch/pytorch/issues/19508 + if per_pixel: + return torch.empty(patch_size, dtype=dtype, device=device).normal_() + elif rand_color: + return torch.empty((patch_size[0], 1, 1), dtype=dtype, device=device).normal_() + else: + return torch.zeros((patch_size[0], 1, 1), dtype=dtype, device=device) + + +class RandomErasing: + """ Randomly selects a rectangle region in an image and erases its pixels. + 'Random Erasing Data Augmentation' by Zhong et al. + See https://arxiv.org/pdf/1708.04896.pdf + + This variant of RandomErasing is intended to be applied to either a batch + or single image tensor after it has been normalized by dataset mean and std. + Args: + probability: Probability that the Random Erasing operation will be performed. + min_area: Minimum percentage of erased area wrt input image area. + max_area: Maximum percentage of erased area wrt input image area. + min_aspect: Minimum aspect ratio of erased area. + mode: pixel color mode, one of 'const', 'rand', or 'pixel' + 'const' - erase block is constant color of 0 for all channels + 'rand' - erase block is same per-channel random (normal) color + 'pixel' - erase block is per-pixel random (normal) color + max_count: maximum number of erasing blocks per image, area per box is scaled by count. + per-image count is randomly chosen between 1 and this value. + """ + + def __init__( + self, + probability=0.5, min_area=0.02, max_area=1 / 3, min_aspect=0.3, max_aspect=None, + mode='const', min_count=1, max_count=None, num_splits=0, device='cuda'): + self.probability = probability + self.min_area = min_area + self.max_area = max_area + max_aspect = max_aspect or 1 / min_aspect + self.log_aspect_ratio = (math.log(min_aspect), math.log(max_aspect)) + self.min_count = min_count + self.max_count = max_count or min_count + self.num_splits = num_splits + mode = mode.lower() + self.rand_color = False + self.per_pixel = False + if mode == 'rand': + self.rand_color = True # per block random normal + elif mode == 'pixel': + self.per_pixel = True # per pixel random normal + else: + assert not mode or mode == 'const' + self.device = device + + def _erase(self, img, chan, img_h, img_w, dtype): + if random.random() > self.probability: + return + area = img_h * img_w + count = self.min_count if self.min_count == self.max_count else \ + random.randint(self.min_count, self.max_count) + for _ in range(count): + for attempt in range(10): + target_area = random.uniform(self.min_area, self.max_area) * area / count + aspect_ratio = math.exp(random.uniform(*self.log_aspect_ratio)) + h = int(round(math.sqrt(target_area * aspect_ratio))) + w = int(round(math.sqrt(target_area / aspect_ratio))) + if w < img_w and h < img_h: + top = random.randint(0, img_h - h) + left = random.randint(0, img_w - w) + img[:, top:top + h, left:left + w] = _get_pixels( + self.per_pixel, self.rand_color, (chan, h, w), + dtype=dtype, device=self.device) + break + + def __call__(self, input): + if len(input.size()) == 3: + self._erase(input, *input.size(), input.dtype) + else: + batch_size, chan, img_h, img_w = input.size() + # skip first slice of batch if num_splits is set (for clean portion of samples) + batch_start = batch_size // self.num_splits if self.num_splits > 1 else 0 + for i in range(batch_start, batch_size): + self._erase(input[i], chan, img_h, img_w, input.dtype) + return input diff --git a/fourm/utils/timm/registry.py b/fourm/utils/timm/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..2532fcde57ab7d74392c3c7593173f0ed8bc5fc4 --- /dev/null +++ b/fourm/utils/timm/registry.py @@ -0,0 +1,153 @@ +# -------------------------------------------------------- +# Based on the timm code base +# https://github.com/huggingface/pytorch-image-models +# -------------------------------------------------------- +""" Model Registry +Hacked together by / Copyright 2020 Ross Wightman +""" + +import fnmatch +import re +import sys +from collections import defaultdict +from copy import deepcopy + +__all__ = ['list_models', 'is_model', 'model_entrypoint', 'list_modules', 'is_model_in_modules', + 'is_model_default_key', 'has_model_default_key', 'get_model_default_value', 'is_model_pretrained'] + +_module_to_models = defaultdict(set) # dict of sets to check membership of model in module +_model_to_module = {} # mapping of model names to module names +_model_entrypoints = {} # mapping of model names to entrypoint fns +_model_has_pretrained = set() # set of model names that have pretrained weight url present +_model_default_cfgs = dict() # central repo for model default_cfgs + + +def register_model(fn): + # lookup containing module + mod = sys.modules[fn.__module__] + module_name_split = fn.__module__.split('.') + module_name = module_name_split[-1] if len(module_name_split) else '' + + # add model to __all__ in module + model_name = fn.__name__ + if hasattr(mod, '__all__'): + mod.__all__.append(model_name) + else: + mod.__all__ = [model_name] + + # add entries to registry dict/sets + _model_entrypoints[model_name] = fn + _model_to_module[model_name] = module_name + _module_to_models[module_name].add(model_name) + has_pretrained = False # check if model has a pretrained url to allow filtering on this + if hasattr(mod, 'default_cfgs') and model_name in mod.default_cfgs: + # this will catch all models that have entrypoint matching cfg key, but miss any aliasing + # entrypoints or non-matching combos + has_pretrained = 'url' in mod.default_cfgs[model_name] and 'http' in mod.default_cfgs[model_name]['url'] + _model_default_cfgs[model_name] = deepcopy(mod.default_cfgs[model_name]) + if has_pretrained: + _model_has_pretrained.add(model_name) + return fn + + +def _natural_key(string_): + return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_.lower())] + + +def list_models(filter='', module='', pretrained=False, exclude_filters='', name_matches_cfg=False): + """ Return list of available model names, sorted alphabetically + + Args: + filter (str) - Wildcard filter string that works with fnmatch + module (str) - Limit model selection to a specific sub-module (ie 'gen_efficientnet') + pretrained (bool) - Include only models with pretrained weights if True + exclude_filters (str or list[str]) - Wildcard filters to exclude models after including them with filter + name_matches_cfg (bool) - Include only models w/ model_name matching default_cfg name (excludes some aliases) + + Example: + model_list('gluon_resnet*') -- returns all models starting with 'gluon_resnet' + model_list('*resnext*, 'resnet') -- returns all models with 'resnext' in 'resnet' module + """ + if module: + all_models = list(_module_to_models[module]) + else: + all_models = _model_entrypoints.keys() + if filter: + models = [] + include_filters = filter if isinstance(filter, (tuple, list)) else [filter] + for f in include_filters: + include_models = fnmatch.filter(all_models, f) # include these models + if len(include_models): + models = set(models).union(include_models) + else: + models = all_models + if exclude_filters: + if not isinstance(exclude_filters, (tuple, list)): + exclude_filters = [exclude_filters] + for xf in exclude_filters: + exclude_models = fnmatch.filter(models, xf) # exclude these models + if len(exclude_models): + models = set(models).difference(exclude_models) + if pretrained: + models = _model_has_pretrained.intersection(models) + if name_matches_cfg: + models = set(_model_default_cfgs).intersection(models) + return list(sorted(models, key=_natural_key)) + + +def is_model(model_name): + """ Check if a model name exists + """ + return model_name in _model_entrypoints + + +def model_entrypoint(model_name): + """Fetch a model entrypoint for specified model name + """ + return _model_entrypoints[model_name] + + +def list_modules(): + """ Return list of module names that contain models / model entrypoints + """ + modules = _module_to_models.keys() + return list(sorted(modules)) + + +def is_model_in_modules(model_name, module_names): + """Check if a model exists within a subset of modules + Args: + model_name (str) - name of model to check + module_names (tuple, list, set) - names of modules to search in + """ + assert isinstance(module_names, (tuple, list, set)) + return any(model_name in _module_to_models[n] for n in module_names) + + +def has_model_default_key(model_name, cfg_key): + """ Query model default_cfgs for existence of a specific key. + """ + if model_name in _model_default_cfgs and cfg_key in _model_default_cfgs[model_name]: + return True + return False + + +def is_model_default_key(model_name, cfg_key): + """ Return truthy value for specified model default_cfg key, False if does not exist. + """ + if model_name in _model_default_cfgs and _model_default_cfgs[model_name].get(cfg_key, False): + return True + return False + + +def get_model_default_value(model_name, cfg_key): + """ Get a specific model default_cfg value by key. None if it doesn't exist. + """ + if model_name in _model_default_cfgs: + return _model_default_cfgs[model_name].get(cfg_key, None) + else: + return None + + +def is_model_pretrained(model_name): + return model_name in _model_has_pretrained diff --git a/fourm/utils/timm/transforms.py b/fourm/utils/timm/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..063d83633e6bbdb5df43d5a1841ece92eea58961 --- /dev/null +++ b/fourm/utils/timm/transforms.py @@ -0,0 +1,162 @@ +# -------------------------------------------------------- +# Based on the timm code base +# https://github.com/huggingface/pytorch-image-models +# -------------------------------------------------------- + +import math +import random +import warnings + +import numpy as np +import torch +import torchvision.transforms.functional as F +from PIL import Image + + +class ToNumpy: + + def __call__(self, pil_img): + np_img = np.array(pil_img, dtype=np.uint8) + if np_img.ndim < 3: + np_img = np.expand_dims(np_img, axis=-1) + np_img = np.rollaxis(np_img, 2) # HWC to CHW + return np_img + + +class ToTensor: + + def __init__(self, dtype=torch.float32): + self.dtype = dtype + + def __call__(self, pil_img): + np_img = np.array(pil_img, dtype=np.uint8) + if np_img.ndim < 3: + np_img = np.expand_dims(np_img, axis=-1) + np_img = np.rollaxis(np_img, 2) # HWC to CHW + return torch.from_numpy(np_img).to(dtype=self.dtype) + + +_pil_interpolation_to_str = { + Image.NEAREST: 'PIL.Image.NEAREST', + Image.BILINEAR: 'PIL.Image.BILINEAR', + Image.BICUBIC: 'PIL.Image.BICUBIC', + Image.LANCZOS: 'PIL.Image.LANCZOS', + Image.HAMMING: 'PIL.Image.HAMMING', + Image.BOX: 'PIL.Image.BOX', +} + + +def _pil_interp(method): + if method == 'bicubic': + return Image.BICUBIC + elif method == 'lanczos': + return Image.LANCZOS + elif method == 'hamming': + return Image.HAMMING + else: + # default bilinear, do we want to allow nearest? + return Image.BILINEAR + + +_RANDOM_INTERPOLATION = (Image.BILINEAR, Image.BICUBIC) + + +class RandomResizedCropAndInterpolation: + """Crop the given PIL Image to random size and aspect ratio with random interpolation. + + A crop of random size (default: of 0.08 to 1.0) of the original size and a random + aspect ratio (default: of 3/4 to 4/3) of the original aspect ratio is made. This crop + is finally resized to given size. + This is popularly used to train the Inception networks. + + Args: + size: expected output size of each edge + scale: range of size of the origin size cropped + ratio: range of aspect ratio of the origin aspect ratio cropped + interpolation: Default: PIL.Image.BILINEAR + """ + + def __init__(self, size, scale=(0.08, 1.0), ratio=(3. / 4., 4. / 3.), + interpolation='bilinear'): + if isinstance(size, (list, tuple)): + self.size = tuple(size) + else: + self.size = (size, size) + if (scale[0] > scale[1]) or (ratio[0] > ratio[1]): + warnings.warn("range should be of kind (min, max)") + + if interpolation == 'random': + self.interpolation = _RANDOM_INTERPOLATION + else: + self.interpolation = _pil_interp(interpolation) + self.scale = scale + self.ratio = ratio + + @staticmethod + def get_params(img, scale, ratio): + """Get parameters for ``crop`` for a random sized crop. + + Args: + img (PIL Image): Image to be cropped. + scale (tuple): range of size of the origin size cropped + ratio (tuple): range of aspect ratio of the origin aspect ratio cropped + + Returns: + tuple: params (i, j, h, w) to be passed to ``crop`` for a random + sized crop. + """ + area = img.size[0] * img.size[1] + + for attempt in range(10): + target_area = random.uniform(*scale) * area + log_ratio = (math.log(ratio[0]), math.log(ratio[1])) + aspect_ratio = math.exp(random.uniform(*log_ratio)) + + w = int(round(math.sqrt(target_area * aspect_ratio))) + h = int(round(math.sqrt(target_area / aspect_ratio))) + + if w <= img.size[0] and h <= img.size[1]: + i = random.randint(0, img.size[1] - h) + j = random.randint(0, img.size[0] - w) + return i, j, h, w + + # Fallback to central crop + in_ratio = img.size[0] / img.size[1] + if in_ratio < min(ratio): + w = img.size[0] + h = int(round(w / min(ratio))) + elif in_ratio > max(ratio): + h = img.size[1] + w = int(round(h * max(ratio))) + else: # whole image + w = img.size[0] + h = img.size[1] + i = (img.size[1] - h) // 2 + j = (img.size[0] - w) // 2 + return i, j, h, w + + def __call__(self, img): + """ + Args: + img (PIL Image): Image to be cropped and resized. + + Returns: + PIL Image: Randomly cropped and resized image. + """ + i, j, h, w = self.get_params(img, self.scale, self.ratio) + if isinstance(self.interpolation, (tuple, list)): + interpolation = random.choice(self.interpolation) + else: + interpolation = self.interpolation + return F.resized_crop(img, i, j, h, w, self.size, interpolation) + + def __repr__(self): + if isinstance(self.interpolation, (tuple, list)): + interpolate_str = ' '.join([_pil_interpolation_to_str[x] for x in self.interpolation]) + else: + interpolate_str = _pil_interpolation_to_str[self.interpolation] + format_string = self.__class__.__name__ + '(size={0}'.format(self.size) + format_string += ', scale={0}'.format(tuple(round(s, 4) for s in self.scale)) + format_string += ', ratio={0}'.format(tuple(round(r, 4) for r in self.ratio)) + format_string += ', interpolation={0})'.format(interpolate_str) + return format_string diff --git a/fourm/utils/timm/transforms_factory.py b/fourm/utils/timm/transforms_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..1eb422d01db6599e69b45ba8643ef86bacb82439 --- /dev/null +++ b/fourm/utils/timm/transforms_factory.py @@ -0,0 +1,236 @@ +# -------------------------------------------------------- +# Based on the timm code base +# https://github.com/huggingface/pytorch-image-models +# -------------------------------------------------------- +""" Transforms Factory +Factory methods for building image transforms for use with TIMM (PyTorch Image Models) + +Hacked together by / Copyright 2020 Ross Wightman +""" +import math + +import torch +from torchvision import transforms + +from .auto_augment import (augment_and_mix_transform, auto_augment_transform, + rand_augment_transform) +from ..data_constants import (DEFAULT_CROP_PCT, IMAGENET_DEFAULT_MEAN, + IMAGENET_DEFAULT_STD) +from .random_erasing import RandomErasing +from .transforms import RandomResizedCropAndInterpolation, ToNumpy, _pil_interp + + +def transforms_noaug_train( + img_size=224, + interpolation='bilinear', + use_prefetcher=False, + mean=IMAGENET_DEFAULT_MEAN, + std=IMAGENET_DEFAULT_STD, +): + if interpolation == 'random': + # random interpolation not supported with no-aug + interpolation = 'bilinear' + tfl = [ + transforms.Resize(img_size, _pil_interp(interpolation)), + transforms.CenterCrop(img_size) + ] + if use_prefetcher: + # prefetcher and collate will handle tensor conversion and norm + tfl += [ToNumpy()] + else: + tfl += [ + transforms.ToTensor(), + transforms.Normalize( + mean=torch.tensor(mean), + std=torch.tensor(std)) + ] + return transforms.Compose(tfl) + + +def transforms_imagenet_train( + img_size=224, + scale=None, + ratio=None, + hflip=0.5, + vflip=0., + color_jitter=0.4, + auto_augment=None, + interpolation='random', + use_prefetcher=False, + mean=IMAGENET_DEFAULT_MEAN, + std=IMAGENET_DEFAULT_STD, + re_prob=0., + re_mode='const', + re_count=1, + re_num_splits=0, + separate=False, +): + """ + If separate==True, the transforms are returned as a tuple of 3 separate transforms + for use in a mixing dataset that passes + * all data through the first (primary) transform, called the 'clean' data + * a portion of the data through the secondary transform + * normalizes and converts the branches above with the third, final transform + """ + scale = tuple(scale or (0.08, 1.0)) # default imagenet scale range + ratio = tuple(ratio or (3. / 4., 4. / 3.)) # default imagenet ratio range + primary_tfl = [ + RandomResizedCropAndInterpolation(img_size, scale=scale, ratio=ratio, interpolation=interpolation)] + if hflip > 0.: + primary_tfl += [transforms.RandomHorizontalFlip(p=hflip)] + if vflip > 0.: + primary_tfl += [transforms.RandomVerticalFlip(p=vflip)] + + secondary_tfl = [] + if auto_augment: + assert isinstance(auto_augment, str) + if isinstance(img_size, (tuple, list)): + img_size_min = min(img_size) + else: + img_size_min = img_size + aa_params = dict( + translate_const=int(img_size_min * 0.45), + img_mean=tuple([min(255, round(255 * x)) for x in mean]), + ) + if interpolation and interpolation != 'random': + aa_params['interpolation'] = _pil_interp(interpolation) + if auto_augment.startswith('rand'): + secondary_tfl += [rand_augment_transform(auto_augment, aa_params)] + elif auto_augment.startswith('augmix'): + aa_params['translate_pct'] = 0.3 + secondary_tfl += [augment_and_mix_transform(auto_augment, aa_params)] + else: + secondary_tfl += [auto_augment_transform(auto_augment, aa_params)] + elif color_jitter is not None: + # color jitter is enabled when not using AA + if isinstance(color_jitter, (list, tuple)): + # color jitter should be a 3-tuple/list if spec brightness/contrast/saturation + # or 4 if also augmenting hue + assert len(color_jitter) in (3, 4) + else: + # if it's a scalar, duplicate for brightness, contrast, and saturation, no hue + color_jitter = (float(color_jitter),) * 3 + secondary_tfl += [transforms.ColorJitter(*color_jitter)] + + final_tfl = [] + if use_prefetcher: + # prefetcher and collate will handle tensor conversion and norm + final_tfl += [ToNumpy()] + else: + final_tfl += [ + transforms.ToTensor(), + transforms.Normalize( + mean=torch.tensor(mean), + std=torch.tensor(std)) + ] + if re_prob > 0.: + final_tfl.append( + RandomErasing(re_prob, mode=re_mode, max_count=re_count, num_splits=re_num_splits, device='cpu')) + + if separate: + return transforms.Compose(primary_tfl), transforms.Compose(secondary_tfl), transforms.Compose(final_tfl) + else: + return transforms.Compose(primary_tfl + secondary_tfl + final_tfl) + + +def transforms_imagenet_eval( + img_size=224, + crop_pct=None, + interpolation='bilinear', + use_prefetcher=False, + mean=IMAGENET_DEFAULT_MEAN, + std=IMAGENET_DEFAULT_STD): + crop_pct = crop_pct or DEFAULT_CROP_PCT + + if isinstance(img_size, (tuple, list)): + assert len(img_size) == 2 + if img_size[-1] == img_size[-2]: + # fall-back to older behaviour so Resize scales to shortest edge if target is square + scale_size = int(math.floor(img_size[0] / crop_pct)) + else: + scale_size = tuple([int(x / crop_pct) for x in img_size]) + else: + scale_size = int(math.floor(img_size / crop_pct)) + + tfl = [ + transforms.Resize(scale_size, _pil_interp(interpolation)), + transforms.CenterCrop(img_size), + ] + if use_prefetcher: + # prefetcher and collate will handle tensor conversion and norm + tfl += [ToNumpy()] + else: + tfl += [ + transforms.ToTensor(), + transforms.Normalize( + mean=torch.tensor(mean), + std=torch.tensor(std)) + ] + + return transforms.Compose(tfl) + + +def create_transform( + input_size, + is_training=False, + use_prefetcher=False, + no_aug=False, + scale=None, + ratio=None, + hflip=0.5, + vflip=0., + color_jitter=0.4, + auto_augment=None, + interpolation='bilinear', + mean=IMAGENET_DEFAULT_MEAN, + std=IMAGENET_DEFAULT_STD, + re_prob=0., + re_mode='const', + re_count=1, + re_num_splits=0, + crop_pct=None, + tf_preprocessing=False, + separate=False): + if isinstance(input_size, (tuple, list)): + img_size = input_size[-2:] + else: + img_size = input_size + + + if is_training and no_aug: + assert not separate, "Cannot perform split augmentation with no_aug" + transform = transforms_noaug_train( + img_size, + interpolation=interpolation, + use_prefetcher=use_prefetcher, + mean=mean, + std=std) + elif is_training: + transform = transforms_imagenet_train( + img_size, + scale=scale, + ratio=ratio, + hflip=hflip, + vflip=vflip, + color_jitter=color_jitter, + auto_augment=auto_augment, + interpolation=interpolation, + use_prefetcher=use_prefetcher, + mean=mean, + std=std, + re_prob=re_prob, + re_mode=re_mode, + re_count=re_count, + re_num_splits=re_num_splits, + separate=separate) + else: + assert not separate, "Separate transforms not supported for validation preprocessing" + transform = transforms_imagenet_eval( + img_size, + interpolation=interpolation, + use_prefetcher=use_prefetcher, + mean=mean, + std=std, + crop_pct=crop_pct) + + return transform diff --git a/fourm/utils/tokenizer/__init__.py b/fourm/utils/tokenizer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..aeddeb9ffec07fa6f5fbd510d818e07703e4cad2 --- /dev/null +++ b/fourm/utils/tokenizer/__init__.py @@ -0,0 +1 @@ +from .text_tokenizer import * diff --git a/fourm/utils/tokenizer/object_classes.json b/fourm/utils/tokenizer/object_classes.json new file mode 100644 index 0000000000000000000000000000000000000000..aa54e06ee68b44e2db5f12ed266f95ef79f688db --- /dev/null +++ b/fourm/utils/tokenizer/object_classes.json @@ -0,0 +1 @@ +{"lvis": ["aerosol_can", "air_conditioner", "airplane", "alarm_clock", "alcohol", "alligator", "almond", "ambulance", "amplifier", "anklet", "antenna", "apple", "applesauce", "apricot", "apron", "aquarium", "arctic_(type_of_shoe)", "armband", "armchair", "armoire", "armor", "army_tank", "artichoke", "ashtray", "asparagus", "atomizer", "automatic_washer", "avocado", "award", "awning", "ax", "baboon", "baby_buggy", "backpack", "bagel", "bagpipe", "baguet", "bait", "ball", "ballet_skirt", "balloon", "bamboo", "banana", "Band_Aid", "bandage", "bandanna", "banjo", "banner", "barbell", "barge", "barrel", "barrette", "barrow", "baseball", "baseball_base", "baseball_bat", "baseball_cap", "baseball_glove", "basket", "basketball", "basketball_backboard", "bass_horn", "bat_(animal)", "bath_mat", "bath_towel", "bathrobe", "bathtub", "batter_(food)", "battery", "beachball", "bead", "bean_curd", "beanbag", "beanie", "bear", "bed", "bedpan", "bedspread", "beef_(food)", "beeper", "beer_bottle", "beer_can", "beetle", "bell", "bell_pepper", "belt", "belt_buckle", "bench", "beret", "bib", "Bible", "bicycle", "billboard", "binder", "binoculars", "bird", "birdbath", "birdcage", "birdfeeder", "birdhouse", "birthday_cake", "birthday_card", "black_sheep", "blackberry", "blackboard", "blanket", "blazer", "blender", "blimp", "blinder_(for_horses)", "blinker", "blouse", "blueberry", "boat", "bob", "bobbin", "bobby_pin", "boiled_egg", "bolo_tie", "bolt", "bonnet", "book", "bookcase", "booklet", "bookmark", "boom_microphone", "boot", "bottle", "bottle_cap", "bottle_opener", "bouquet", "bow-tie", "bow_(decorative_ribbons)", "bow_(weapon)", "bowl", "bowler_hat", "bowling_ball", "box", "boxing_glove", "bracelet", "brake_light", "brass_plaque", "brassiere", "bread", "bread-bin", "breechcloth", "bridal_gown", "briefcase", "broach", "broccoli", "broom", "brownie", "brussels_sprouts", "bubble_gum", "bucket", "bulldog", "bulldozer", "bullet_train", "bulletin_board", "bulletproof_vest", "bullhorn", "bun", "bunk_bed", "buoy", "burrito", "bus_(vehicle)", "business_card", "butter", "butterfly", "button", "cab_(taxi)", "cabana", "cabin_car", "cabinet", "cake", "calculator", "calendar", "calf", "camcorder", "camel", "camera", "camera_lens", "camper_(vehicle)", "can", "can_opener", "candle", "candle_holder", "candy_bar", "candy_cane", "canister", "canoe", "cantaloup", "canteen", "cap_(headwear)", "cape", "cappuccino", "car_(automobile)", "car_battery", "card", "cardigan", "cargo_ship", "carnation", "carrot", "cart", "carton", "cash_register", "casserole", "cassette", "cast", "cat", "cauliflower", "cayenne_(spice)", "CD_player", "celery", "cellular_telephone", "chain_mail", "chair", "chaise_longue", "chalice", "chandelier", "chap", "checkbook", "checkerboard", "cherry", "chessboard", "chicken_(animal)", "chickpea", "chili_(vegetable)", "chime", "chinaware", "chocolate_bar", "chocolate_cake", "chocolate_milk", "chocolate_mousse", "choker", "chopping_board", "chopstick", "Christmas_tree", "cider", "cigar_box", "cigarette", "cigarette_case", "cincture", "cistern", "clarinet", "clasp", "cleansing_agent", "cleat_(for_securing_rope)", "clementine", "clip", "clipboard", "clippers_(for_plants)", "cloak", "clock", "clock_tower", "clothes_hamper", "clothespin", "clutch_bag", "coaster", "coat", "coat_hanger", "coatrack", "cock", "cockroach", "cocoa_(beverage)", "coconut", "coffee_maker", "coffee_table", "coffeepot", "coil", "coin", "colander", "coleslaw", "coloring_material", "combination_lock", "comic_book", "compass", "computer_keyboard", "condiment", "cone", "control", "convertible_(automobile)", "cooker", "cookie", "cooking_utensil", "cooler_(for_food)", "cork_(bottle_plug)", "corkboard", "corkscrew", "cornbread", "cornet", "cornice", "cornmeal", "corset", "costume", "cougar", "cover", "coverall", "cow", "cowbell", "cowboy_hat", "crab_(animal)", "crabmeat", "cracker", "crape", "crate", "crawfish", "crayon", "cream_pitcher", "crescent_roll", "crib", "crisp_(potato_chip)", "crock_pot", "crossbar", "crouton", "crow", "crowbar", "crown", "crucifix", "cruise_ship", "crumb", "crutch", "cub_(animal)", "cube", "cucumber", "cufflink", "cup", "cupboard", "cupcake", "curling_iron", "curtain", "cushion", "cylinder", "cymbal", "dagger", "dalmatian", "dartboard", "date_(fruit)", "deadbolt", "deck_chair", "deer", "dental_floss", "desk", "detergent", "diaper", "diary", "die", "dinghy", "dining_table", "dirt_bike", "dish", "dish_antenna", "dishrag", "dishtowel", "dishwasher", "dishwasher_detergent", "dispenser", "diving_board", "Dixie_cup", "dog", "dog_collar", "doll", "dollar", "dollhouse", "dolphin", "domestic_ass", "doorknob", "doormat", "doughnut", "dove", "dragonfly", "drawer", "dress", "dress_hat", "dress_suit", "dresser", "drill", "drone", "dropper", "drum_(musical_instrument)", "drumstick", "duck", "duckling", "duct_tape", "duffel_bag", "dumbbell", "dumpster", "dustpan", "eagle", "earphone", "earplug", "earring", "easel", "eclair", "edible_corn", "eel", "egg", "egg_roll", "egg_yolk", "eggbeater", "eggplant", "electric_chair", "elephant", "elevator_car", "elk", "envelope", "eraser", "escargot", "eyepatch", "falcon", "fan", "faucet", "fedora", "ferret", "Ferris_wheel", "ferry", "fig_(fruit)", "fighter_jet", "figurine", "file_(tool)", "file_cabinet", "fire_alarm", "fire_engine", "fire_extinguisher", "fire_hose", "fireplace", "fireplug", "first-aid_kit", "fish", "fish_(food)", "fishbowl", "fishing_rod", "flag", "flagpole", "flamingo", "flannel", "flap", "flash", "flashlight", "fleece", "flip-flop_(sandal)", "flipper_(footwear)", "flower_arrangement", "flowerpot", "flute_glass", "foal", "folding_chair", "food_processor", "football_(American)", "football_helmet", "footstool", "fork", "forklift", "freight_car", "French_toast", "freshener", "frisbee", "frog", "fruit_juice", "frying_pan", "fudge", "fume_hood", "funnel", "futon", "gag", "gameboard", "garbage", "garbage_truck", "garden_hose", "gargle", "gargoyle", "garlic", "gasmask", "gazelle", "gelatin", "gemstone", "generator", "giant_panda", "gift_wrap", "ginger", "giraffe", "glass_(drink_container)", "globe", "glove", "goat", "goggles", "goldfish", "golf_club", "golfcart", "gondola_(boat)", "goose", "gorilla", "gourd", "grape", "grater", "gravestone", "gravy_boat", "green_bean", "green_onion", "griddle", "grill", "grits", "grizzly", "grocery_bag", "guitar", "gull", "gun", "hair_curler", "hair_dryer", "hairbrush", "hairnet", "hairpin", "halter_top", "ham", "hamburger", "hammer", "hammock", "hamper", "hamster", "hand_glass", "hand_towel", "handbag", "handcart", "handcuff", "handkerchief", "handle", "handsaw", "hardback_book", "harmonium", "hat", "hatbox", "headband", "headboard", "headlight", "headscarf", "headset", "headstall_(for_horses)", "heart", "heater", "helicopter", "helmet", "heron", "highchair", "hinge", "hippopotamus", "hockey_stick", "hog", "home_plate_(baseball)", "honey", "hook", "hookah", "horned_cow", "hornet", "horse", "horse_buggy", "horse_carriage", "hose", "hot-air_balloon", "hot_sauce", "hotplate", "hourglass", "houseboat", "hummingbird", "hummus", "ice_maker", "ice_pack", "ice_skate", "icecream", "identity_card", "igniter", "inhaler", "inkpad", "iPod", "iron_(for_clothing)", "ironing_board", "jacket", "jam", "jar", "jean", "jeep", "jelly_bean", "jersey", "jet_plane", "jewel", "jewelry", "joystick", "jumpsuit", "kayak", "keg", "kennel", "kettle", "key", "keycard", "kilt", "kimono", "kitchen_sink", "kitchen_table", "kite", "kitten", "kiwi_fruit", "knee_pad", "knife", "knitting_needle", "knob", "knocker_(on_a_door)", "koala", "lab_coat", "ladder", "ladle", "ladybug", "lamb-chop", "lamb_(animal)", "lamp", "lamppost", "lampshade", "lantern", "lanyard", "laptop_computer", "lasagna", "latch", "lawn_mower", "leather", "legging_(clothing)", "Lego", "legume", "lemon", "lemonade", "lettuce", "license_plate", "life_buoy", "life_jacket", "lightbulb", "lightning_rod", "lime", "limousine", "lion", "lip_balm", "liquor", "lizard", "locker", "log", "lollipop", "loveseat", "machine_gun", "magazine", "magnet", "mail_slot", "mailbox_(at_home)", "mallard", "mallet", "mammoth", "manatee", "mandarin_orange", "manger", "manhole", "map", "marker", "martini", "mascot", "mashed_potato", "masher", "mask", "mast", "mat_(gym_equipment)", "matchbox", "mattress", "measuring_cup", "measuring_stick", "meatball", "medicine", "melon", "microphone", "microscope", "microwave_oven", "milestone", "milk", "milk_can", "milkshake", "minivan", "mint_candy", "mirror", "mitten", "mixer_(kitchen_tool)", "money", "monitor_(computer_equipment) computer_monitor", "monkey", "mop", "motor", "motor_scooter", "motor_vehicle", "motorcycle", "mound_(baseball)", "mouse_(computer_equipment)", "mousepad", "muffin", "mug", "mushroom", "music_stool", "musical_instrument", "nailfile", "napkin", "neckerchief", "necklace", "necktie", "needle", "nest", "newspaper", "newsstand", "nightshirt", "nosebag_(for_animals)", "noseband_(for_animals)", "notebook", "notepad", "nut", "nutcracker", "oar", "octopus_(animal)", "octopus_(food)", "oil_lamp", "olive_oil", "omelet", "onion", "orange_(fruit)", "orange_juice", "ostrich", "ottoman", "oven", "overalls_(clothing)", "owl", "pacifier", "packet", "pad", "paddle", "padlock", "paintbrush", "painting", "pajamas", "palette", "pan_(for_cooking)", "pan_(metal_container)", "pancake", "pantyhose", "papaya", "paper_plate", "paper_towel", "paperback_book", "paperweight", "parachute", "parakeet", "parasail_(sports)", "parasol", "parchment", "parka", "parking_meter", "parrot", "passenger_car_(part_of_a_train)", "passenger_ship", "passport", "pastry", "patty_(food)", "pea_(food)", "peach", "peanut_butter", "pear", "peeler_(tool_for_fruit_and_vegetables)", "pegboard", "pelican", "pen", "pencil", "pencil_box", "pencil_sharpener", "pendulum", "penguin", "pennant", "penny_(coin)", "pepper", "pepper_mill", "perfume", "persimmon", "person", "pet", "pew_(church_bench)", "phonebook", "phonograph_record", "piano", "pickle", "pickup_truck", "pie", "pigeon", "piggy_bank", "pillow", "pin_(non_jewelry)", "pineapple", "pinecone", "ping-pong_ball", "pinwheel", "pipe", "pipe_bowl", "pirate_flag", "pistol", "pita_(bread)", "pitcher_(vessel_for_liquid)", "pitchfork", "pizza", "place_mat", "plastic_bag", "plate", "platter", "playpen", "pliers", "plow_(farm_equipment)", "plume", "pocket_watch", "pocketknife", "poker_(fire_stirring_tool)", "poker_chip", "polar_bear", "pole", "police_cruiser", "polo_shirt", "poncho", "pony", "pool_table", "pop_(soda)", "popsicle", "postbox_(public)", "postcard", "poster", "pot", "potato", "potholder", "pottery", "pouch", "power_shovel", "prawn", "pretzel", "printer", "projectile_(weapon)", "projector", "propeller", "prune", "pudding", "puffer_(fish)", "puffin", "pug-dog", "pumpkin", "puncher", "puppet", "puppy", "quesadilla", "quiche", "quilt", "rabbit", "race_car", "racket", "radar", "radiator", "radio_receiver", "radish", "raft", "rag_doll", "railcar_(part_of_a_train)", "raincoat", "ram_(animal)", "raspberry", "rat", "razorblade", "reamer_(juicer)", "rearview_mirror", "receipt", "recliner", "record_player", "reflector", "refrigerator", "remote_control", "rhinoceros", "rib_(food)", "rifle", "ring", "river_boat", "road_map", "robe", "rocking_chair", "rodent", "roller_skate", "Rollerblade", "rolling_pin", "root_beer", "router_(computer_equipment)", "rubber_band", "runner_(carpet)", "saddle_(on_an_animal)", "saddle_blanket", "saddlebag", "safety_pin", "sail", "salad", "salad_plate", "salami", "salmon_(fish)", "salmon_(food)", "salsa", "saltshaker", "sandal_(type_of_shoe)", "sandwich", "satchel", "saucepan", "saucer", "sausage", "sawhorse", "saxophone", "scale_(measuring_instrument)", "scarecrow", "scarf", "school_bus", "scissors", "scoreboard", "scraper", "screwdriver", "scrubbing_brush", "sculpture", "seabird", "seahorse", "seaplane", "seashell", "sewing_machine", "shaker", "shampoo", "shark", "sharpener", "Sharpie", "shaver_(electric)", "shaving_cream", "shawl", "shears", "sheep", "shepherd_dog", "sherbert", "shield", "shirt", "shoe", "shopping_bag", "shopping_cart", "short_pants", "shot_glass", "shoulder_bag", "shovel", "shower_cap", "shower_curtain", "shower_head", "shredder_(for_paper)", "signboard", "silo", "sink", "skateboard", "skewer", "ski", "ski_boot", "ski_parka", "ski_pole", "skirt", "skullcap", "sled", "sleeping_bag", "slide", "sling_(bandage)", "slipper_(footwear)", "smoothie", "snake", "snowboard", "snowman", "snowmobile", "soap", "soccer_ball", "sock", "sofa", "sofa_bed", "softball", "solar_array", "sombrero", "soup", "soup_bowl", "soupspoon", "sour_cream", "soya_milk", "space_shuttle", "sparkler_(fireworks)", "spatula", "speaker_(stero_equipment)", "spear", "spectacles", "spice_rack", "spider", "sponge", "spoon", "sportswear", "spotlight", "squid_(food)", "squirrel", "stagecoach", "stapler_(stapling_machine)", "starfish", "statue_(sculpture)", "steak_(food)", "steak_knife", "steering_wheel", "step_stool", "stepladder", "stereo_(sound_system)", "stew", "stirrer", "stirrup", "stool", "stop_sign", "stove", "strainer", "strap", "straw_(for_drinking)", "strawberry", "street_sign", "streetlight", "string_cheese", "stylus", "subwoofer", "sugar_bowl", "sugarcane_(plant)", "suit_(clothing)", "suitcase", "sunflower", "sunglasses", "sunhat", "surfboard", "sushi", "suspenders", "sweat_pants", "sweatband", "sweater", "sweatshirt", "sweet_potato", "swimsuit", "sword", "syringe", "Tabasco_sauce", "table", "table-tennis_table", "table_lamp", "tablecloth", "tachometer", "taco", "tag", "taillight", "tambourine", "tank_(storage_vessel)", "tank_top_(clothing)", "tape_(sticky_cloth_or_paper)", "tape_measure", "tapestry", "tarp", "tartan", "tassel", "tea_bag", "teacup", "teakettle", "teapot", "teddy_bear", "telephone", "telephone_booth", "telephone_pole", "telephoto_lens", "television_camera", "television_set", "tennis_ball", "tennis_racket", "tequila", "thermometer", "thermos_bottle", "thermostat", "thimble", "thread", "thumbtack", "tiara", "tiger", "tights_(clothing)", "timer", "tinfoil", "tinsel", "tissue_paper", "toast_(food)", "toaster", "toaster_oven", "tobacco_pipe", "toilet", "toilet_tissue", "tomato", "tongs", "toolbox", "toothbrush", "toothpaste", "toothpick", "tortilla", "tote_bag", "tow_truck", "towel", "towel_rack", "toy", "tractor_(farm_equipment)", "traffic_light", "trailer_truck", "train_(railroad_vehicle)", "trampoline", "trash_can", "tray", "trench_coat", "triangle_(musical_instrument)", "tricycle", "tripod", "trophy_cup", "trousers", "truck", "truffle_(chocolate)", "trunk", "turban", "turkey_(food)", "turnip", "turtle", "turtleneck_(clothing)", "tux", "typewriter", "umbrella", "underdrawers", "underwear", "unicycle", "urinal", "urn", "vacuum_cleaner", "vase", "vat", "veil", "vending_machine", "vent", "vest", "videotape", "vinegar", "violin", "visor", "vodka", "volleyball", "vulture", "waffle", "waffle_iron", "wagon", "wagon_wheel", "walking_cane", "walking_stick", "wall_clock", "wall_socket", "wallet", "walrus", "wardrobe", "washbasin", "watch", "water_bottle", "water_cooler", "water_faucet", "water_gun", "water_heater", "water_jug", "water_scooter", "water_ski", "water_tower", "watering_can", "watermelon", "weathervane", "webcam", "wedding_cake", "wedding_ring", "wet_suit", "wheel", "wheelchair", "whipped_cream", "whistle", "wig", "wind_chime", "windmill", "window_box_(for_plants)", "windshield_wiper", "windsock", "wine_bottle", "wine_bucket", "wineglass", "wok", "wolf", "wooden_leg", "wooden_spoon", "wreath", "wrench", "wristband", "wristlet", "yacht", "yogurt", "yoke_(animal_equipment)", "zebra", "zucchini"], "coco": ["airplane", "apple", "backpack", "banana", "baseball bat", "baseball glove", "bear", "bed", "bench", "bicycle", "bird", "boat", "book", "bottle", "bowl", "broccoli", "bus", "cake", "car", "carrot", "cat", "cell phone", "chair", "clock", "couch", "cow", "cup", "dining table", "dog", "donut", "elephant", "fire hydrant", "fork", "frisbee", "giraffe", "hair drier", "handbag", "horse", "hot dog", "keyboard", "kite", "knife", "laptop", "microwave", "motorcycle", "mouse", "orange", "oven", "parking meter", "person", "pizza", "potted plant", "refrigerator", "remote", "sandwich", "scissors", "sheep", "sink", "skateboard", "skis", "snowboard", "spoon", "sports ball", "stop sign", "suitcase", "surfboard", "teddy bear", "tennis racket", "tie", "toaster", "toilet", "toothbrush", "traffic light", "train", "truck", "tv", "umbrella", "vase", "wine glass", "zebra"], "objects365": ["Air Conditioner", "Airplane", "Ambulance", "American Football", "Antelope", "Apple", "Asparagus", "Avocado", "Awning", "Backpack", "Ballon", "Banana", "Baozi", "Barbell", "Barrel/bucket", "Baseball", "Baseball Bat", "Baseball Glove", "Basket", "Basketball", "Bathtub", "Bear", "Bed", "Belt", "Bench", "Bicycle", "Billards", "Binoculars", "Blackboard/Whiteboard", "Blender", "Board Eraser", "Boat", "Book", "Boots", "Bottle", "Bow Tie", "Bowl/Basin", "Bracelet", "Bread", "Briefcase", "Broccoli", "Broom", "Brush", "Bus", "Butterfly", "Cabbage", "Cabinet/shelf", "Cake", "Calculator", "Camel", "Camera", "Candle", "Candy", "Canned", "Car", "Carpet", "Carriage", "Carrot", "Cat", "CD", "Cell Phone", "Cello", "Chainsaw", "Chair", "Cheese", "Cherry", "Chicken", "Chips", "Chopsticks", "Cigar/Cigarette ", "Cleaning Products", "Clock", "Coconut", "Coffee Machine", "Coffee Table", "Comb", "Computer Box", "Converter", "Cookies", "Corn", "Cosmetics", "Cosmetics Brush/Eyeliner Pencil", "Cosmetics Mirror", "Couch", "Cow", "Crab", "Crane", "Crosswalk Sign", "Cucumber", "Cue", "Cup", "Curling", "Cutting/chopping Board", "Cymbal", "Deer", "Desk", "Dessert", "Dining Table", "Dishwasher", "Dog", "Dolphin", "Donkey", "Donut", "Drum", "Duck", "Dumbbell", "Dumpling", "Durian", "earphone", "Egg", "Egg tart", "Eggplant", "Electric Drill", "Elephant", "Eraser", "Extension Cord", "Extractor", "Fan", "Faucet", "Fire Extinguisher", "Fire Hydrant", "Fire Truck", "Fishing Rod", "Flag", "Flask", "Flower", "Flute", "Folder", "Fork", "Formula 1 ", "French", "French Fries", "Frisbee", "Game board", "Garlic", "Gas stove", "Giraffe", "Glasses", "Globe", "Gloves", "Goldfish", "Golf Ball", "Golf Club", "Goose", "Grape", "Grapefruit", "Green beans", "Green Onion", "Green Vegetables", "Guitar", "Gun", "Hair Dryer", "Hamburger", "Hami melon", "Hammer", "Handbag/Satchel", "Hanger", "Hat", "Head Phone", "Heavy Truck", "Helicopter", "Helmet", "High Heels", "Hockey Stick", "Horse", "Hot air balloon", "Hot dog", "Hoverboard", "Hurdle", "Ice cream", "Induction Cooker", "Jellyfish", "Jug", "Kettle", "Key", "Keyboard", "Kite", "Kiwi fruit", "Knife", "Ladder", "Lamp", "Lantern", "Laptop", "Leather Shoes", "Lemon", "Lettuce", "Lifesaver", "Lighter", "Lion", "Lipstick", "Lobster", "Luggage", "Machinery Vehicle", "Mango", "Marker", "Mask", "Meatball", "Medal", "Megaphone", "Microphone", "Microwave", "Mirror", "Monitor/TV", "Monkey", "Mop", "Motorcycle", "Mouse", "Mushroom", "Napkin", "Necklace", "Nightstand", "Noddles", "Notepaper", "Nuts", "Okra", "Onion", "Orange/Tangerine", "Other Balls", "Other Fish", "Other Shoes", "Oven", "Oyster", "Paddle", "Paint Brush", "Papaya", "Parking meter", "Parrot", "Pasta", "Peach", "Pear", "Pen/Pencil", "Pencil Case", "Penguin", "Pepper", "Person", "Piano", "Pickup Truck", "Picture/Frame", "Pie", "Pig", "Pigeon", "Pillow", "Pineapple", "Pizza", "Plate", "Pliers", "Plum", "Poker Card", "Pomegranate", "Pot", "Potato", "Potted Plant", "Power outlet", "Printer", "Projector", "Pumpkin", "Rabbit", "Radiator", "Radish", "Recorder", "Red Cabbage", "Refrigerator", "Remote", "Rice", "Rice Cooker", "Rickshaw", "Ring", "Router/modem", "Sailboat", "Sandals", "Sandwich", "Sausage", "Saxophone", "Scale", "Scallop", "Scissors", "Scooter", "Screwdriver", "Seal", "Sheep", "Ship", "Shovel", "Showerhead", "Shrimp", "Side Table", "Sink", "Skateboard", "Skating and Skiing shoes", "Skiboard", "Slide", "Slippers", "Sneakers", "Snowboard", "Soap", "Soccer", "Speaker", "Speed Limit Sign", "Spoon", "Sports Car", "Spring Rolls", "Stapler", "Steak", "Stool", "Stop Sign", "Storage box", "Strawberry", "Street Lights", "Stroller", "Stuffed Toy", "Surfboard", "Surveillance Camera", "Sushi", "SUV", "Swan", "Swing", "Table Tennis ", "Table Tennis paddle", "Tablet", "Tape", "Tape Measure/ Ruler", "Target", "Teapot", "Telephone", "Tennis", "Tennis Racket", "Tent", "Tie", "Tissue", "Toaster", "Toilet", "Toilet Paper", "Toiletry", "Tomato", "Tong", "Toothbrush", "Towel", "Traffic cone", "Traffic Light", "Traffic Sign", "Train", "Trash bin Can", "Treadmill", "Tricycle", "Tripod", "Trolley", "Trombone", "Trophy", "Truck", "Trumpet", "Tuba", "Umbrella", "Urinal", "Van", "Vase", "Violin", "Volleyball", "Wallet/Purse", "Washing Machine/Drying Machine", "Watch", "Watermelon", "Wheelchair", "Wild Bird", "Wine Glass", "Yak", "Zebra"], "openimages": ["Accordion", "Adhesive tape", "Aircraft", "Airplane", "Alarm clock", "Alpaca", "Ambulance", "Animal", "Ant", "Antelope", "Apple", "Artichoke", "Asparagus", "Backpack", "Bagel", "Ball", "Balloon", "Banana", "Barge", "Barrel", "Baseball bat", "Baseball glove", "Bat", "Bathroom cabinet", "Bathtub", "Beaker", "Bear", "Bed", "Bee", "Beehive", "Beer", "Beetle", "Bell pepper", "Belt", "Bench", "Bench", "Bicycle", "Bicycle helmet", "Bicycle wheel", "Bidet", "Billboard", "Billiard table", "Binoculars", "Bird", "Blender", "Blue jay", "Boat", "Book", "Bookcase", "Boot", "Bottle", "Bow and arrow", "Bowl", "Box", "Boy", "Brassiere", "Bread", "Briefcase", "Broccoli", "Bronze sculpture", "Brown bear", "Building", "Bull", "Burrito", "Bus", "Bust", "Butterfly", "Cabbage", "Cabinetry", "Cake", "Cake stand", "Camel", "Camera", "Canary", "Candle", "Candy", "Cannon", "Canoe", "Car", "Carnivore", "Carrot", "Cart", "Castle", "Cat", "Caterpillar", "Cattle", "Ceiling fan", "Cello", "Centipede", "Chair", "Cheetah", "Chest of drawers", "Chicken", "Chopsticks", "Christmas tree", "Clock", "Coat", "Cocktail", "Coconut", "Coffee", "Coffee cup", "Coffee table", "Coffeemaker", "Coin", "Common fig", "Computer keyboard", "Computer monitor", "Convenience store", "Cookie", "Corded phone", "Couch", "Countertop", "Cowboy hat", "Crab", "Cricket ball", "Crocodile", "Croissant", "Crown", "Crutch", "Cucumber", "Cupboard", "Curtain", "Cutting board", "Dagger", "Deer", "Desk", "Dessert", "Dice", "Digital clock", "Dinosaur", "Dog", "Dog bed", "Doll", "Dolphin", "Door", "Door handle", "Doughnut", "Dragonfly", "Drawer", "Dress", "Drink", "Drinking straw", "Drum", "Duck", "Dumbbell", "Eagle", "Earrings", "Egg", "Elephant", "Envelope", "Falcon", "Fedora", "Filing cabinet", "Fire hydrant", "Fireplace", "Fish", "Flag", "Flashlight", "Flower", "Flowerpot", "Flute", "Food processor", "Football", "Football helmet", "Footwear", "Fork", "Fountain", "Fox", "French fries", "Frog", "Fruit", "Frying pan", "Furniture", "Gas stove", "Giraffe", "Girl", "Glasses", "Glove", "Goat", "Goggles", "Goldfish", "Golf ball", "Golf cart", "Gondola", "Goose", "Grape", "Grapefruit", "Guacamole", "Guitar", "Hamburger", "Hamster", "Handbag", "Handgun", "Harbor seal", "Harp", "Harpsichord", "Hat", "Headphones", "Helicopter", "Helmet", "High heels", "Home appliance", "Honeycomb", "Horn", "Horse", "Hot dog", "House", "Houseplant", "Human arm", "Human beard", "Human ear", "Human eye", "Human face", "Human foot", "Human hair", "Human hand", "Human head", "Human leg", "Human mouth", "Human nose", "Ice cream", "Infant bed", "Insect", "Invertebrate", "Jacket", "Jaguar", "Jeans", "Jellyfish", "Jet ski", "Jug", "Juice", "Kangaroo", "Kettle", "Kitchen & dining room table", "Kitchen appliance", "Kitchen knife", "Kite", "Knife", "Ladder", "Ladybug", "Lamp", "Land vehicle", "Lantern", "Laptop", "Lavender", "Lemon", "Leopard", "Lifejacket", "Light bulb", "Light switch", "Lighthouse", "Lily", "Limousine", "Lion", "Lizard", "Lobster", "Loveseat", "Luggage and bags", "Lynx", "Man", "Mango", "Maple", "Marine invertebrates", "Marine mammal", "Measuring cup", "Mechanical fan", "Microphone", "Microwave oven", "Miniskirt", "Mirror", "Missile", "Mixer", "Mobile phone", "Monkey", "Moths and butterflies", "Motorcycle", "Mouse", "Mouse", "Muffin", "Mug", "Mule", "Mushroom", "Musical instrument", "Musical keyboard", "Nail", "Necklace", "Nightstand", "Oboe", "Office building", "Office supplies", "Orange", "Organ", "Ostrich", "Otter", "Oven", "Owl", "Oyster", "Paddle", "Palm tree", "Pancake", "Paper towel", "Parachute", "Parrot", "Pasta", "Peach", "Pear", "Pen", "Penguin", "Person", "Personal care", "Piano", "Picnic basket", "Picture frame", "Pig", "Pillow", "Pineapple", "Pitcher", "Pizza", "Plastic bag", "Plate", "Platter", "Plumbing fixture", "Polar bear", "Pomegranate", "Popcorn", "Porch", "Porcupine", "Poster", "Potato", "Power plugs and sockets", "Pressure cooker", "Pretzel", "Printer", "Pumpkin", "Punching bag", "Rabbit", "Raccoon", "Racket", "Radish", "Raven", "Refrigerator", "Reptile", "Rhinoceros", "Rifle", "Ring binder", "Rocket", "Roller skates", "Rose", "Rugby ball", "Ruler", "Salad", "Salt and pepper shakers", "Sandal", "Sandwich", "Saucer", "Saxophone", "Scarf", "Scissors", "Scoreboard", "Screwdriver", "Sculpture", "Sea lion", "Sea turtle", "Seafood", "Seahorse", "Seat belt", "Segway", "Serving tray", "Sewing machine", "Shark", "Sheep", "Shelf", "Shellfish", "Shirt", "Shorts", "Shotgun", "Shower", "Shrimp", "Sink", "Skateboard", "Ski", "Skirt", "Skull", "Skyscraper", "Slow cooker", "Snail", "Snake", "Snowboard", "Snowman", "Snowmobile", "Snowplow", "Sock", "Sofa bed", "Sombrero", "Sparrow", "Spatula", "Spider", "Spoon", "Sports uniform", "Squash", "Squirrel", "Stairs", "Starfish", "Stationary bicycle", "Stool", "Stop sign", "Strawberry", "Street light", "Stretcher", "studio couch", "Submarine sandwich", "Suit", "Suitcase", "Sun hat", "Sunflower", "Sunglasses", "Surfboard", "Sushi", "Swan", "Swim cap", "Swimming pool", "Swimwear", "Sword", "Table", "Table tennis racket", "Tablet computer", "Tableware", "Taco", "Tank", "Tap", "Tart", "Taxi", "Tea", "Teapot", "Teddy bear", "Telephone", "Television", "Tennis ball", "Tennis racket", "Tent", "Tiara", "Tick", "Tie", "Tiger", "Tin can", "Tire", "Toaster", "Toilet", "Toilet paper", "Tomato", "Torch", "Tortoise", "Towel", "Tower", "Toy", "Traffic light", "Traffic sign", "Train", "Treadmill", "Tree", "Tripod", "Trombone", "Trousers", "Truck", "Trumpet", "Turkey", "Turtle", "Umbrella", "Van", "Vase", "Vegetable", "Vehicle", "Vehicle registration plate", "Violin", "Volleyball", "Waffle", "Wall clock", "Washing machine", "Waste container", "Watch", "Watercraft", "Watermelon", "Weapon", "Whale", "Wheel", "Wheelchair", "Whiteboard", "Willow", "Window", "Window blind", "Wine", "Wine glass", "Winter melon", "Wok", "Woman", "Wood-burning stove", "Woodpecker", "Wrench", "Zebra", "Zucchini"], "bigdetection": ["accordion", "air_conditioner", "airplane", "alarm_clock", "alligator", "alpaca", "ambulance", "ant", "apple", "armadillo", "army_tank", "artichoke", "asparagus", "automatic_washer", "avocado", "award", "awning", "ax", "baby_buggy", "backpack", "bagel", "balance_beam", "ball", "balloon", "banana", "band_aid", "banjo", "baozi", "barbell", "barge", "barrel", "baseball", "baseball_bat", "baseball_glove", "basket", "basketball", "bass_horn", "bat_(animal)", "bathtub", "bear", "bed", "bee", "beehive", "beer_bottle", "bell_pepper", "belt", "bench", "bicycle", "billboard", "binder", "binoculars", "bird", "blackboard", "blender", "board_eraser", "boat", "book", "bookcase", "boot", "bottle", "bottle_opener", "bow-tie", "bow_(weapon)", "bowl", "bowling_ball", "box", "bracelet", "brassiere", "bread", "briefcase", "broccoli", "broom", "building", "bulldozer", "bullhorn", "burrito", "bus_(vehicle)", "butterfly", "cabbage", "cabinet", "cake", "cake_stand", "calculator", "camel", "camera", "can", "can_opener", "canary", "candle", "candy_bar", "cannon", "canoe", "cantaloup", "car_(automobile)", "card", "cargo_ship", "carrot", "cart", "cassette_deck", "castle", "cat", "cat_furniture", "caterpillar", "cd", "cello", "cellular_telephone", "centipede", "chair", "cherry", "chicken_(animal)", "chisel", "chopping_board", "chopstick", "christmas_tree", "cigarette", "clarinet", "cleansing_agent", "clock", "clutch_bag", "coat", "coat_hanger", "cocktail", "coconut", "coffee_maker", "coffee_table", "coin", "computer_box", "computer_keyboard", "computer_monitor", "cone", "converter", "cooker", "cookie", "cooking_utensil", "cosmetics", "cow", "cowboy_hat", "crab_(animal)", "crane", "crisp_(potato_chip)", "crow", "crown", "crutch", "cucumber", "cue", "cup", "cupboard", "curling", "curtain", "cymbal", "dagger", "dartboard", "deer", "desk", "dessert", "diaper", "dice", "digital_clock", "dinosaur", "dishwasher", "dispenser", "dog", "dog_bed", "doll", "dolphin", "door", "doorknob", "doughnut", "dragonfly", "drawer", "dress", "drill", "drum_(musical_instrument)", "duck", "duffel_bag", "dumbbell", "dumpling", "durian", "eagle", "earphone", "earring", "edible_corn", "egg", "egg_roll", "egg_tart", "eggbeater", "eggplant", "elephant", "envelope", "eraser", "fan", "faucet", "fax", "ferret", "ferris_wheel", "fig_(fruit)", "file_cabinet", "fire_engine", "fire_extinguisher", "fireplace", "fireplug", "fish", "fishing_rod", "flag", "flashlight", "flower_arrangement", "flowerpot", "flute", "food_processor", "football_(American)", "football_helmet", "fork", "fountain", "fox", "french_fries", "frisbee", "frog", "fruit", "fruit_juice", "frying_pan", "fume_hood", "gameboard", "garlic", "gazelle", "giant_panda", "giraffe", "glass_(drink_container)", "globe", "glove", "goat", "goggles", "goldfish", "golf_ball", "golf_club", "golfcart", "gondola_(boat)", "goose", "gorilla", "grape", "grapefruit", "green_bean", "green_onion", "grinder", "guacamole", "guitar", "gun", "hair_dryer", "hairbrush", "hamburger", "hammer", "hamster", "hand_dryer", "handbag", "handcart", "handle", "handsaw", "harbor_seal", "harmonica", "harmonium", "harp", "hat", "headset", "heater", "hedgehog", "helicopter", "helmet", "high_heels", "hippopotamus", "hockey_stick", "hog", "horizontal_bar", "horse", "horse_carriage", "hose", "hot-air_balloon", "hot_dog", "house", "humidifier", "hurdle", "icecream", "igniter", "indoor_rower", "infant_bed", "insect", "iPod", "iron_(for_clothing)", "isopod", "jacket", "jaguar", "jean", "jellyfish", "kangaroo", "kettle", "key", "kitchen_table", "kite", "kiwi_fruit", "knife", "koala", "ladder", "ladle", "ladybug", "lamp", "lantern", "laptop_computer", "lavender", "leather_shoes", "lemon", "lettuce", "license_plate", "life_buoy", "life_jacket", "light_switch", "lightbulb", "lighthouse", "lily", "limousine", "lion", "lizard", "lobster", "lynx", "mango", "maple", "marker", "mask", "measuring_cup", "measuring_stick", "meatball", "melon", "microphone", "microwave_oven", "milk", "minivan", "mirror", "missile", "mixer_(kitchen_tool)", "monkey", "mop", "motor_scooter", "motorcycle", "mouse_(computer_equipment)", "muffin", "mug", "mule", "mushroom", "musical_instrument", "nail", "napkin", "necklace", "necktie", "nightstand", "noodles", "notepad", "nuts", "okra", "olive_oil", "onion", "orange_(fruit)", "ostrich", "otter", "oven", "owl", "oyster", "paddle", "paintbrush", "painting", "palm_tree", "pancake", "papaya", "paper_towel", "parachute", "parking_meter", "parrot", "pasta", "pastry", "peach", "pear", "pen", "pencil_box", "pencil_sharpener", "penguin", "pepper", "perfume", "person", "piano", "pickup_truck", "pie", "pigeon", "pillow", "pineapple", "ping-pong_ball", "pistol", "pizza", "pizza_cutter", "plastic_bag", "plate", "pliers", "plum", "poker_card", "polar_bear", "pomegranate", "pool_table", "popcorn", "porch", "porcupine", "poster", "pot", "potato", "potted_plant", "pretzel", "printer", "projector", "pumpkin", "punching_bag", "rabbit", "raccoon", "race_car", "racket", "radio_receiver", "radish", "rays_and_skates", "red_panda", "refrigerator", "remote_control", "rhinoceros", "rice", "rickshaw", "ring", "roller_skate", "rose", "router_(computer_equipment)", "runner_(carpet)", "salad", "saltshaker", "sandal_(type_of_shoe)", "sandwich", "saucer", "sausage", "saxophone", "scale_(measuring_instrument)", "scarf", "scissors", "scoreboard", "scorpion", "screwdriver", "scrubbing_brush", "sculpture", "seahorse", "seashell", "seat_belt", "segway", "sewing_machine", "shaker", "shampoo", "shark", "shaving_cream", "sheep", "shirt", "shoe", "short_pants", "shovel", "shower_head", "shrimp", "sink", "skateboard", "ski", "skirt", "skullcap", "skunk", "slide", "slipper_(footwear)", "snail", "snake", "sneakers", "snowboard", "snowman", "snowmobile", "snowplow", "soap", "soccer_ball", "sock", "sofa", "sofa_bed", "sombrero", "space_shuttle", "sparrow", "spatula", "speaker_(stero_equipment)", "spectacles", "spice_rack", "spider", "spoon", "sportswear", "squid_(food)", "squirrel", "stapler_(stapling_machine)", "starfish", "stationary_bicycle", "steak_(food)", "stethoscope", "stool", "stop_sign", "stove", "straw_(for_drinking)", "strawberry", "street_sign", "streetlight", "stretcher", "string_cheese", "submarine", "suit_(clothing)", "suitcase", "sunflower", "sunglasses", "sunhat", "surfboard", "surveillance", "sushi", "swim_cap", "swimming_pool", "swimsuit", "swing", "sword", "syringe", "table", "table_tennis_racket", "tablet_computer", "taco", "tape_(sticky_cloth_or_paper)", "taxi", "teapot", "teddy_bear", "telephone", "television_set", "tennis_ball", "tennis_racket", "tent", "thermos_bottle", "tiger", "tissue_paper", "toaster", "toilet", "toilet_tissue", "tomato", "tongs", "toothbrush", "towel", "tower", "toy", "traffic_light", "trailer_truck", "train_(railroad_vehicle)", "training_bench", "trash_can", "tray", "treadmill", "tree", "tree_house", "tricycle", "tripod", "trombone", "trophy_cup", "trousers", "truck", "trumpet", "turkey", "turtle", "typewriter", "umbrella", "unicycle", "urinal", "vacuum_cleaner", "vase", "vegetables", "violin", "volleyball", "waffle", "waffle_iron", "wall_socket", "wardrobe", "watch", "water_jug", "water_scooter", "watermelon", "whale", "wheel", "wheelchair", "willow", "wind_chime", "windmill", "window", "window_blind", "wine_rack", "wineglass", "wolf", "worm", "wrench", "yak", "zebra", "zucchini"]} \ No newline at end of file diff --git a/fourm/utils/tokenizer/text_tokenizer.py b/fourm/utils/tokenizer/text_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..e2c387a4a65dc4f9069f61c8527e98dd97486a9f --- /dev/null +++ b/fourm/utils/tokenizer/text_tokenizer.py @@ -0,0 +1,135 @@ +import json +import os +from collections import defaultdict +from typing import Optional, Union, List + +from tokenizers import AddedToken, decoders, trainers +from tokenizers import Tokenizer +from tokenizers.models import WordPiece +from tokenizers.normalizers import BertNormalizer +from tokenizers.pre_tokenizers import BertPreTokenizer + + +def generate_sentinel_tokens(num=100, start_id=0): + tokens = [ + AddedToken(content=f"[S_{i}]", single_word=True, normalized=False) + for i in range(start_id, num + start_id) + ] + + return tokens + +def generate_coord_tokens(bins=1000): + """Extra tokens that are used for bounding box coordinates, + xmin, ymin, xmax, ymax, but also other modalities like color + maps, metadata, or poses. + """ + tokens = [] + coords_str = ["v0={}", "v1={}", "v2={}", "v3={}"] + + for s in coords_str: + for i in range(bins): + tokens.append(AddedToken(content=s.format(i), single_word=True, normalized=False)) + + return tokens + +def generate_object_class_tokens(dataset="coco"): + with open(os.path.join(os.path.dirname(__file__), 'object_classes.json')) as f: + object_classes = json.load(f)[dataset] + + tokens = [ + AddedToken(content=class_name, single_word=True, normalized=True) + for class_name in object_classes + ] + + return tokens + + +def train_unified_wordpiece_tokenizer( + files, + vocab_size, + sentinel_tokens: List[Union[str, AddedToken]] = None, + coord_tokens: List[Union[str, AddedToken]] = None, + object_class_tokens: List[Union[str, AddedToken]] = None, + unk_token: Union[str, AddedToken] = "[UNK]", + pad_token: Union[str, AddedToken] = "[PAD]", + sos_token: Union[str, AddedToken] = "[SOS]", + eos_token: Union[str, AddedToken] = "[EOS]", + additional_special_tokens: List[Union[str, AddedToken]] = None, + min_frequency=0, + clean_text: bool = True, + handle_chinese_chars: bool = True, + strip_accents: Optional[bool] = None, + lowercase: bool = True, + wordpieces_prefix: str = "##", + show_progress=True, +): + tokenizer = Tokenizer(WordPiece(unk_token=str(unk_token))) + + tokenizer.normalizer = BertNormalizer( + clean_text=clean_text, + handle_chinese_chars=handle_chinese_chars, + strip_accents=strip_accents, + lowercase=lowercase, + ) + tokenizer.pre_tokenizer = BertPreTokenizer() + tokenizer.decoder = decoders.WordPiece(prefix=wordpieces_prefix) + + special_tokens = [] + special_tokens.append(pad_token) + special_tokens.append(unk_token) + special_tokens.append(sos_token) + special_tokens.append(eos_token) + + if sentinel_tokens is not None: + special_tokens.extend(sentinel_tokens) + if coord_tokens is not None: + special_tokens.extend(coord_tokens) + if object_class_tokens is not None: + special_tokens.extend(object_class_tokens) + if additional_special_tokens is not None: + special_tokens.extend(additional_special_tokens) + + trainer = trainers.WordPieceTrainer( + vocab_size=vocab_size, + min_frequency=min_frequency, + show_progress=show_progress, + continuing_subword_prefix=wordpieces_prefix, + special_tokens=special_tokens, + ) + + if isinstance(files, str): + files = [files] + + tokenizer.train(files, trainer=trainer) + + return tokenizer + + +def get_sentinel_to_id_mapping(tokenizer, match_str="[S_"): + sentinel_tokens = {k: v for k, v in tokenizer.get_vocab().items() if k.startswith(match_str)} + # Extract the sentinel token id, the id is of the form "[S_0]", "[S_1]", etc. + sentinel_to_id = {int(k.split("_")[1][:-1]): v for k, v in sorted(sentinel_tokens.items(), key=lambda x:x[1])} + return sentinel_to_id + + +def split_by_sentinel(seq_ids, sentinel_ids): + splits = defaultdict(list) + cur_sentinel = None + for token in seq_ids: + if token in sentinel_ids: + cur_sentinel = token + else: + splits[cur_sentinel].append(token) + + return splits + + +def merge_span_masking(input_seq, decoder_seq, sentinel_ids): + decoder_splits = split_by_sentinel(decoder_seq, sentinel_ids) + out_seq = [] + for token in input_seq: + if token in sentinel_ids: + out_seq.extend(decoder_splits[token]) + else: + out_seq.append(token) + return out_seq diff --git a/fourm/utils/tokenizer/trained/text_tokenizer_4m_wordpiece_30k.json b/fourm/utils/tokenizer/trained/text_tokenizer_4m_wordpiece_30k.json new file mode 100644 index 0000000000000000000000000000000000000000..c8d3f6edd643dd5f216b01946c60a391a4e968fc --- /dev/null +++ b/fourm/utils/tokenizer/trained/text_tokenizer_4m_wordpiece_30k.json @@ -0,0 +1,68587 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "[PAD]", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1, + "content": "[UNK]", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2, + "content": "[SOS]", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3, + "content": "[EOS]", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4, + "content": "[S_0]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 5, + "content": "[S_1]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 6, + "content": "[S_2]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 7, + "content": "[S_3]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 8, + "content": "[S_4]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 9, + "content": "[S_5]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 10, + "content": "[S_6]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 11, + "content": "[S_7]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 12, + "content": "[S_8]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 13, + "content": "[S_9]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 14, + "content": "[S_10]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 15, + "content": "[S_11]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 16, + "content": "[S_12]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 17, + "content": "[S_13]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 18, + "content": "[S_14]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 19, + "content": "[S_15]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 20, + "content": "[S_16]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 21, + "content": "[S_17]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 22, + "content": "[S_18]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 23, + "content": "[S_19]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 24, + "content": "[S_20]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 25, + "content": "[S_21]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 26, + "content": "[S_22]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 27, + "content": "[S_23]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 28, + "content": "[S_24]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 29, + "content": "[S_25]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 30, + "content": "[S_26]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 31, + "content": "[S_27]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 32, + "content": "[S_28]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 33, + "content": "[S_29]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 34, + "content": "[S_30]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 35, + "content": "[S_31]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 36, + "content": "[S_32]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 37, + "content": "[S_33]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 38, + "content": "[S_34]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 39, + "content": "[S_35]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 40, + "content": "[S_36]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 41, + "content": "[S_37]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 42, + "content": "[S_38]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 43, + "content": "[S_39]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 44, + "content": "[S_40]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 45, + "content": "[S_41]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 46, + "content": "[S_42]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 47, + "content": "[S_43]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 48, + "content": "[S_44]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 49, + "content": "[S_45]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 50, + "content": "[S_46]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 51, + "content": "[S_47]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 52, + "content": "[S_48]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 53, + "content": "[S_49]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 54, + "content": "[S_50]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 55, + "content": "[S_51]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 56, + "content": "[S_52]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 57, + "content": "[S_53]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 58, + "content": "[S_54]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 59, + "content": "[S_55]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 60, + "content": "[S_56]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 61, + "content": "[S_57]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 62, + "content": "[S_58]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 63, + "content": "[S_59]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 64, + "content": "[S_60]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 65, + "content": "[S_61]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 66, + "content": "[S_62]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 67, + "content": "[S_63]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 68, + "content": "[S_64]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 69, + "content": "[S_65]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 70, + "content": "[S_66]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 71, + "content": "[S_67]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 72, + "content": "[S_68]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 73, + "content": "[S_69]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 74, + "content": "[S_70]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 75, + "content": "[S_71]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 76, + "content": "[S_72]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 77, + "content": "[S_73]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 78, + "content": "[S_74]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 79, + "content": "[S_75]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 80, + "content": "[S_76]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 81, + "content": "[S_77]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 82, + "content": "[S_78]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 83, + "content": "[S_79]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 84, + "content": "[S_80]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 85, + "content": "[S_81]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 86, + "content": "[S_82]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 87, + "content": "[S_83]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 88, + "content": "[S_84]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 89, + "content": "[S_85]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 90, + "content": "[S_86]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 91, + "content": "[S_87]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 92, + "content": "[S_88]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 93, + "content": "[S_89]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 94, + "content": "[S_90]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 95, + "content": "[S_91]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 96, + "content": "[S_92]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 97, + "content": "[S_93]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 98, + "content": "[S_94]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 99, + "content": "[S_95]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 100, + "content": "[S_96]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 101, + "content": "[S_97]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 102, + "content": "[S_98]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 103, + "content": "[S_99]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 104, + "content": "[S_100]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 105, + "content": "[S_101]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 106, + "content": "[S_102]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 107, + "content": "[S_103]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 108, + "content": "[S_104]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 109, + "content": "[S_105]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 110, + "content": "[S_106]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 111, + "content": "[S_107]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 112, + "content": "[S_108]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 113, + "content": "[S_109]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 114, + "content": "[S_110]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 115, + "content": "[S_111]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 116, + "content": "[S_112]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 117, + "content": "[S_113]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 118, + "content": "[S_114]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 119, + "content": "[S_115]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 120, + "content": "[S_116]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 121, + "content": "[S_117]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 122, + "content": "[S_118]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 123, + "content": "[S_119]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 124, + "content": "[S_120]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 125, + "content": "[S_121]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 126, + "content": "[S_122]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 127, + "content": "[S_123]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128, + "content": "[S_124]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 129, + "content": "[S_125]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 130, + "content": "[S_126]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 131, + "content": "[S_127]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 132, + "content": "[S_128]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 133, + "content": "[S_129]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 134, + "content": "[S_130]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 135, + "content": "[S_131]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 136, + "content": "[S_132]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 137, + "content": "[S_133]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 138, + "content": "[S_134]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 139, + "content": "[S_135]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 140, + "content": "[S_136]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 141, + "content": "[S_137]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 142, + "content": "[S_138]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 143, + "content": "[S_139]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 144, + "content": "[S_140]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 145, + "content": "[S_141]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 146, + "content": "[S_142]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 147, + "content": "[S_143]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 148, + "content": "[S_144]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 149, + "content": "[S_145]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 150, + "content": "[S_146]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 151, + "content": "[S_147]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 152, + "content": "[S_148]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 153, + "content": "[S_149]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 154, + "content": "[S_150]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 155, + "content": "[S_151]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 156, + "content": "[S_152]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 157, + "content": "[S_153]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 158, + "content": "[S_154]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 159, + "content": "[S_155]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 160, + "content": "[S_156]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 161, + "content": "[S_157]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 162, + "content": "[S_158]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 163, + "content": "[S_159]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 164, + "content": "[S_160]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 165, + "content": "[S_161]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 166, + "content": "[S_162]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 167, + "content": "[S_163]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 168, + "content": "[S_164]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 169, + "content": "[S_165]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 170, + "content": "[S_166]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 171, + "content": "[S_167]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 172, + "content": "[S_168]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 173, + "content": "[S_169]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 174, + "content": "[S_170]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 175, + "content": "[S_171]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 176, + "content": "[S_172]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 177, + "content": "[S_173]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 178, + "content": "[S_174]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 179, + "content": "[S_175]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 180, + "content": "[S_176]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 181, + "content": "[S_177]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 182, + "content": "[S_178]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 183, + "content": "[S_179]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 184, + "content": "[S_180]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 185, + "content": "[S_181]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 186, + "content": "[S_182]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 187, + "content": "[S_183]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 188, + "content": "[S_184]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 189, + "content": "[S_185]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 190, + "content": "[S_186]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 191, + "content": "[S_187]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 192, + "content": "[S_188]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 193, + "content": "[S_189]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 194, + "content": "[S_190]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 195, + "content": "[S_191]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 196, + "content": "[S_192]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 197, + "content": "[S_193]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 198, + "content": "[S_194]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 199, + "content": "[S_195]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 200, + "content": "[S_196]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 201, + "content": "[S_197]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 202, + "content": "[S_198]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 203, + "content": "[S_199]", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 204, + "content": "v0=0", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 205, + "content": "v0=1", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 206, + "content": "v0=2", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 207, + "content": "v0=3", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 208, + "content": "v0=4", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 209, + "content": "v0=5", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 210, + "content": "v0=6", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 211, + "content": "v0=7", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 212, + "content": "v0=8", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 213, + "content": "v0=9", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 214, + "content": "v0=10", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 215, + "content": "v0=11", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 216, + "content": "v0=12", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 217, + "content": "v0=13", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 218, + "content": "v0=14", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 219, + "content": "v0=15", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 220, + "content": "v0=16", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 221, + "content": "v0=17", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 222, + "content": "v0=18", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 223, + "content": "v0=19", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 224, + "content": "v0=20", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 225, + "content": "v0=21", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 226, + "content": "v0=22", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 227, + "content": "v0=23", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 228, + "content": "v0=24", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 229, + "content": "v0=25", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 230, + "content": "v0=26", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 231, + "content": "v0=27", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 232, + "content": "v0=28", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 233, + "content": "v0=29", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 234, + "content": "v0=30", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 235, + "content": "v0=31", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 236, + "content": "v0=32", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 237, + "content": "v0=33", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 238, + "content": "v0=34", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 239, + "content": "v0=35", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 240, + "content": "v0=36", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 241, + "content": "v0=37", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 242, + "content": "v0=38", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 243, + "content": "v0=39", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 244, + "content": "v0=40", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 245, + "content": "v0=41", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 246, + "content": "v0=42", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 247, + "content": "v0=43", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 248, + "content": "v0=44", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 249, + "content": "v0=45", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 250, + "content": "v0=46", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 251, + "content": "v0=47", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 252, + "content": "v0=48", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 253, + "content": "v0=49", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 254, + "content": "v0=50", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 255, + "content": "v0=51", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 256, + "content": "v0=52", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 257, + "content": "v0=53", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 258, + "content": "v0=54", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 259, + "content": "v0=55", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 260, + "content": "v0=56", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 261, + "content": "v0=57", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 262, + "content": "v0=58", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 263, + "content": "v0=59", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 264, + "content": "v0=60", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 265, + "content": "v0=61", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 266, + "content": "v0=62", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 267, + "content": "v0=63", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 268, + "content": "v0=64", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 269, + "content": "v0=65", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 270, + "content": "v0=66", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 271, + "content": "v0=67", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 272, + "content": "v0=68", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 273, + "content": "v0=69", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 274, + "content": "v0=70", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 275, + "content": "v0=71", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 276, + "content": "v0=72", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 277, + "content": "v0=73", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 278, + "content": "v0=74", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 279, + "content": "v0=75", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 280, + "content": "v0=76", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 281, + "content": "v0=77", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 282, + "content": "v0=78", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 283, + "content": "v0=79", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 284, + "content": "v0=80", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 285, + "content": "v0=81", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 286, + "content": "v0=82", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 287, + "content": "v0=83", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 288, + "content": "v0=84", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 289, + "content": "v0=85", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 290, + "content": "v0=86", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 291, + "content": "v0=87", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 292, + "content": "v0=88", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 293, + "content": "v0=89", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 294, + "content": "v0=90", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 295, + "content": "v0=91", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 296, + "content": "v0=92", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 297, + "content": "v0=93", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 298, + "content": "v0=94", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 299, + "content": "v0=95", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 300, + "content": "v0=96", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 301, + "content": "v0=97", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 302, + "content": "v0=98", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 303, + "content": "v0=99", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 304, + "content": "v0=100", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 305, + "content": "v0=101", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 306, + "content": "v0=102", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 307, + "content": "v0=103", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 308, + "content": "v0=104", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 309, + "content": "v0=105", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 310, + "content": "v0=106", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 311, + "content": "v0=107", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 312, + "content": "v0=108", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 313, + "content": "v0=109", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 314, + "content": "v0=110", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 315, + "content": "v0=111", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 316, + "content": "v0=112", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 317, + "content": "v0=113", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 318, + "content": "v0=114", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 319, + "content": "v0=115", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 320, + "content": "v0=116", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 321, + "content": "v0=117", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 322, + "content": "v0=118", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 323, + "content": "v0=119", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 324, + "content": "v0=120", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 325, + "content": "v0=121", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 326, + "content": "v0=122", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 327, + "content": "v0=123", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 328, + "content": "v0=124", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 329, + "content": "v0=125", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 330, + "content": "v0=126", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 331, + "content": "v0=127", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 332, + "content": "v0=128", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 333, + "content": "v0=129", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 334, + "content": "v0=130", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 335, + "content": "v0=131", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 336, + "content": "v0=132", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 337, + "content": "v0=133", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 338, + "content": "v0=134", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 339, + "content": "v0=135", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 340, + "content": "v0=136", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 341, + "content": "v0=137", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 342, + "content": "v0=138", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 343, + "content": "v0=139", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 344, + "content": "v0=140", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 345, + "content": "v0=141", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 346, + "content": "v0=142", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 347, + "content": "v0=143", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 348, + "content": "v0=144", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 349, + "content": "v0=145", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 350, + "content": "v0=146", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 351, + "content": "v0=147", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 352, + "content": "v0=148", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 353, + "content": "v0=149", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 354, + "content": "v0=150", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 355, + "content": "v0=151", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 356, + "content": "v0=152", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 357, + "content": "v0=153", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 358, + "content": "v0=154", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 359, + "content": "v0=155", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 360, + "content": "v0=156", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 361, + "content": "v0=157", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 362, + "content": "v0=158", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 363, + "content": "v0=159", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 364, + "content": "v0=160", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 365, + "content": "v0=161", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 366, + "content": "v0=162", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 367, + "content": "v0=163", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 368, + "content": "v0=164", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 369, + "content": "v0=165", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 370, + "content": "v0=166", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 371, + "content": "v0=167", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 372, + "content": "v0=168", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 373, + "content": "v0=169", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 374, + "content": "v0=170", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 375, + "content": "v0=171", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 376, + "content": "v0=172", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 377, + "content": "v0=173", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 378, + "content": "v0=174", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 379, + "content": "v0=175", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 380, + "content": "v0=176", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 381, + "content": "v0=177", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 382, + "content": "v0=178", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 383, + "content": "v0=179", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 384, + "content": "v0=180", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 385, + "content": "v0=181", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 386, + "content": "v0=182", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 387, + "content": "v0=183", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 388, + "content": "v0=184", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 389, + "content": "v0=185", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 390, + "content": "v0=186", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 391, + "content": "v0=187", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 392, + "content": "v0=188", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 393, + "content": "v0=189", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 394, + "content": "v0=190", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 395, + "content": "v0=191", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 396, + "content": "v0=192", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 397, + "content": "v0=193", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 398, + "content": "v0=194", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 399, + "content": "v0=195", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 400, + "content": "v0=196", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 401, + "content": "v0=197", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 402, + "content": "v0=198", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 403, + "content": "v0=199", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 404, + "content": "v0=200", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 405, + "content": "v0=201", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 406, + "content": "v0=202", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 407, + "content": "v0=203", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 408, + "content": "v0=204", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 409, + "content": "v0=205", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 410, + "content": "v0=206", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 411, + "content": "v0=207", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 412, + "content": "v0=208", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 413, + "content": "v0=209", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 414, + "content": "v0=210", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 415, + "content": "v0=211", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 416, + "content": "v0=212", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 417, + "content": "v0=213", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 418, + "content": "v0=214", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 419, + "content": "v0=215", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 420, + "content": "v0=216", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 421, + "content": "v0=217", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 422, + "content": "v0=218", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 423, + "content": "v0=219", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 424, + "content": "v0=220", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 425, + "content": "v0=221", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 426, + "content": "v0=222", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 427, + "content": "v0=223", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 428, + "content": "v0=224", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 429, + "content": "v0=225", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 430, + "content": "v0=226", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 431, + "content": "v0=227", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 432, + "content": "v0=228", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 433, + "content": "v0=229", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 434, + "content": "v0=230", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 435, + "content": "v0=231", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 436, + "content": "v0=232", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 437, + "content": "v0=233", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 438, + "content": "v0=234", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 439, + "content": "v0=235", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 440, + "content": "v0=236", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 441, + "content": "v0=237", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 442, + "content": "v0=238", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 443, + "content": "v0=239", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 444, + "content": "v0=240", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 445, + "content": "v0=241", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 446, + "content": "v0=242", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 447, + "content": "v0=243", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 448, + "content": "v0=244", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 449, + "content": "v0=245", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 450, + "content": "v0=246", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 451, + "content": "v0=247", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 452, + "content": "v0=248", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 453, + "content": "v0=249", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 454, + "content": "v0=250", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 455, + "content": "v0=251", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 456, + "content": "v0=252", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 457, + "content": "v0=253", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 458, + "content": "v0=254", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 459, + "content": "v0=255", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 460, + "content": "v0=256", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 461, + "content": "v0=257", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 462, + "content": "v0=258", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 463, + "content": "v0=259", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 464, + "content": "v0=260", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 465, + "content": "v0=261", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 466, + "content": "v0=262", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 467, + "content": "v0=263", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 468, + "content": "v0=264", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 469, + "content": "v0=265", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 470, + "content": "v0=266", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 471, + "content": "v0=267", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 472, + "content": "v0=268", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 473, + "content": "v0=269", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 474, + "content": "v0=270", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 475, + "content": "v0=271", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 476, + "content": "v0=272", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 477, + "content": "v0=273", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 478, + "content": "v0=274", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 479, + "content": "v0=275", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 480, + "content": "v0=276", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 481, + "content": "v0=277", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 482, + "content": "v0=278", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 483, + "content": "v0=279", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 484, + "content": "v0=280", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 485, + "content": "v0=281", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 486, + "content": "v0=282", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 487, + "content": "v0=283", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 488, + "content": "v0=284", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 489, + "content": "v0=285", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 490, + "content": "v0=286", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 491, + "content": "v0=287", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 492, + "content": "v0=288", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 493, + "content": "v0=289", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 494, + "content": "v0=290", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 495, + "content": "v0=291", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 496, + "content": "v0=292", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 497, + "content": "v0=293", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 498, + "content": "v0=294", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 499, + "content": "v0=295", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 500, + "content": "v0=296", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 501, + "content": "v0=297", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 502, + "content": "v0=298", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 503, + "content": "v0=299", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 504, + "content": "v0=300", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 505, + "content": "v0=301", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 506, + "content": "v0=302", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 507, + "content": "v0=303", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 508, + "content": "v0=304", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 509, + "content": "v0=305", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 510, + "content": "v0=306", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 511, + "content": "v0=307", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 512, + "content": "v0=308", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 513, + "content": "v0=309", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 514, + "content": "v0=310", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 515, + "content": "v0=311", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 516, + "content": "v0=312", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 517, + "content": "v0=313", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 518, + "content": "v0=314", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 519, + "content": "v0=315", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 520, + "content": "v0=316", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 521, + "content": "v0=317", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 522, + "content": "v0=318", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 523, + "content": "v0=319", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 524, + "content": "v0=320", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 525, + "content": "v0=321", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 526, + "content": "v0=322", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 527, + "content": "v0=323", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 528, + "content": "v0=324", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 529, + "content": "v0=325", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 530, + "content": "v0=326", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 531, + "content": "v0=327", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 532, + "content": "v0=328", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 533, + "content": "v0=329", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 534, + "content": "v0=330", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 535, + "content": "v0=331", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 536, + "content": "v0=332", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 537, + "content": "v0=333", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 538, + "content": "v0=334", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 539, + "content": "v0=335", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 540, + "content": "v0=336", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 541, + "content": "v0=337", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 542, + "content": "v0=338", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 543, + "content": "v0=339", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 544, + "content": "v0=340", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 545, + "content": "v0=341", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 546, + "content": "v0=342", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 547, + "content": "v0=343", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 548, + "content": "v0=344", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 549, + "content": "v0=345", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 550, + "content": "v0=346", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 551, + "content": "v0=347", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 552, + "content": "v0=348", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 553, + "content": "v0=349", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 554, + "content": "v0=350", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 555, + "content": "v0=351", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 556, + "content": "v0=352", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 557, + "content": "v0=353", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 558, + "content": "v0=354", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 559, + "content": "v0=355", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 560, + "content": "v0=356", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 561, + "content": "v0=357", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 562, + "content": "v0=358", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 563, + "content": "v0=359", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 564, + "content": "v0=360", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 565, + "content": "v0=361", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 566, + "content": "v0=362", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 567, + "content": "v0=363", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 568, + "content": "v0=364", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 569, + "content": "v0=365", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 570, + "content": "v0=366", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 571, + "content": "v0=367", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 572, + "content": "v0=368", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 573, + "content": "v0=369", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 574, + "content": "v0=370", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 575, + "content": "v0=371", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 576, + "content": "v0=372", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 577, + "content": "v0=373", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 578, + "content": "v0=374", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 579, + "content": "v0=375", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 580, + "content": "v0=376", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 581, + "content": "v0=377", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 582, + "content": "v0=378", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 583, + "content": "v0=379", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 584, + "content": "v0=380", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 585, + "content": "v0=381", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 586, + "content": "v0=382", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 587, + "content": "v0=383", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 588, + "content": "v0=384", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 589, + "content": "v0=385", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 590, + "content": "v0=386", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 591, + "content": "v0=387", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 592, + "content": "v0=388", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 593, + "content": "v0=389", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 594, + "content": "v0=390", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 595, + "content": "v0=391", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 596, + "content": "v0=392", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 597, + "content": "v0=393", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 598, + "content": "v0=394", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 599, + "content": "v0=395", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 600, + "content": "v0=396", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 601, + "content": "v0=397", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 602, + "content": "v0=398", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 603, + "content": "v0=399", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 604, + "content": "v0=400", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 605, + "content": "v0=401", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 606, + "content": "v0=402", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 607, + "content": "v0=403", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 608, + "content": "v0=404", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 609, + "content": "v0=405", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 610, + "content": "v0=406", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 611, + "content": "v0=407", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 612, + "content": "v0=408", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 613, + "content": "v0=409", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 614, + "content": "v0=410", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 615, + "content": "v0=411", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 616, + "content": "v0=412", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 617, + "content": "v0=413", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 618, + "content": "v0=414", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 619, + "content": "v0=415", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 620, + "content": "v0=416", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 621, + "content": "v0=417", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 622, + "content": "v0=418", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 623, + "content": "v0=419", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 624, + "content": "v0=420", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 625, + "content": "v0=421", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 626, + "content": "v0=422", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 627, + "content": "v0=423", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 628, + "content": "v0=424", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 629, + "content": "v0=425", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 630, + "content": "v0=426", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 631, + "content": "v0=427", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 632, + "content": "v0=428", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 633, + "content": "v0=429", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 634, + "content": "v0=430", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 635, + "content": "v0=431", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 636, + "content": "v0=432", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 637, + "content": "v0=433", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 638, + "content": "v0=434", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 639, + "content": "v0=435", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 640, + "content": "v0=436", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 641, + "content": "v0=437", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 642, + "content": "v0=438", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 643, + "content": "v0=439", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 644, + "content": "v0=440", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 645, + "content": "v0=441", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 646, + "content": "v0=442", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 647, + "content": "v0=443", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 648, + "content": "v0=444", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 649, + "content": "v0=445", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 650, + "content": "v0=446", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 651, + "content": "v0=447", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 652, + "content": "v0=448", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 653, + "content": "v0=449", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 654, + "content": "v0=450", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 655, + "content": "v0=451", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 656, + "content": "v0=452", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 657, + "content": "v0=453", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 658, + "content": "v0=454", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 659, + "content": "v0=455", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 660, + "content": "v0=456", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 661, + "content": "v0=457", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 662, + "content": "v0=458", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 663, + "content": "v0=459", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 664, + "content": "v0=460", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 665, + "content": "v0=461", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 666, + "content": "v0=462", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 667, + "content": "v0=463", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 668, + "content": "v0=464", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 669, + "content": "v0=465", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 670, + "content": "v0=466", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 671, + "content": "v0=467", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 672, + "content": "v0=468", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 673, + "content": "v0=469", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 674, + "content": "v0=470", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 675, + "content": "v0=471", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 676, + "content": "v0=472", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 677, + "content": "v0=473", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 678, + "content": "v0=474", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 679, + "content": "v0=475", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 680, + "content": "v0=476", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 681, + "content": "v0=477", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 682, + "content": "v0=478", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 683, + "content": "v0=479", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 684, + "content": "v0=480", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 685, + "content": "v0=481", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 686, + "content": "v0=482", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 687, + "content": "v0=483", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 688, + "content": "v0=484", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 689, + "content": "v0=485", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 690, + "content": "v0=486", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 691, + "content": "v0=487", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 692, + "content": "v0=488", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 693, + "content": "v0=489", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 694, + "content": "v0=490", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 695, + "content": "v0=491", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 696, + "content": "v0=492", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 697, + "content": "v0=493", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 698, + "content": "v0=494", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 699, + "content": "v0=495", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 700, + "content": "v0=496", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 701, + "content": "v0=497", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 702, + "content": "v0=498", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 703, + "content": "v0=499", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 704, + "content": "v0=500", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 705, + "content": "v0=501", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 706, + "content": "v0=502", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 707, + "content": "v0=503", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 708, + "content": "v0=504", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 709, + "content": "v0=505", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 710, + "content": "v0=506", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 711, + "content": "v0=507", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 712, + "content": "v0=508", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 713, + "content": "v0=509", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 714, + "content": "v0=510", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 715, + "content": "v0=511", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 716, + "content": "v0=512", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 717, + "content": "v0=513", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 718, + "content": "v0=514", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 719, + "content": "v0=515", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 720, + "content": "v0=516", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 721, + "content": "v0=517", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 722, + "content": "v0=518", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 723, + "content": "v0=519", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 724, + "content": "v0=520", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 725, + "content": "v0=521", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 726, + "content": "v0=522", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 727, + "content": "v0=523", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 728, + "content": "v0=524", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 729, + "content": "v0=525", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 730, + "content": "v0=526", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 731, + "content": "v0=527", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 732, + "content": "v0=528", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 733, + "content": "v0=529", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 734, + "content": "v0=530", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 735, + "content": "v0=531", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 736, + "content": "v0=532", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 737, + "content": "v0=533", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 738, + "content": "v0=534", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 739, + "content": "v0=535", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 740, + "content": "v0=536", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 741, + "content": "v0=537", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 742, + "content": "v0=538", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 743, + "content": "v0=539", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 744, + "content": "v0=540", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 745, + "content": "v0=541", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 746, + "content": "v0=542", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 747, + "content": "v0=543", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 748, + "content": "v0=544", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 749, + "content": "v0=545", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 750, + "content": "v0=546", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 751, + "content": "v0=547", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 752, + "content": "v0=548", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 753, + "content": "v0=549", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 754, + "content": "v0=550", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 755, + "content": "v0=551", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 756, + "content": "v0=552", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 757, + "content": "v0=553", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 758, + "content": "v0=554", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 759, + "content": "v0=555", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 760, + "content": "v0=556", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 761, + "content": "v0=557", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 762, + "content": "v0=558", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 763, + "content": "v0=559", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 764, + "content": "v0=560", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 765, + "content": "v0=561", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 766, + "content": "v0=562", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 767, + "content": "v0=563", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 768, + "content": "v0=564", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 769, + "content": "v0=565", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 770, + "content": "v0=566", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 771, + "content": "v0=567", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 772, + "content": "v0=568", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 773, + "content": "v0=569", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 774, + "content": "v0=570", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 775, + "content": "v0=571", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 776, + "content": "v0=572", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 777, + "content": "v0=573", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 778, + "content": "v0=574", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 779, + "content": "v0=575", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 780, + "content": "v0=576", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 781, + "content": "v0=577", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 782, + "content": "v0=578", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 783, + "content": "v0=579", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 784, + "content": "v0=580", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 785, + "content": "v0=581", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 786, + "content": "v0=582", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 787, + "content": "v0=583", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 788, + "content": "v0=584", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 789, + "content": "v0=585", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 790, + "content": "v0=586", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 791, + "content": "v0=587", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 792, + "content": "v0=588", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 793, + "content": "v0=589", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 794, + "content": "v0=590", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 795, + "content": "v0=591", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 796, + "content": "v0=592", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 797, + "content": "v0=593", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 798, + "content": "v0=594", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 799, + "content": "v0=595", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 800, + "content": "v0=596", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 801, + "content": "v0=597", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 802, + "content": "v0=598", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 803, + "content": "v0=599", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 804, + "content": "v0=600", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 805, + "content": "v0=601", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 806, + "content": "v0=602", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 807, + "content": "v0=603", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 808, + "content": "v0=604", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 809, + "content": "v0=605", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 810, + "content": "v0=606", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 811, + "content": "v0=607", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 812, + "content": "v0=608", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 813, + "content": "v0=609", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 814, + "content": "v0=610", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 815, + "content": "v0=611", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 816, + "content": "v0=612", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 817, + "content": "v0=613", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 818, + "content": "v0=614", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 819, + "content": "v0=615", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 820, + "content": "v0=616", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 821, + "content": "v0=617", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 822, + "content": "v0=618", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 823, + "content": "v0=619", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 824, + "content": "v0=620", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 825, + "content": "v0=621", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 826, + "content": "v0=622", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 827, + "content": "v0=623", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 828, + "content": "v0=624", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 829, + "content": "v0=625", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 830, + "content": "v0=626", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 831, + "content": "v0=627", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 832, + "content": "v0=628", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 833, + "content": "v0=629", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 834, + "content": "v0=630", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 835, + "content": "v0=631", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 836, + "content": "v0=632", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 837, + "content": "v0=633", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 838, + "content": "v0=634", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 839, + "content": "v0=635", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 840, + "content": "v0=636", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 841, + "content": "v0=637", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 842, + "content": "v0=638", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 843, + "content": "v0=639", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 844, + "content": "v0=640", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 845, + "content": "v0=641", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 846, + "content": "v0=642", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 847, + "content": "v0=643", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 848, + "content": "v0=644", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 849, + "content": "v0=645", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 850, + "content": "v0=646", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 851, + "content": "v0=647", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 852, + "content": "v0=648", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 853, + "content": "v0=649", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 854, + "content": "v0=650", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 855, + "content": "v0=651", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 856, + "content": "v0=652", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 857, + "content": "v0=653", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 858, + "content": "v0=654", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 859, + "content": "v0=655", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 860, + "content": "v0=656", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 861, + "content": "v0=657", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 862, + "content": "v0=658", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 863, + "content": "v0=659", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 864, + "content": "v0=660", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 865, + "content": "v0=661", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 866, + "content": "v0=662", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 867, + "content": "v0=663", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 868, + "content": "v0=664", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 869, + "content": "v0=665", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 870, + "content": "v0=666", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 871, + "content": "v0=667", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 872, + "content": "v0=668", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 873, + "content": "v0=669", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 874, + "content": "v0=670", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 875, + "content": "v0=671", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 876, + "content": "v0=672", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 877, + "content": "v0=673", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 878, + "content": "v0=674", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 879, + "content": "v0=675", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 880, + "content": "v0=676", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 881, + "content": "v0=677", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 882, + "content": "v0=678", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 883, + "content": "v0=679", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 884, + "content": "v0=680", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 885, + "content": "v0=681", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 886, + "content": "v0=682", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 887, + "content": "v0=683", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 888, + "content": "v0=684", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 889, + "content": "v0=685", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 890, + "content": "v0=686", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 891, + "content": "v0=687", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 892, + "content": "v0=688", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 893, + "content": "v0=689", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 894, + "content": "v0=690", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 895, + "content": "v0=691", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 896, + "content": "v0=692", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 897, + "content": "v0=693", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 898, + "content": "v0=694", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 899, + "content": "v0=695", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 900, + "content": "v0=696", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 901, + "content": "v0=697", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 902, + "content": "v0=698", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 903, + "content": "v0=699", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 904, + "content": "v0=700", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 905, + "content": "v0=701", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 906, + "content": "v0=702", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 907, + "content": "v0=703", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 908, + "content": "v0=704", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 909, + "content": "v0=705", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 910, + "content": "v0=706", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 911, + "content": "v0=707", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 912, + "content": "v0=708", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 913, + "content": "v0=709", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 914, + "content": "v0=710", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 915, + "content": "v0=711", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 916, + "content": "v0=712", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 917, + "content": "v0=713", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 918, + "content": "v0=714", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 919, + "content": "v0=715", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 920, + "content": "v0=716", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 921, + "content": "v0=717", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 922, + "content": "v0=718", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 923, + "content": "v0=719", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 924, + "content": "v0=720", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 925, + "content": "v0=721", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 926, + "content": "v0=722", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 927, + "content": "v0=723", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 928, + "content": "v0=724", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 929, + "content": "v0=725", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 930, + "content": "v0=726", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 931, + "content": "v0=727", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 932, + "content": "v0=728", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 933, + "content": "v0=729", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 934, + "content": "v0=730", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 935, + "content": "v0=731", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 936, + "content": "v0=732", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 937, + "content": "v0=733", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 938, + "content": "v0=734", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 939, + "content": "v0=735", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 940, + "content": "v0=736", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 941, + "content": "v0=737", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 942, + "content": "v0=738", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 943, + "content": "v0=739", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 944, + "content": "v0=740", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 945, + "content": "v0=741", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 946, + "content": "v0=742", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 947, + "content": "v0=743", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 948, + "content": "v0=744", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 949, + "content": "v0=745", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 950, + "content": "v0=746", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 951, + "content": "v0=747", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 952, + "content": "v0=748", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 953, + "content": "v0=749", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 954, + "content": "v0=750", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 955, + "content": "v0=751", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 956, + "content": "v0=752", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 957, + "content": "v0=753", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 958, + "content": "v0=754", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 959, + "content": "v0=755", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 960, + "content": "v0=756", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 961, + "content": "v0=757", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 962, + "content": "v0=758", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 963, + "content": "v0=759", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 964, + "content": "v0=760", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 965, + "content": "v0=761", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 966, + "content": "v0=762", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 967, + "content": "v0=763", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 968, + "content": "v0=764", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 969, + "content": "v0=765", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 970, + "content": "v0=766", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 971, + "content": "v0=767", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 972, + "content": "v0=768", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 973, + "content": "v0=769", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 974, + "content": "v0=770", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 975, + "content": "v0=771", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 976, + "content": "v0=772", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 977, + "content": "v0=773", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 978, + "content": "v0=774", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 979, + "content": "v0=775", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 980, + "content": "v0=776", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 981, + "content": "v0=777", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 982, + "content": "v0=778", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 983, + "content": "v0=779", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 984, + "content": "v0=780", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 985, + "content": "v0=781", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 986, + "content": "v0=782", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 987, + "content": "v0=783", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 988, + "content": "v0=784", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 989, + "content": "v0=785", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 990, + "content": "v0=786", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 991, + "content": "v0=787", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 992, + "content": "v0=788", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 993, + "content": "v0=789", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 994, + "content": "v0=790", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 995, + "content": "v0=791", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 996, + "content": "v0=792", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 997, + "content": "v0=793", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 998, + "content": "v0=794", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 999, + "content": "v0=795", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1000, + "content": "v0=796", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1001, + "content": "v0=797", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1002, + "content": "v0=798", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1003, + "content": "v0=799", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1004, + "content": "v0=800", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1005, + "content": "v0=801", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1006, + "content": "v0=802", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1007, + "content": "v0=803", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1008, + "content": "v0=804", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1009, + "content": "v0=805", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1010, + "content": "v0=806", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1011, + "content": "v0=807", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1012, + "content": "v0=808", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1013, + "content": "v0=809", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1014, + "content": "v0=810", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1015, + "content": "v0=811", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1016, + "content": "v0=812", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1017, + "content": "v0=813", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1018, + "content": "v0=814", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1019, + "content": "v0=815", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1020, + "content": "v0=816", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1021, + "content": "v0=817", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1022, + "content": "v0=818", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1023, + "content": "v0=819", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1024, + "content": "v0=820", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1025, + "content": "v0=821", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1026, + "content": "v0=822", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1027, + "content": "v0=823", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1028, + "content": "v0=824", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1029, + "content": "v0=825", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1030, + "content": "v0=826", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1031, + "content": "v0=827", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1032, + "content": "v0=828", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1033, + "content": "v0=829", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1034, + "content": "v0=830", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1035, + "content": "v0=831", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1036, + "content": "v0=832", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1037, + "content": "v0=833", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1038, + "content": "v0=834", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1039, + "content": "v0=835", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1040, + "content": "v0=836", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1041, + "content": "v0=837", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1042, + "content": "v0=838", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1043, + "content": "v0=839", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1044, + "content": "v0=840", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1045, + "content": "v0=841", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1046, + "content": "v0=842", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1047, + "content": "v0=843", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1048, + "content": "v0=844", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1049, + "content": "v0=845", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1050, + "content": "v0=846", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1051, + "content": "v0=847", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1052, + "content": "v0=848", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1053, + "content": "v0=849", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1054, + "content": "v0=850", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1055, + "content": "v0=851", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1056, + "content": "v0=852", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1057, + "content": "v0=853", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1058, + "content": "v0=854", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1059, + "content": "v0=855", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1060, + "content": "v0=856", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1061, + "content": "v0=857", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1062, + "content": "v0=858", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1063, + "content": "v0=859", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1064, + "content": "v0=860", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1065, + "content": "v0=861", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1066, + "content": "v0=862", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1067, + "content": "v0=863", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1068, + "content": "v0=864", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1069, + "content": "v0=865", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1070, + "content": "v0=866", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1071, + "content": "v0=867", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1072, + "content": "v0=868", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1073, + "content": "v0=869", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1074, + "content": "v0=870", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1075, + "content": "v0=871", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1076, + "content": "v0=872", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1077, + "content": "v0=873", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1078, + "content": "v0=874", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1079, + "content": "v0=875", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1080, + "content": "v0=876", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1081, + "content": "v0=877", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1082, + "content": "v0=878", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1083, + "content": "v0=879", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1084, + "content": "v0=880", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1085, + "content": "v0=881", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1086, + "content": "v0=882", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1087, + "content": "v0=883", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1088, + "content": "v0=884", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1089, + "content": "v0=885", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1090, + "content": "v0=886", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1091, + "content": "v0=887", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1092, + "content": "v0=888", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1093, + "content": "v0=889", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1094, + "content": "v0=890", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1095, + "content": "v0=891", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1096, + "content": "v0=892", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1097, + "content": "v0=893", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1098, + "content": "v0=894", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1099, + "content": "v0=895", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1100, + "content": "v0=896", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1101, + "content": "v0=897", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1102, + "content": "v0=898", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1103, + "content": "v0=899", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1104, + "content": "v0=900", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1105, + "content": "v0=901", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1106, + "content": "v0=902", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1107, + "content": "v0=903", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1108, + "content": "v0=904", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1109, + "content": "v0=905", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1110, + "content": "v0=906", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1111, + "content": "v0=907", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1112, + "content": "v0=908", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1113, + "content": "v0=909", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1114, + "content": "v0=910", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1115, + "content": "v0=911", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1116, + "content": "v0=912", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1117, + "content": "v0=913", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1118, + "content": "v0=914", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1119, + "content": "v0=915", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1120, + "content": "v0=916", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1121, + "content": "v0=917", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1122, + "content": "v0=918", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1123, + "content": "v0=919", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1124, + "content": "v0=920", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1125, + "content": "v0=921", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1126, + "content": "v0=922", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1127, + "content": "v0=923", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1128, + "content": "v0=924", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1129, + "content": "v0=925", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1130, + "content": "v0=926", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1131, + "content": "v0=927", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1132, + "content": "v0=928", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1133, + "content": "v0=929", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1134, + "content": "v0=930", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1135, + "content": "v0=931", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1136, + "content": "v0=932", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1137, + "content": "v0=933", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1138, + "content": "v0=934", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1139, + "content": "v0=935", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1140, + "content": "v0=936", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1141, + "content": "v0=937", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1142, + "content": "v0=938", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1143, + "content": "v0=939", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1144, + "content": "v0=940", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1145, + "content": "v0=941", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1146, + "content": "v0=942", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1147, + "content": "v0=943", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1148, + "content": "v0=944", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1149, + "content": "v0=945", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1150, + "content": "v0=946", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1151, + "content": "v0=947", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1152, + "content": "v0=948", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1153, + "content": "v0=949", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1154, + "content": "v0=950", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1155, + "content": "v0=951", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1156, + "content": "v0=952", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1157, + "content": "v0=953", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1158, + "content": "v0=954", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1159, + "content": "v0=955", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1160, + "content": "v0=956", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1161, + "content": "v0=957", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1162, + "content": "v0=958", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1163, + "content": "v0=959", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1164, + "content": "v0=960", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1165, + "content": "v0=961", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1166, + "content": "v0=962", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1167, + "content": "v0=963", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1168, + "content": "v0=964", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1169, + "content": "v0=965", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1170, + "content": "v0=966", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1171, + "content": "v0=967", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1172, + "content": "v0=968", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1173, + "content": "v0=969", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1174, + "content": "v0=970", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1175, + "content": "v0=971", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1176, + "content": "v0=972", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1177, + "content": "v0=973", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1178, + "content": "v0=974", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1179, + "content": "v0=975", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1180, + "content": "v0=976", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1181, + "content": "v0=977", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1182, + "content": "v0=978", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1183, + "content": "v0=979", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1184, + "content": "v0=980", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1185, + "content": "v0=981", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1186, + "content": "v0=982", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1187, + "content": "v0=983", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1188, + "content": "v0=984", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1189, + "content": "v0=985", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1190, + "content": "v0=986", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1191, + "content": "v0=987", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1192, + "content": "v0=988", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1193, + "content": "v0=989", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1194, + "content": "v0=990", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1195, + "content": "v0=991", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1196, + "content": "v0=992", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1197, + "content": "v0=993", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1198, + "content": "v0=994", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1199, + "content": "v0=995", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1200, + "content": "v0=996", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1201, + "content": "v0=997", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1202, + "content": "v0=998", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1203, + "content": "v0=999", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1204, + "content": "v1=0", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1205, + "content": "v1=1", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1206, + "content": "v1=2", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1207, + "content": "v1=3", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1208, + "content": "v1=4", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1209, + "content": "v1=5", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1210, + "content": "v1=6", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1211, + "content": "v1=7", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1212, + "content": "v1=8", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1213, + "content": "v1=9", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1214, + "content": "v1=10", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1215, + "content": "v1=11", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1216, + "content": "v1=12", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1217, + "content": "v1=13", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1218, + "content": "v1=14", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1219, + "content": "v1=15", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1220, + "content": "v1=16", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1221, + "content": "v1=17", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1222, + "content": "v1=18", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1223, + "content": "v1=19", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1224, + "content": "v1=20", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1225, + "content": "v1=21", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1226, + "content": "v1=22", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1227, + "content": "v1=23", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1228, + "content": "v1=24", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1229, + "content": "v1=25", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1230, + "content": "v1=26", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1231, + "content": "v1=27", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1232, + "content": "v1=28", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1233, + "content": "v1=29", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1234, + "content": "v1=30", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1235, + "content": "v1=31", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1236, + "content": "v1=32", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1237, + "content": "v1=33", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1238, + "content": "v1=34", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1239, + "content": "v1=35", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1240, + "content": "v1=36", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1241, + "content": "v1=37", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1242, + "content": "v1=38", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1243, + "content": "v1=39", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1244, + "content": "v1=40", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1245, + "content": "v1=41", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1246, + "content": "v1=42", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1247, + "content": "v1=43", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1248, + "content": "v1=44", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1249, + "content": "v1=45", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1250, + "content": "v1=46", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1251, + "content": "v1=47", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1252, + "content": "v1=48", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1253, + "content": "v1=49", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1254, + "content": "v1=50", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1255, + "content": "v1=51", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1256, + "content": "v1=52", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1257, + "content": "v1=53", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1258, + "content": "v1=54", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1259, + "content": "v1=55", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1260, + "content": "v1=56", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1261, + "content": "v1=57", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1262, + "content": "v1=58", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1263, + "content": "v1=59", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1264, + "content": "v1=60", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1265, + "content": "v1=61", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1266, + "content": "v1=62", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1267, + "content": "v1=63", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1268, + "content": "v1=64", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1269, + "content": "v1=65", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1270, + "content": "v1=66", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1271, + "content": "v1=67", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1272, + "content": "v1=68", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1273, + "content": "v1=69", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1274, + "content": "v1=70", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1275, + "content": "v1=71", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1276, + "content": "v1=72", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1277, + "content": "v1=73", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1278, + "content": "v1=74", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1279, + "content": "v1=75", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1280, + "content": "v1=76", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1281, + "content": "v1=77", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1282, + "content": "v1=78", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1283, + "content": "v1=79", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1284, + "content": "v1=80", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1285, + "content": "v1=81", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1286, + "content": "v1=82", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1287, + "content": "v1=83", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1288, + "content": "v1=84", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1289, + "content": "v1=85", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1290, + "content": "v1=86", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1291, + "content": "v1=87", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1292, + "content": "v1=88", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1293, + "content": "v1=89", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1294, + "content": "v1=90", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1295, + "content": "v1=91", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1296, + "content": "v1=92", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1297, + "content": "v1=93", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1298, + "content": "v1=94", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1299, + "content": "v1=95", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1300, + "content": "v1=96", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1301, + "content": "v1=97", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1302, + "content": "v1=98", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1303, + "content": "v1=99", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1304, + "content": "v1=100", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1305, + "content": "v1=101", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1306, + "content": "v1=102", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1307, + "content": "v1=103", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1308, + "content": "v1=104", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1309, + "content": "v1=105", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1310, + "content": "v1=106", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1311, + "content": "v1=107", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1312, + "content": "v1=108", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1313, + "content": "v1=109", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1314, + "content": "v1=110", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1315, + "content": "v1=111", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1316, + "content": "v1=112", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1317, + "content": "v1=113", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1318, + "content": "v1=114", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1319, + "content": "v1=115", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1320, + "content": "v1=116", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1321, + "content": "v1=117", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1322, + "content": "v1=118", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1323, + "content": "v1=119", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1324, + "content": "v1=120", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1325, + "content": "v1=121", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1326, + "content": "v1=122", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1327, + "content": "v1=123", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1328, + "content": "v1=124", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1329, + "content": "v1=125", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1330, + "content": "v1=126", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1331, + "content": "v1=127", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1332, + "content": "v1=128", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1333, + "content": "v1=129", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1334, + "content": "v1=130", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1335, + "content": "v1=131", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1336, + "content": "v1=132", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1337, + "content": "v1=133", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1338, + "content": "v1=134", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1339, + "content": "v1=135", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1340, + "content": "v1=136", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1341, + "content": "v1=137", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1342, + "content": "v1=138", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1343, + "content": "v1=139", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1344, + "content": "v1=140", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1345, + "content": "v1=141", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1346, + "content": "v1=142", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1347, + "content": "v1=143", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1348, + "content": "v1=144", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1349, + "content": "v1=145", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1350, + "content": "v1=146", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1351, + "content": "v1=147", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1352, + "content": "v1=148", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1353, + "content": "v1=149", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1354, + "content": "v1=150", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1355, + "content": "v1=151", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1356, + "content": "v1=152", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1357, + "content": "v1=153", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1358, + "content": "v1=154", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1359, + "content": "v1=155", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1360, + "content": "v1=156", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1361, + "content": "v1=157", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1362, + "content": "v1=158", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1363, + "content": "v1=159", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1364, + "content": "v1=160", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1365, + "content": "v1=161", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1366, + "content": "v1=162", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1367, + "content": "v1=163", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1368, + "content": "v1=164", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1369, + "content": "v1=165", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1370, + "content": "v1=166", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1371, + "content": "v1=167", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1372, + "content": "v1=168", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1373, + "content": "v1=169", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1374, + "content": "v1=170", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1375, + "content": "v1=171", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1376, + "content": "v1=172", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1377, + "content": "v1=173", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1378, + "content": "v1=174", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1379, + "content": "v1=175", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1380, + "content": "v1=176", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1381, + "content": "v1=177", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1382, + "content": "v1=178", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1383, + "content": "v1=179", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1384, + "content": "v1=180", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1385, + "content": "v1=181", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1386, + "content": "v1=182", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1387, + "content": "v1=183", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1388, + "content": "v1=184", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1389, + "content": "v1=185", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1390, + "content": "v1=186", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1391, + "content": "v1=187", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1392, + "content": "v1=188", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1393, + "content": "v1=189", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1394, + "content": "v1=190", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1395, + "content": "v1=191", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1396, + "content": "v1=192", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1397, + "content": "v1=193", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1398, + "content": "v1=194", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1399, + "content": "v1=195", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1400, + "content": "v1=196", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1401, + "content": "v1=197", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1402, + "content": "v1=198", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1403, + "content": "v1=199", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1404, + "content": "v1=200", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1405, + "content": "v1=201", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1406, + "content": "v1=202", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1407, + "content": "v1=203", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1408, + "content": "v1=204", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1409, + "content": "v1=205", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1410, + "content": "v1=206", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1411, + "content": "v1=207", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1412, + "content": "v1=208", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1413, + "content": "v1=209", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1414, + "content": "v1=210", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1415, + "content": "v1=211", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1416, + "content": "v1=212", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1417, + "content": "v1=213", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1418, + "content": "v1=214", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1419, + "content": "v1=215", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1420, + "content": "v1=216", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1421, + "content": "v1=217", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1422, + "content": "v1=218", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1423, + "content": "v1=219", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1424, + "content": "v1=220", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1425, + "content": "v1=221", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1426, + "content": "v1=222", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1427, + "content": "v1=223", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1428, + "content": "v1=224", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1429, + "content": "v1=225", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1430, + "content": "v1=226", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1431, + "content": "v1=227", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1432, + "content": "v1=228", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1433, + "content": "v1=229", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1434, + "content": "v1=230", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1435, + "content": "v1=231", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1436, + "content": "v1=232", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1437, + "content": "v1=233", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1438, + "content": "v1=234", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1439, + "content": "v1=235", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1440, + "content": "v1=236", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1441, + "content": "v1=237", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1442, + "content": "v1=238", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1443, + "content": "v1=239", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1444, + "content": "v1=240", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1445, + "content": "v1=241", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1446, + "content": "v1=242", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1447, + "content": "v1=243", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1448, + "content": "v1=244", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1449, + "content": "v1=245", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1450, + "content": "v1=246", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1451, + "content": "v1=247", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1452, + "content": "v1=248", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1453, + "content": "v1=249", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1454, + "content": "v1=250", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1455, + "content": "v1=251", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1456, + "content": "v1=252", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1457, + "content": "v1=253", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1458, + "content": "v1=254", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1459, + "content": "v1=255", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1460, + "content": "v1=256", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1461, + "content": "v1=257", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1462, + "content": "v1=258", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1463, + "content": "v1=259", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1464, + "content": "v1=260", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1465, + "content": "v1=261", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1466, + "content": "v1=262", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1467, + "content": "v1=263", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1468, + "content": "v1=264", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1469, + "content": "v1=265", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1470, + "content": "v1=266", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1471, + "content": "v1=267", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1472, + "content": "v1=268", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1473, + "content": "v1=269", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1474, + "content": "v1=270", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1475, + "content": "v1=271", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1476, + "content": "v1=272", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1477, + "content": "v1=273", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1478, + "content": "v1=274", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1479, + "content": "v1=275", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1480, + "content": "v1=276", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1481, + "content": "v1=277", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1482, + "content": "v1=278", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1483, + "content": "v1=279", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1484, + "content": "v1=280", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1485, + "content": "v1=281", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1486, + "content": "v1=282", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1487, + "content": "v1=283", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1488, + "content": "v1=284", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1489, + "content": "v1=285", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1490, + "content": "v1=286", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1491, + "content": "v1=287", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1492, + "content": "v1=288", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1493, + "content": "v1=289", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1494, + "content": "v1=290", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1495, + "content": "v1=291", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1496, + "content": "v1=292", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1497, + "content": "v1=293", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1498, + "content": "v1=294", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1499, + "content": "v1=295", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1500, + "content": "v1=296", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1501, + "content": "v1=297", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1502, + "content": "v1=298", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1503, + "content": "v1=299", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1504, + "content": "v1=300", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1505, + "content": "v1=301", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1506, + "content": "v1=302", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1507, + "content": "v1=303", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1508, + "content": "v1=304", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1509, + "content": "v1=305", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1510, + "content": "v1=306", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1511, + "content": "v1=307", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1512, + "content": "v1=308", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1513, + "content": "v1=309", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1514, + "content": "v1=310", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1515, + "content": "v1=311", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1516, + "content": "v1=312", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1517, + "content": "v1=313", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1518, + "content": "v1=314", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1519, + "content": "v1=315", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1520, + "content": "v1=316", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1521, + "content": "v1=317", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1522, + "content": "v1=318", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1523, + "content": "v1=319", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1524, + "content": "v1=320", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1525, + "content": "v1=321", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1526, + "content": "v1=322", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1527, + "content": "v1=323", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1528, + "content": "v1=324", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1529, + "content": "v1=325", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1530, + "content": "v1=326", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1531, + "content": "v1=327", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1532, + "content": "v1=328", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1533, + "content": "v1=329", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1534, + "content": "v1=330", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1535, + "content": "v1=331", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1536, + "content": "v1=332", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1537, + "content": "v1=333", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1538, + "content": "v1=334", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1539, + "content": "v1=335", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1540, + "content": "v1=336", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1541, + "content": "v1=337", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1542, + "content": "v1=338", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1543, + "content": "v1=339", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1544, + "content": "v1=340", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1545, + "content": "v1=341", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1546, + "content": "v1=342", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1547, + "content": "v1=343", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1548, + "content": "v1=344", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1549, + "content": "v1=345", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1550, + "content": "v1=346", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1551, + "content": "v1=347", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1552, + "content": "v1=348", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1553, + "content": "v1=349", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1554, + "content": "v1=350", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1555, + "content": "v1=351", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1556, + "content": "v1=352", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1557, + "content": "v1=353", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1558, + "content": "v1=354", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1559, + "content": "v1=355", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1560, + "content": "v1=356", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1561, + "content": "v1=357", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1562, + "content": "v1=358", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1563, + "content": "v1=359", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1564, + "content": "v1=360", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1565, + "content": "v1=361", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1566, + "content": "v1=362", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1567, + "content": "v1=363", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1568, + "content": "v1=364", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1569, + "content": "v1=365", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1570, + "content": "v1=366", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1571, + "content": "v1=367", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1572, + "content": "v1=368", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1573, + "content": "v1=369", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1574, + "content": "v1=370", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1575, + "content": "v1=371", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1576, + "content": "v1=372", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1577, + "content": "v1=373", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1578, + "content": "v1=374", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1579, + "content": "v1=375", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1580, + "content": "v1=376", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1581, + "content": "v1=377", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1582, + "content": "v1=378", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1583, + "content": "v1=379", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1584, + "content": "v1=380", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1585, + "content": "v1=381", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1586, + "content": "v1=382", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1587, + "content": "v1=383", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1588, + "content": "v1=384", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1589, + "content": "v1=385", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1590, + "content": "v1=386", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1591, + "content": "v1=387", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1592, + "content": "v1=388", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1593, + "content": "v1=389", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1594, + "content": "v1=390", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1595, + "content": "v1=391", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1596, + "content": "v1=392", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1597, + "content": "v1=393", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1598, + "content": "v1=394", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1599, + "content": "v1=395", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1600, + "content": "v1=396", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1601, + "content": "v1=397", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1602, + "content": "v1=398", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1603, + "content": "v1=399", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1604, + "content": "v1=400", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1605, + "content": "v1=401", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1606, + "content": "v1=402", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1607, + "content": "v1=403", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1608, + "content": "v1=404", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1609, + "content": "v1=405", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1610, + "content": "v1=406", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1611, + "content": "v1=407", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1612, + "content": "v1=408", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1613, + "content": "v1=409", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1614, + "content": "v1=410", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1615, + "content": "v1=411", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1616, + "content": "v1=412", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1617, + "content": "v1=413", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1618, + "content": "v1=414", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1619, + "content": "v1=415", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1620, + "content": "v1=416", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1621, + "content": "v1=417", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1622, + "content": "v1=418", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1623, + "content": "v1=419", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1624, + "content": "v1=420", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1625, + "content": "v1=421", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1626, + "content": "v1=422", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1627, + "content": "v1=423", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1628, + "content": "v1=424", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1629, + "content": "v1=425", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1630, + "content": "v1=426", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1631, + "content": "v1=427", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1632, + "content": "v1=428", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1633, + "content": "v1=429", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1634, + "content": "v1=430", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1635, + "content": "v1=431", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1636, + "content": "v1=432", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1637, + "content": "v1=433", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1638, + "content": "v1=434", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1639, + "content": "v1=435", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1640, + "content": "v1=436", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1641, + "content": "v1=437", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1642, + "content": "v1=438", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1643, + "content": "v1=439", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1644, + "content": "v1=440", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1645, + "content": "v1=441", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1646, + "content": "v1=442", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1647, + "content": "v1=443", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1648, + "content": "v1=444", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1649, + "content": "v1=445", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1650, + "content": "v1=446", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1651, + "content": "v1=447", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1652, + "content": "v1=448", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1653, + "content": "v1=449", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1654, + "content": "v1=450", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1655, + "content": "v1=451", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1656, + "content": "v1=452", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1657, + "content": "v1=453", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1658, + "content": "v1=454", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1659, + "content": "v1=455", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1660, + "content": "v1=456", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1661, + "content": "v1=457", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1662, + "content": "v1=458", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1663, + "content": "v1=459", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1664, + "content": "v1=460", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1665, + "content": "v1=461", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1666, + "content": "v1=462", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1667, + "content": "v1=463", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1668, + "content": "v1=464", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1669, + "content": "v1=465", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1670, + "content": "v1=466", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1671, + "content": "v1=467", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1672, + "content": "v1=468", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1673, + "content": "v1=469", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1674, + "content": "v1=470", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1675, + "content": "v1=471", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1676, + "content": "v1=472", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1677, + "content": "v1=473", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1678, + "content": "v1=474", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1679, + "content": "v1=475", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1680, + "content": "v1=476", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1681, + "content": "v1=477", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1682, + "content": "v1=478", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1683, + "content": "v1=479", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1684, + "content": "v1=480", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1685, + "content": "v1=481", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1686, + "content": "v1=482", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1687, + "content": "v1=483", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1688, + "content": "v1=484", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1689, + "content": "v1=485", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1690, + "content": "v1=486", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1691, + "content": "v1=487", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1692, + "content": "v1=488", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1693, + "content": "v1=489", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1694, + "content": "v1=490", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1695, + "content": "v1=491", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1696, + "content": "v1=492", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1697, + "content": "v1=493", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1698, + "content": "v1=494", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1699, + "content": "v1=495", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1700, + "content": "v1=496", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1701, + "content": "v1=497", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1702, + "content": "v1=498", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1703, + "content": "v1=499", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1704, + "content": "v1=500", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1705, + "content": "v1=501", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1706, + "content": "v1=502", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1707, + "content": "v1=503", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1708, + "content": "v1=504", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1709, + "content": "v1=505", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1710, + "content": "v1=506", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1711, + "content": "v1=507", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1712, + "content": "v1=508", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1713, + "content": "v1=509", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1714, + "content": "v1=510", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1715, + "content": "v1=511", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1716, + "content": "v1=512", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1717, + "content": "v1=513", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1718, + "content": "v1=514", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1719, + "content": "v1=515", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1720, + "content": "v1=516", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1721, + "content": "v1=517", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1722, + "content": "v1=518", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1723, + "content": "v1=519", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1724, + "content": "v1=520", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1725, + "content": "v1=521", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1726, + "content": "v1=522", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1727, + "content": "v1=523", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1728, + "content": "v1=524", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1729, + "content": "v1=525", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1730, + "content": "v1=526", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1731, + "content": "v1=527", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1732, + "content": "v1=528", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1733, + "content": "v1=529", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1734, + "content": "v1=530", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1735, + "content": "v1=531", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1736, + "content": "v1=532", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1737, + "content": "v1=533", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1738, + "content": "v1=534", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1739, + "content": "v1=535", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1740, + "content": "v1=536", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1741, + "content": "v1=537", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1742, + "content": "v1=538", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1743, + "content": "v1=539", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1744, + "content": "v1=540", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1745, + "content": "v1=541", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1746, + "content": "v1=542", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1747, + "content": "v1=543", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1748, + "content": "v1=544", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1749, + "content": "v1=545", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1750, + "content": "v1=546", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1751, + "content": "v1=547", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1752, + "content": "v1=548", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1753, + "content": "v1=549", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1754, + "content": "v1=550", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1755, + "content": "v1=551", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1756, + "content": "v1=552", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1757, + "content": "v1=553", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1758, + "content": "v1=554", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1759, + "content": "v1=555", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1760, + "content": "v1=556", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1761, + "content": "v1=557", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1762, + "content": "v1=558", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1763, + "content": "v1=559", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1764, + "content": "v1=560", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1765, + "content": "v1=561", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1766, + "content": "v1=562", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1767, + "content": "v1=563", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1768, + "content": "v1=564", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1769, + "content": "v1=565", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1770, + "content": "v1=566", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1771, + "content": "v1=567", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1772, + "content": "v1=568", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1773, + "content": "v1=569", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1774, + "content": "v1=570", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1775, + "content": "v1=571", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1776, + "content": "v1=572", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1777, + "content": "v1=573", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1778, + "content": "v1=574", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1779, + "content": "v1=575", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1780, + "content": "v1=576", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1781, + "content": "v1=577", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1782, + "content": "v1=578", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1783, + "content": "v1=579", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1784, + "content": "v1=580", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1785, + "content": "v1=581", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1786, + "content": "v1=582", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1787, + "content": "v1=583", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1788, + "content": "v1=584", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1789, + "content": "v1=585", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1790, + "content": "v1=586", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1791, + "content": "v1=587", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1792, + "content": "v1=588", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1793, + "content": "v1=589", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1794, + "content": "v1=590", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1795, + "content": "v1=591", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1796, + "content": "v1=592", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1797, + "content": "v1=593", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1798, + "content": "v1=594", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1799, + "content": "v1=595", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1800, + "content": "v1=596", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1801, + "content": "v1=597", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1802, + "content": "v1=598", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1803, + "content": "v1=599", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1804, + "content": "v1=600", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1805, + "content": "v1=601", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1806, + "content": "v1=602", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1807, + "content": "v1=603", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1808, + "content": "v1=604", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1809, + "content": "v1=605", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1810, + "content": "v1=606", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1811, + "content": "v1=607", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1812, + "content": "v1=608", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1813, + "content": "v1=609", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1814, + "content": "v1=610", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1815, + "content": "v1=611", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1816, + "content": "v1=612", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1817, + "content": "v1=613", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1818, + "content": "v1=614", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1819, + "content": "v1=615", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1820, + "content": "v1=616", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1821, + "content": "v1=617", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1822, + "content": "v1=618", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1823, + "content": "v1=619", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1824, + "content": "v1=620", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1825, + "content": "v1=621", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1826, + "content": "v1=622", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1827, + "content": "v1=623", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1828, + "content": "v1=624", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1829, + "content": "v1=625", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1830, + "content": "v1=626", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1831, + "content": "v1=627", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1832, + "content": "v1=628", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1833, + "content": "v1=629", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1834, + "content": "v1=630", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1835, + "content": "v1=631", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1836, + "content": "v1=632", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1837, + "content": "v1=633", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1838, + "content": "v1=634", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1839, + "content": "v1=635", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1840, + "content": "v1=636", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1841, + "content": "v1=637", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1842, + "content": "v1=638", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1843, + "content": "v1=639", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1844, + "content": "v1=640", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1845, + "content": "v1=641", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1846, + "content": "v1=642", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1847, + "content": "v1=643", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1848, + "content": "v1=644", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1849, + "content": "v1=645", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1850, + "content": "v1=646", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1851, + "content": "v1=647", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1852, + "content": "v1=648", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1853, + "content": "v1=649", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1854, + "content": "v1=650", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1855, + "content": "v1=651", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1856, + "content": "v1=652", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1857, + "content": "v1=653", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1858, + "content": "v1=654", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1859, + "content": "v1=655", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1860, + "content": "v1=656", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1861, + "content": "v1=657", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1862, + "content": "v1=658", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1863, + "content": "v1=659", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1864, + "content": "v1=660", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1865, + "content": "v1=661", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1866, + "content": "v1=662", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1867, + "content": "v1=663", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1868, + "content": "v1=664", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1869, + "content": "v1=665", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1870, + "content": "v1=666", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1871, + "content": "v1=667", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1872, + "content": "v1=668", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1873, + "content": "v1=669", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1874, + "content": "v1=670", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1875, + "content": "v1=671", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1876, + "content": "v1=672", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1877, + "content": "v1=673", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1878, + "content": "v1=674", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1879, + "content": "v1=675", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1880, + "content": "v1=676", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1881, + "content": "v1=677", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1882, + "content": "v1=678", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1883, + "content": "v1=679", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1884, + "content": "v1=680", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1885, + "content": "v1=681", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1886, + "content": "v1=682", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1887, + "content": "v1=683", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1888, + "content": "v1=684", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1889, + "content": "v1=685", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1890, + "content": "v1=686", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1891, + "content": "v1=687", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1892, + "content": "v1=688", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1893, + "content": "v1=689", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1894, + "content": "v1=690", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1895, + "content": "v1=691", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1896, + "content": "v1=692", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1897, + "content": "v1=693", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1898, + "content": "v1=694", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1899, + "content": "v1=695", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1900, + "content": "v1=696", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1901, + "content": "v1=697", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1902, + "content": "v1=698", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1903, + "content": "v1=699", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1904, + "content": "v1=700", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1905, + "content": "v1=701", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1906, + "content": "v1=702", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1907, + "content": "v1=703", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1908, + "content": "v1=704", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1909, + "content": "v1=705", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1910, + "content": "v1=706", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1911, + "content": "v1=707", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1912, + "content": "v1=708", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1913, + "content": "v1=709", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1914, + "content": "v1=710", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1915, + "content": "v1=711", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1916, + "content": "v1=712", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1917, + "content": "v1=713", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1918, + "content": "v1=714", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1919, + "content": "v1=715", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1920, + "content": "v1=716", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1921, + "content": "v1=717", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1922, + "content": "v1=718", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1923, + "content": "v1=719", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1924, + "content": "v1=720", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1925, + "content": "v1=721", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1926, + "content": "v1=722", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1927, + "content": "v1=723", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1928, + "content": "v1=724", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1929, + "content": "v1=725", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1930, + "content": "v1=726", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1931, + "content": "v1=727", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1932, + "content": "v1=728", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1933, + "content": "v1=729", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1934, + "content": "v1=730", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1935, + "content": "v1=731", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1936, + "content": "v1=732", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1937, + "content": "v1=733", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1938, + "content": "v1=734", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1939, + "content": "v1=735", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1940, + "content": "v1=736", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1941, + "content": "v1=737", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1942, + "content": "v1=738", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1943, + "content": "v1=739", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1944, + "content": "v1=740", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1945, + "content": "v1=741", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1946, + "content": "v1=742", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1947, + "content": "v1=743", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1948, + "content": "v1=744", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1949, + "content": "v1=745", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1950, + "content": "v1=746", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1951, + "content": "v1=747", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1952, + "content": "v1=748", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1953, + "content": "v1=749", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1954, + "content": "v1=750", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1955, + "content": "v1=751", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1956, + "content": "v1=752", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1957, + "content": "v1=753", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1958, + "content": "v1=754", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1959, + "content": "v1=755", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1960, + "content": "v1=756", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1961, + "content": "v1=757", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1962, + "content": "v1=758", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1963, + "content": "v1=759", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1964, + "content": "v1=760", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1965, + "content": "v1=761", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1966, + "content": "v1=762", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1967, + "content": "v1=763", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1968, + "content": "v1=764", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1969, + "content": "v1=765", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1970, + "content": "v1=766", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1971, + "content": "v1=767", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1972, + "content": "v1=768", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1973, + "content": "v1=769", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1974, + "content": "v1=770", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1975, + "content": "v1=771", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1976, + "content": "v1=772", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1977, + "content": "v1=773", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1978, + "content": "v1=774", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1979, + "content": "v1=775", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1980, + "content": "v1=776", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1981, + "content": "v1=777", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1982, + "content": "v1=778", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1983, + "content": "v1=779", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1984, + "content": "v1=780", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1985, + "content": "v1=781", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1986, + "content": "v1=782", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1987, + "content": "v1=783", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1988, + "content": "v1=784", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1989, + "content": "v1=785", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1990, + "content": "v1=786", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1991, + "content": "v1=787", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1992, + "content": "v1=788", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1993, + "content": "v1=789", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1994, + "content": "v1=790", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1995, + "content": "v1=791", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1996, + "content": "v1=792", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1997, + "content": "v1=793", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1998, + "content": "v1=794", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1999, + "content": "v1=795", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2000, + "content": "v1=796", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2001, + "content": "v1=797", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2002, + "content": "v1=798", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2003, + "content": "v1=799", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2004, + "content": "v1=800", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2005, + "content": "v1=801", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2006, + "content": "v1=802", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2007, + "content": "v1=803", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2008, + "content": "v1=804", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2009, + "content": "v1=805", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2010, + "content": "v1=806", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2011, + "content": "v1=807", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2012, + "content": "v1=808", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2013, + "content": "v1=809", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2014, + "content": "v1=810", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2015, + "content": "v1=811", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2016, + "content": "v1=812", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2017, + "content": "v1=813", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2018, + "content": "v1=814", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2019, + "content": "v1=815", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2020, + "content": "v1=816", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2021, + "content": "v1=817", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2022, + "content": "v1=818", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2023, + "content": "v1=819", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2024, + "content": "v1=820", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2025, + "content": "v1=821", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2026, + "content": "v1=822", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2027, + "content": "v1=823", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2028, + "content": "v1=824", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2029, + "content": "v1=825", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2030, + "content": "v1=826", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2031, + "content": "v1=827", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2032, + "content": "v1=828", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2033, + "content": "v1=829", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2034, + "content": "v1=830", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2035, + "content": "v1=831", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2036, + "content": "v1=832", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2037, + "content": "v1=833", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2038, + "content": "v1=834", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2039, + "content": "v1=835", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2040, + "content": "v1=836", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2041, + "content": "v1=837", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2042, + "content": "v1=838", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2043, + "content": "v1=839", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2044, + "content": "v1=840", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2045, + "content": "v1=841", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2046, + "content": "v1=842", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2047, + "content": "v1=843", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2048, + "content": "v1=844", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2049, + "content": "v1=845", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2050, + "content": "v1=846", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2051, + "content": "v1=847", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2052, + "content": "v1=848", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2053, + "content": "v1=849", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2054, + "content": "v1=850", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2055, + "content": "v1=851", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2056, + "content": "v1=852", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2057, + "content": "v1=853", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2058, + "content": "v1=854", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2059, + "content": "v1=855", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2060, + "content": "v1=856", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2061, + "content": "v1=857", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2062, + "content": "v1=858", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2063, + "content": "v1=859", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2064, + "content": "v1=860", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2065, + "content": "v1=861", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2066, + "content": "v1=862", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2067, + "content": "v1=863", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2068, + "content": "v1=864", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2069, + "content": "v1=865", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2070, + "content": "v1=866", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2071, + "content": "v1=867", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2072, + "content": "v1=868", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2073, + "content": "v1=869", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2074, + "content": "v1=870", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2075, + "content": "v1=871", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2076, + "content": "v1=872", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2077, + "content": "v1=873", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2078, + "content": "v1=874", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2079, + "content": "v1=875", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2080, + "content": "v1=876", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2081, + "content": "v1=877", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2082, + "content": "v1=878", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2083, + "content": "v1=879", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2084, + "content": "v1=880", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2085, + "content": "v1=881", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2086, + "content": "v1=882", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2087, + "content": "v1=883", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2088, + "content": "v1=884", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2089, + "content": "v1=885", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2090, + "content": "v1=886", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2091, + "content": "v1=887", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2092, + "content": "v1=888", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2093, + "content": "v1=889", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2094, + "content": "v1=890", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2095, + "content": "v1=891", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2096, + "content": "v1=892", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2097, + "content": "v1=893", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2098, + "content": "v1=894", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2099, + "content": "v1=895", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2100, + "content": "v1=896", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2101, + "content": "v1=897", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2102, + "content": "v1=898", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2103, + "content": "v1=899", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2104, + "content": "v1=900", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2105, + "content": "v1=901", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2106, + "content": "v1=902", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2107, + "content": "v1=903", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2108, + "content": "v1=904", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2109, + "content": "v1=905", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2110, + "content": "v1=906", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2111, + "content": "v1=907", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2112, + "content": "v1=908", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2113, + "content": "v1=909", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2114, + "content": "v1=910", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2115, + "content": "v1=911", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2116, + "content": "v1=912", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2117, + "content": "v1=913", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2118, + "content": "v1=914", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2119, + "content": "v1=915", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2120, + "content": "v1=916", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2121, + "content": "v1=917", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2122, + "content": "v1=918", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2123, + "content": "v1=919", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2124, + "content": "v1=920", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2125, + "content": "v1=921", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2126, + "content": "v1=922", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2127, + "content": "v1=923", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2128, + "content": "v1=924", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2129, + "content": "v1=925", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2130, + "content": "v1=926", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2131, + "content": "v1=927", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2132, + "content": "v1=928", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2133, + "content": "v1=929", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2134, + "content": "v1=930", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2135, + "content": "v1=931", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2136, + "content": "v1=932", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2137, + "content": "v1=933", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2138, + "content": "v1=934", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2139, + "content": "v1=935", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2140, + "content": "v1=936", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2141, + "content": "v1=937", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2142, + "content": "v1=938", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2143, + "content": "v1=939", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2144, + "content": "v1=940", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2145, + "content": "v1=941", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2146, + "content": "v1=942", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2147, + "content": "v1=943", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2148, + "content": "v1=944", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2149, + "content": "v1=945", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2150, + "content": "v1=946", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2151, + "content": "v1=947", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2152, + "content": "v1=948", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2153, + "content": "v1=949", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2154, + "content": "v1=950", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2155, + "content": "v1=951", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2156, + "content": "v1=952", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2157, + "content": "v1=953", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2158, + "content": "v1=954", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2159, + "content": "v1=955", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2160, + "content": "v1=956", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2161, + "content": "v1=957", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2162, + "content": "v1=958", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2163, + "content": "v1=959", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2164, + "content": "v1=960", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2165, + "content": "v1=961", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2166, + "content": "v1=962", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2167, + "content": "v1=963", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2168, + "content": "v1=964", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2169, + "content": "v1=965", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2170, + "content": "v1=966", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2171, + "content": "v1=967", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2172, + "content": "v1=968", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2173, + "content": "v1=969", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2174, + "content": "v1=970", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2175, + "content": "v1=971", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2176, + "content": "v1=972", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2177, + "content": "v1=973", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2178, + "content": "v1=974", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2179, + "content": "v1=975", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2180, + "content": "v1=976", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2181, + "content": "v1=977", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2182, + "content": "v1=978", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2183, + "content": "v1=979", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2184, + "content": "v1=980", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2185, + "content": "v1=981", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2186, + "content": "v1=982", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2187, + "content": "v1=983", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2188, + "content": "v1=984", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2189, + "content": "v1=985", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2190, + "content": "v1=986", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2191, + "content": "v1=987", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2192, + "content": "v1=988", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2193, + "content": "v1=989", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2194, + "content": "v1=990", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2195, + "content": "v1=991", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2196, + "content": "v1=992", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2197, + "content": "v1=993", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2198, + "content": "v1=994", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2199, + "content": "v1=995", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2200, + "content": "v1=996", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2201, + "content": "v1=997", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2202, + "content": "v1=998", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2203, + "content": "v1=999", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2204, + "content": "v2=0", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2205, + "content": "v2=1", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2206, + "content": "v2=2", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2207, + "content": "v2=3", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2208, + "content": "v2=4", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2209, + "content": "v2=5", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2210, + "content": "v2=6", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2211, + "content": "v2=7", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2212, + "content": "v2=8", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2213, + "content": "v2=9", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2214, + "content": "v2=10", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2215, + "content": "v2=11", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2216, + "content": "v2=12", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2217, + "content": "v2=13", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2218, + "content": "v2=14", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2219, + "content": "v2=15", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2220, + "content": "v2=16", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2221, + "content": "v2=17", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2222, + "content": "v2=18", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2223, + "content": "v2=19", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2224, + "content": "v2=20", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2225, + "content": "v2=21", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2226, + "content": "v2=22", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2227, + "content": "v2=23", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2228, + "content": "v2=24", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2229, + "content": "v2=25", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2230, + "content": "v2=26", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2231, + "content": "v2=27", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2232, + "content": "v2=28", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2233, + "content": "v2=29", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2234, + "content": "v2=30", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2235, + "content": "v2=31", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2236, + "content": "v2=32", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2237, + "content": "v2=33", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2238, + "content": "v2=34", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2239, + "content": "v2=35", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2240, + "content": "v2=36", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2241, + "content": "v2=37", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2242, + "content": "v2=38", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2243, + "content": "v2=39", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2244, + "content": "v2=40", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2245, + "content": "v2=41", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2246, + "content": "v2=42", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2247, + "content": "v2=43", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2248, + "content": "v2=44", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2249, + "content": "v2=45", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2250, + "content": "v2=46", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2251, + "content": "v2=47", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2252, + "content": "v2=48", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2253, + "content": "v2=49", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2254, + "content": "v2=50", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2255, + "content": "v2=51", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2256, + "content": "v2=52", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2257, + "content": "v2=53", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2258, + "content": "v2=54", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2259, + "content": "v2=55", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2260, + "content": "v2=56", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2261, + "content": "v2=57", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2262, + "content": "v2=58", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2263, + "content": "v2=59", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2264, + "content": "v2=60", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2265, + "content": "v2=61", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2266, + "content": "v2=62", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2267, + "content": "v2=63", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2268, + "content": "v2=64", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2269, + "content": "v2=65", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2270, + "content": "v2=66", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2271, + "content": "v2=67", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2272, + "content": "v2=68", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2273, + "content": "v2=69", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2274, + "content": "v2=70", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2275, + "content": "v2=71", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2276, + "content": "v2=72", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2277, + "content": "v2=73", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2278, + "content": "v2=74", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2279, + "content": "v2=75", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2280, + "content": "v2=76", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2281, + "content": "v2=77", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2282, + "content": "v2=78", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2283, + "content": "v2=79", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2284, + "content": "v2=80", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2285, + "content": "v2=81", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2286, + "content": "v2=82", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2287, + "content": "v2=83", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2288, + "content": "v2=84", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2289, + "content": "v2=85", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2290, + "content": "v2=86", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2291, + "content": "v2=87", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2292, + "content": "v2=88", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2293, + "content": "v2=89", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2294, + "content": "v2=90", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2295, + "content": "v2=91", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2296, + "content": "v2=92", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2297, + "content": "v2=93", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2298, + "content": "v2=94", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2299, + "content": "v2=95", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2300, + "content": "v2=96", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2301, + "content": "v2=97", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2302, + "content": "v2=98", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2303, + "content": "v2=99", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2304, + "content": "v2=100", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2305, + "content": "v2=101", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2306, + "content": "v2=102", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2307, + "content": "v2=103", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2308, + "content": "v2=104", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2309, + "content": "v2=105", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2310, + "content": "v2=106", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2311, + "content": "v2=107", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2312, + "content": "v2=108", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2313, + "content": "v2=109", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2314, + "content": "v2=110", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2315, + "content": "v2=111", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2316, + "content": "v2=112", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2317, + "content": "v2=113", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2318, + "content": "v2=114", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2319, + "content": "v2=115", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2320, + "content": "v2=116", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2321, + "content": "v2=117", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2322, + "content": "v2=118", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2323, + "content": "v2=119", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2324, + "content": "v2=120", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2325, + "content": "v2=121", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2326, + "content": "v2=122", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2327, + "content": "v2=123", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2328, + "content": "v2=124", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2329, + "content": "v2=125", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2330, + "content": "v2=126", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2331, + "content": "v2=127", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2332, + "content": "v2=128", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2333, + "content": "v2=129", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2334, + "content": "v2=130", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2335, + "content": "v2=131", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2336, + "content": "v2=132", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2337, + "content": "v2=133", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2338, + "content": "v2=134", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2339, + "content": "v2=135", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2340, + "content": "v2=136", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2341, + "content": "v2=137", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2342, + "content": "v2=138", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2343, + "content": "v2=139", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2344, + "content": "v2=140", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2345, + "content": "v2=141", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2346, + "content": "v2=142", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2347, + "content": "v2=143", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2348, + "content": "v2=144", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2349, + "content": "v2=145", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2350, + "content": "v2=146", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2351, + "content": "v2=147", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2352, + "content": "v2=148", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2353, + "content": "v2=149", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2354, + "content": "v2=150", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2355, + "content": "v2=151", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2356, + "content": "v2=152", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2357, + "content": "v2=153", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2358, + "content": "v2=154", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2359, + "content": "v2=155", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2360, + "content": "v2=156", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2361, + "content": "v2=157", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2362, + "content": "v2=158", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2363, + "content": "v2=159", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2364, + "content": "v2=160", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2365, + "content": "v2=161", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2366, + "content": "v2=162", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2367, + "content": "v2=163", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2368, + "content": "v2=164", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2369, + "content": "v2=165", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2370, + "content": "v2=166", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2371, + "content": "v2=167", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2372, + "content": "v2=168", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2373, + "content": "v2=169", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2374, + "content": "v2=170", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2375, + "content": "v2=171", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2376, + "content": "v2=172", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2377, + "content": "v2=173", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2378, + "content": "v2=174", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2379, + "content": "v2=175", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2380, + "content": "v2=176", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2381, + "content": "v2=177", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2382, + "content": "v2=178", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2383, + "content": "v2=179", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2384, + "content": "v2=180", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2385, + "content": "v2=181", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2386, + "content": "v2=182", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2387, + "content": "v2=183", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2388, + "content": "v2=184", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2389, + "content": "v2=185", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2390, + "content": "v2=186", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2391, + "content": "v2=187", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2392, + "content": "v2=188", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2393, + "content": "v2=189", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2394, + "content": "v2=190", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2395, + "content": "v2=191", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2396, + "content": "v2=192", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2397, + "content": "v2=193", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2398, + "content": "v2=194", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2399, + "content": "v2=195", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2400, + "content": "v2=196", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2401, + "content": "v2=197", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2402, + "content": "v2=198", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2403, + "content": "v2=199", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2404, + "content": "v2=200", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2405, + "content": "v2=201", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2406, + "content": "v2=202", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2407, + "content": "v2=203", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2408, + "content": "v2=204", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2409, + "content": "v2=205", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2410, + "content": "v2=206", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2411, + "content": "v2=207", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2412, + "content": "v2=208", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2413, + "content": "v2=209", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2414, + "content": "v2=210", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2415, + "content": "v2=211", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2416, + "content": "v2=212", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2417, + "content": "v2=213", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2418, + "content": "v2=214", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2419, + "content": "v2=215", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2420, + "content": "v2=216", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2421, + "content": "v2=217", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2422, + "content": "v2=218", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2423, + "content": "v2=219", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2424, + "content": "v2=220", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2425, + "content": "v2=221", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2426, + "content": "v2=222", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2427, + "content": "v2=223", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2428, + "content": "v2=224", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2429, + "content": "v2=225", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2430, + "content": "v2=226", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2431, + "content": "v2=227", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2432, + "content": "v2=228", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2433, + "content": "v2=229", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2434, + "content": "v2=230", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2435, + "content": "v2=231", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2436, + "content": "v2=232", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2437, + "content": "v2=233", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2438, + "content": "v2=234", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2439, + "content": "v2=235", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2440, + "content": "v2=236", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2441, + "content": "v2=237", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2442, + "content": "v2=238", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2443, + "content": "v2=239", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2444, + "content": "v2=240", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2445, + "content": "v2=241", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2446, + "content": "v2=242", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2447, + "content": "v2=243", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2448, + "content": "v2=244", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2449, + "content": "v2=245", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2450, + "content": "v2=246", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2451, + "content": "v2=247", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2452, + "content": "v2=248", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2453, + "content": "v2=249", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2454, + "content": "v2=250", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2455, + "content": "v2=251", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2456, + "content": "v2=252", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2457, + "content": "v2=253", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2458, + "content": "v2=254", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2459, + "content": "v2=255", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2460, + "content": "v2=256", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2461, + "content": "v2=257", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2462, + "content": "v2=258", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2463, + "content": "v2=259", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2464, + "content": "v2=260", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2465, + "content": "v2=261", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2466, + "content": "v2=262", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2467, + "content": "v2=263", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2468, + "content": "v2=264", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2469, + "content": "v2=265", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2470, + "content": "v2=266", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2471, + "content": "v2=267", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2472, + "content": "v2=268", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2473, + "content": "v2=269", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2474, + "content": "v2=270", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2475, + "content": "v2=271", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2476, + "content": "v2=272", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2477, + "content": "v2=273", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2478, + "content": "v2=274", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2479, + "content": "v2=275", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2480, + "content": "v2=276", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2481, + "content": "v2=277", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2482, + "content": "v2=278", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2483, + "content": "v2=279", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2484, + "content": "v2=280", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2485, + "content": "v2=281", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2486, + "content": "v2=282", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2487, + "content": "v2=283", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2488, + "content": "v2=284", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2489, + "content": "v2=285", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2490, + "content": "v2=286", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2491, + "content": "v2=287", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2492, + "content": "v2=288", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2493, + "content": "v2=289", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2494, + "content": "v2=290", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2495, + "content": "v2=291", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2496, + "content": "v2=292", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2497, + "content": "v2=293", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2498, + "content": "v2=294", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2499, + "content": "v2=295", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2500, + "content": "v2=296", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2501, + "content": "v2=297", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2502, + "content": "v2=298", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2503, + "content": "v2=299", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2504, + "content": "v2=300", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2505, + "content": "v2=301", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2506, + "content": "v2=302", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2507, + "content": "v2=303", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2508, + "content": "v2=304", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2509, + "content": "v2=305", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2510, + "content": "v2=306", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2511, + "content": "v2=307", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2512, + "content": "v2=308", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2513, + "content": "v2=309", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2514, + "content": "v2=310", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2515, + "content": "v2=311", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2516, + "content": "v2=312", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2517, + "content": "v2=313", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2518, + "content": "v2=314", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2519, + "content": "v2=315", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2520, + "content": "v2=316", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2521, + "content": "v2=317", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2522, + "content": "v2=318", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2523, + "content": "v2=319", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2524, + "content": "v2=320", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2525, + "content": "v2=321", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2526, + "content": "v2=322", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2527, + "content": "v2=323", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2528, + "content": "v2=324", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2529, + "content": "v2=325", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2530, + "content": "v2=326", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2531, + "content": "v2=327", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2532, + "content": "v2=328", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2533, + "content": "v2=329", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2534, + "content": "v2=330", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2535, + "content": "v2=331", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2536, + "content": "v2=332", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2537, + "content": "v2=333", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2538, + "content": "v2=334", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2539, + "content": "v2=335", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2540, + "content": "v2=336", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2541, + "content": "v2=337", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2542, + "content": "v2=338", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2543, + "content": "v2=339", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2544, + "content": "v2=340", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2545, + "content": "v2=341", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2546, + "content": "v2=342", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2547, + "content": "v2=343", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2548, + "content": "v2=344", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2549, + "content": "v2=345", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2550, + "content": "v2=346", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2551, + "content": "v2=347", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2552, + "content": "v2=348", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2553, + "content": "v2=349", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2554, + "content": "v2=350", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2555, + "content": "v2=351", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2556, + "content": "v2=352", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2557, + "content": "v2=353", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2558, + "content": "v2=354", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2559, + "content": "v2=355", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2560, + "content": "v2=356", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2561, + "content": "v2=357", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2562, + "content": "v2=358", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2563, + "content": "v2=359", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2564, + "content": "v2=360", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2565, + "content": "v2=361", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2566, + "content": "v2=362", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2567, + "content": "v2=363", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2568, + "content": "v2=364", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2569, + "content": "v2=365", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2570, + "content": "v2=366", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2571, + "content": "v2=367", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2572, + "content": "v2=368", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2573, + "content": "v2=369", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2574, + "content": "v2=370", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2575, + "content": "v2=371", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2576, + "content": "v2=372", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2577, + "content": "v2=373", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2578, + "content": "v2=374", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2579, + "content": "v2=375", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2580, + "content": "v2=376", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2581, + "content": "v2=377", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2582, + "content": "v2=378", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2583, + "content": "v2=379", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2584, + "content": "v2=380", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2585, + "content": "v2=381", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2586, + "content": "v2=382", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2587, + "content": "v2=383", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2588, + "content": "v2=384", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2589, + "content": "v2=385", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2590, + "content": "v2=386", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2591, + "content": "v2=387", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2592, + "content": "v2=388", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2593, + "content": "v2=389", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2594, + "content": "v2=390", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2595, + "content": "v2=391", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2596, + "content": "v2=392", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2597, + "content": "v2=393", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2598, + "content": "v2=394", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2599, + "content": "v2=395", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2600, + "content": "v2=396", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2601, + "content": "v2=397", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2602, + "content": "v2=398", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2603, + "content": "v2=399", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2604, + "content": "v2=400", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2605, + "content": "v2=401", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2606, + "content": "v2=402", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2607, + "content": "v2=403", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2608, + "content": "v2=404", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2609, + "content": "v2=405", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2610, + "content": "v2=406", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2611, + "content": "v2=407", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2612, + "content": "v2=408", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2613, + "content": "v2=409", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2614, + "content": "v2=410", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2615, + "content": "v2=411", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2616, + "content": "v2=412", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2617, + "content": "v2=413", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2618, + "content": "v2=414", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2619, + "content": "v2=415", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2620, + "content": "v2=416", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2621, + "content": "v2=417", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2622, + "content": "v2=418", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2623, + "content": "v2=419", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2624, + "content": "v2=420", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2625, + "content": "v2=421", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2626, + "content": "v2=422", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2627, + "content": "v2=423", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2628, + "content": "v2=424", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2629, + "content": "v2=425", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2630, + "content": "v2=426", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2631, + "content": "v2=427", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2632, + "content": "v2=428", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2633, + "content": "v2=429", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2634, + "content": "v2=430", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2635, + "content": "v2=431", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2636, + "content": "v2=432", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2637, + "content": "v2=433", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2638, + "content": "v2=434", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2639, + "content": "v2=435", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2640, + "content": "v2=436", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2641, + "content": "v2=437", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2642, + "content": "v2=438", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2643, + "content": "v2=439", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2644, + "content": "v2=440", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2645, + "content": "v2=441", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2646, + "content": "v2=442", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2647, + "content": "v2=443", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2648, + "content": "v2=444", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2649, + "content": "v2=445", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2650, + "content": "v2=446", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2651, + "content": "v2=447", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2652, + "content": "v2=448", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2653, + "content": "v2=449", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2654, + "content": "v2=450", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2655, + "content": "v2=451", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2656, + "content": "v2=452", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2657, + "content": "v2=453", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2658, + "content": "v2=454", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2659, + "content": "v2=455", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2660, + "content": "v2=456", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2661, + "content": "v2=457", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2662, + "content": "v2=458", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2663, + "content": "v2=459", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2664, + "content": "v2=460", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2665, + "content": "v2=461", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2666, + "content": "v2=462", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2667, + "content": "v2=463", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2668, + "content": "v2=464", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2669, + "content": "v2=465", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2670, + "content": "v2=466", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2671, + "content": "v2=467", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2672, + "content": "v2=468", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2673, + "content": "v2=469", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2674, + "content": "v2=470", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2675, + "content": "v2=471", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2676, + "content": "v2=472", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2677, + "content": "v2=473", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2678, + "content": "v2=474", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2679, + "content": "v2=475", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2680, + "content": "v2=476", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2681, + "content": "v2=477", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2682, + "content": "v2=478", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2683, + "content": "v2=479", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2684, + "content": "v2=480", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2685, + "content": "v2=481", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2686, + "content": "v2=482", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2687, + "content": "v2=483", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2688, + "content": "v2=484", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2689, + "content": "v2=485", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2690, + "content": "v2=486", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2691, + "content": "v2=487", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2692, + "content": "v2=488", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2693, + "content": "v2=489", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2694, + "content": "v2=490", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2695, + "content": "v2=491", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2696, + "content": "v2=492", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2697, + "content": "v2=493", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2698, + "content": "v2=494", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2699, + "content": "v2=495", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2700, + "content": "v2=496", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2701, + "content": "v2=497", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2702, + "content": "v2=498", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2703, + "content": "v2=499", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2704, + "content": "v2=500", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2705, + "content": "v2=501", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2706, + "content": "v2=502", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2707, + "content": "v2=503", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2708, + "content": "v2=504", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2709, + "content": "v2=505", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2710, + "content": "v2=506", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2711, + "content": "v2=507", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2712, + "content": "v2=508", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2713, + "content": "v2=509", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2714, + "content": "v2=510", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2715, + "content": "v2=511", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2716, + "content": "v2=512", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2717, + "content": "v2=513", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2718, + "content": "v2=514", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2719, + "content": "v2=515", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2720, + "content": "v2=516", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2721, + "content": "v2=517", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2722, + "content": "v2=518", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2723, + "content": "v2=519", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2724, + "content": "v2=520", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2725, + "content": "v2=521", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2726, + "content": "v2=522", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2727, + "content": "v2=523", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2728, + "content": "v2=524", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2729, + "content": "v2=525", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2730, + "content": "v2=526", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2731, + "content": "v2=527", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2732, + "content": "v2=528", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2733, + "content": "v2=529", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2734, + "content": "v2=530", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2735, + "content": "v2=531", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2736, + "content": "v2=532", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2737, + "content": "v2=533", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2738, + "content": "v2=534", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2739, + "content": "v2=535", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2740, + "content": "v2=536", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2741, + "content": "v2=537", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2742, + "content": "v2=538", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2743, + "content": "v2=539", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2744, + "content": "v2=540", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2745, + "content": "v2=541", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2746, + "content": "v2=542", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2747, + "content": "v2=543", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2748, + "content": "v2=544", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2749, + "content": "v2=545", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2750, + "content": "v2=546", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2751, + "content": "v2=547", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2752, + "content": "v2=548", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2753, + "content": "v2=549", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2754, + "content": "v2=550", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2755, + "content": "v2=551", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2756, + "content": "v2=552", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2757, + "content": "v2=553", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2758, + "content": "v2=554", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2759, + "content": "v2=555", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2760, + "content": "v2=556", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2761, + "content": "v2=557", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2762, + "content": "v2=558", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2763, + "content": "v2=559", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2764, + "content": "v2=560", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2765, + "content": "v2=561", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2766, + "content": "v2=562", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2767, + "content": "v2=563", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2768, + "content": "v2=564", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2769, + "content": "v2=565", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2770, + "content": "v2=566", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2771, + "content": "v2=567", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2772, + "content": "v2=568", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2773, + "content": "v2=569", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2774, + "content": "v2=570", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2775, + "content": "v2=571", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2776, + "content": "v2=572", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2777, + "content": "v2=573", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2778, + "content": "v2=574", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2779, + "content": "v2=575", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2780, + "content": "v2=576", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2781, + "content": "v2=577", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2782, + "content": "v2=578", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2783, + "content": "v2=579", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2784, + "content": "v2=580", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2785, + "content": "v2=581", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2786, + "content": "v2=582", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2787, + "content": "v2=583", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2788, + "content": "v2=584", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2789, + "content": "v2=585", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2790, + "content": "v2=586", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2791, + "content": "v2=587", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2792, + "content": "v2=588", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2793, + "content": "v2=589", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2794, + "content": "v2=590", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2795, + "content": "v2=591", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2796, + "content": "v2=592", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2797, + "content": "v2=593", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2798, + "content": "v2=594", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2799, + "content": "v2=595", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2800, + "content": "v2=596", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2801, + "content": "v2=597", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2802, + "content": "v2=598", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2803, + "content": "v2=599", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2804, + "content": "v2=600", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2805, + "content": "v2=601", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2806, + "content": "v2=602", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2807, + "content": "v2=603", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2808, + "content": "v2=604", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2809, + "content": "v2=605", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2810, + "content": "v2=606", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2811, + "content": "v2=607", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2812, + "content": "v2=608", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2813, + "content": "v2=609", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2814, + "content": "v2=610", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2815, + "content": "v2=611", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2816, + "content": "v2=612", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2817, + "content": "v2=613", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2818, + "content": "v2=614", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2819, + "content": "v2=615", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2820, + "content": "v2=616", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2821, + "content": "v2=617", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2822, + "content": "v2=618", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2823, + "content": "v2=619", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2824, + "content": "v2=620", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2825, + "content": "v2=621", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2826, + "content": "v2=622", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2827, + "content": "v2=623", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2828, + "content": "v2=624", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2829, + "content": "v2=625", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2830, + "content": "v2=626", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2831, + "content": "v2=627", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2832, + "content": "v2=628", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2833, + "content": "v2=629", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2834, + "content": "v2=630", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2835, + "content": "v2=631", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2836, + "content": "v2=632", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2837, + "content": "v2=633", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2838, + "content": "v2=634", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2839, + "content": "v2=635", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2840, + "content": "v2=636", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2841, + "content": "v2=637", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2842, + "content": "v2=638", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2843, + "content": "v2=639", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2844, + "content": "v2=640", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2845, + "content": "v2=641", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2846, + "content": "v2=642", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2847, + "content": "v2=643", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2848, + "content": "v2=644", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2849, + "content": "v2=645", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2850, + "content": "v2=646", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2851, + "content": "v2=647", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2852, + "content": "v2=648", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2853, + "content": "v2=649", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2854, + "content": "v2=650", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2855, + "content": "v2=651", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2856, + "content": "v2=652", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2857, + "content": "v2=653", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2858, + "content": "v2=654", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2859, + "content": "v2=655", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2860, + "content": "v2=656", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2861, + "content": "v2=657", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2862, + "content": "v2=658", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2863, + "content": "v2=659", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2864, + "content": "v2=660", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2865, + "content": "v2=661", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2866, + "content": "v2=662", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2867, + "content": "v2=663", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2868, + "content": "v2=664", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2869, + "content": "v2=665", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2870, + "content": "v2=666", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2871, + "content": "v2=667", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2872, + "content": "v2=668", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2873, + "content": "v2=669", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2874, + "content": "v2=670", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2875, + "content": "v2=671", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2876, + "content": "v2=672", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2877, + "content": "v2=673", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2878, + "content": "v2=674", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2879, + "content": "v2=675", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2880, + "content": "v2=676", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2881, + "content": "v2=677", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2882, + "content": "v2=678", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2883, + "content": "v2=679", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2884, + "content": "v2=680", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2885, + "content": "v2=681", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2886, + "content": "v2=682", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2887, + "content": "v2=683", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2888, + "content": "v2=684", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2889, + "content": "v2=685", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2890, + "content": "v2=686", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2891, + "content": "v2=687", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2892, + "content": "v2=688", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2893, + "content": "v2=689", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2894, + "content": "v2=690", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2895, + "content": "v2=691", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2896, + "content": "v2=692", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2897, + "content": "v2=693", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2898, + "content": "v2=694", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2899, + "content": "v2=695", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2900, + "content": "v2=696", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2901, + "content": "v2=697", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2902, + "content": "v2=698", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2903, + "content": "v2=699", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2904, + "content": "v2=700", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2905, + "content": "v2=701", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2906, + "content": "v2=702", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2907, + "content": "v2=703", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2908, + "content": "v2=704", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2909, + "content": "v2=705", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2910, + "content": "v2=706", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2911, + "content": "v2=707", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2912, + "content": "v2=708", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2913, + "content": "v2=709", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2914, + "content": "v2=710", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2915, + "content": "v2=711", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2916, + "content": "v2=712", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2917, + "content": "v2=713", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2918, + "content": "v2=714", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2919, + "content": "v2=715", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2920, + "content": "v2=716", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2921, + "content": "v2=717", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2922, + "content": "v2=718", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2923, + "content": "v2=719", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2924, + "content": "v2=720", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2925, + "content": "v2=721", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2926, + "content": "v2=722", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2927, + "content": "v2=723", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2928, + "content": "v2=724", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2929, + "content": "v2=725", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2930, + "content": "v2=726", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2931, + "content": "v2=727", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2932, + "content": "v2=728", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2933, + "content": "v2=729", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2934, + "content": "v2=730", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2935, + "content": "v2=731", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2936, + "content": "v2=732", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2937, + "content": "v2=733", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2938, + "content": "v2=734", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2939, + "content": "v2=735", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2940, + "content": "v2=736", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2941, + "content": "v2=737", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2942, + "content": "v2=738", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2943, + "content": "v2=739", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2944, + "content": "v2=740", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2945, + "content": "v2=741", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2946, + "content": "v2=742", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2947, + "content": "v2=743", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2948, + "content": "v2=744", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2949, + "content": "v2=745", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2950, + "content": "v2=746", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2951, + "content": "v2=747", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2952, + "content": "v2=748", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2953, + "content": "v2=749", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2954, + "content": "v2=750", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2955, + "content": "v2=751", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2956, + "content": "v2=752", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2957, + "content": "v2=753", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2958, + "content": "v2=754", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2959, + "content": "v2=755", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2960, + "content": "v2=756", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2961, + "content": "v2=757", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2962, + "content": "v2=758", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2963, + "content": "v2=759", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2964, + "content": "v2=760", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2965, + "content": "v2=761", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2966, + "content": "v2=762", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2967, + "content": "v2=763", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2968, + "content": "v2=764", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2969, + "content": "v2=765", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2970, + "content": "v2=766", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2971, + "content": "v2=767", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2972, + "content": "v2=768", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2973, + "content": "v2=769", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2974, + "content": "v2=770", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2975, + "content": "v2=771", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2976, + "content": "v2=772", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2977, + "content": "v2=773", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2978, + "content": "v2=774", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2979, + "content": "v2=775", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2980, + "content": "v2=776", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2981, + "content": "v2=777", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2982, + "content": "v2=778", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2983, + "content": "v2=779", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2984, + "content": "v2=780", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2985, + "content": "v2=781", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2986, + "content": "v2=782", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2987, + "content": "v2=783", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2988, + "content": "v2=784", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2989, + "content": "v2=785", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2990, + "content": "v2=786", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2991, + "content": "v2=787", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2992, + "content": "v2=788", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2993, + "content": "v2=789", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2994, + "content": "v2=790", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2995, + "content": "v2=791", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2996, + "content": "v2=792", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2997, + "content": "v2=793", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2998, + "content": "v2=794", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2999, + "content": "v2=795", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3000, + "content": "v2=796", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3001, + "content": "v2=797", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3002, + "content": "v2=798", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3003, + "content": "v2=799", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3004, + "content": "v2=800", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3005, + "content": "v2=801", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3006, + "content": "v2=802", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3007, + "content": "v2=803", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3008, + "content": "v2=804", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3009, + "content": "v2=805", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3010, + "content": "v2=806", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3011, + "content": "v2=807", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3012, + "content": "v2=808", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3013, + "content": "v2=809", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3014, + "content": "v2=810", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3015, + "content": "v2=811", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3016, + "content": "v2=812", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3017, + "content": "v2=813", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3018, + "content": "v2=814", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3019, + "content": "v2=815", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3020, + "content": "v2=816", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3021, + "content": "v2=817", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3022, + "content": "v2=818", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3023, + "content": "v2=819", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3024, + "content": "v2=820", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3025, + "content": "v2=821", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3026, + "content": "v2=822", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3027, + "content": "v2=823", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3028, + "content": "v2=824", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3029, + "content": "v2=825", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3030, + "content": "v2=826", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3031, + "content": "v2=827", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3032, + "content": "v2=828", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3033, + "content": "v2=829", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3034, + "content": "v2=830", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3035, + "content": "v2=831", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3036, + "content": "v2=832", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3037, + "content": "v2=833", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3038, + "content": "v2=834", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3039, + "content": "v2=835", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3040, + "content": "v2=836", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3041, + "content": "v2=837", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3042, + "content": "v2=838", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3043, + "content": "v2=839", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3044, + "content": "v2=840", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3045, + "content": "v2=841", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3046, + "content": "v2=842", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3047, + "content": "v2=843", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3048, + "content": "v2=844", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3049, + "content": "v2=845", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3050, + "content": "v2=846", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3051, + "content": "v2=847", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3052, + "content": "v2=848", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3053, + "content": "v2=849", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3054, + "content": "v2=850", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3055, + "content": "v2=851", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3056, + "content": "v2=852", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3057, + "content": "v2=853", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3058, + "content": "v2=854", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3059, + "content": "v2=855", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3060, + "content": "v2=856", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3061, + "content": "v2=857", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3062, + "content": "v2=858", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3063, + "content": "v2=859", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3064, + "content": "v2=860", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3065, + "content": "v2=861", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3066, + "content": "v2=862", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3067, + "content": "v2=863", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3068, + "content": "v2=864", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3069, + "content": "v2=865", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3070, + "content": "v2=866", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3071, + "content": "v2=867", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3072, + "content": "v2=868", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3073, + "content": "v2=869", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3074, + "content": "v2=870", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3075, + "content": "v2=871", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3076, + "content": "v2=872", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3077, + "content": "v2=873", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3078, + "content": "v2=874", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3079, + "content": "v2=875", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3080, + "content": "v2=876", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3081, + "content": "v2=877", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3082, + "content": "v2=878", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3083, + "content": "v2=879", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3084, + "content": "v2=880", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3085, + "content": "v2=881", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3086, + "content": "v2=882", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3087, + "content": "v2=883", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3088, + "content": "v2=884", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3089, + "content": "v2=885", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3090, + "content": "v2=886", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3091, + "content": "v2=887", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3092, + "content": "v2=888", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3093, + "content": "v2=889", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3094, + "content": "v2=890", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3095, + "content": "v2=891", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3096, + "content": "v2=892", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3097, + "content": "v2=893", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3098, + "content": "v2=894", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3099, + "content": "v2=895", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3100, + "content": "v2=896", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3101, + "content": "v2=897", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3102, + "content": "v2=898", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3103, + "content": "v2=899", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3104, + "content": "v2=900", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3105, + "content": "v2=901", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3106, + "content": "v2=902", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3107, + "content": "v2=903", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3108, + "content": "v2=904", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3109, + "content": "v2=905", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3110, + "content": "v2=906", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3111, + "content": "v2=907", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3112, + "content": "v2=908", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3113, + "content": "v2=909", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3114, + "content": "v2=910", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3115, + "content": "v2=911", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3116, + "content": "v2=912", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3117, + "content": "v2=913", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3118, + "content": "v2=914", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3119, + "content": "v2=915", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3120, + "content": "v2=916", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3121, + "content": "v2=917", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3122, + "content": "v2=918", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3123, + "content": "v2=919", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3124, + "content": "v2=920", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3125, + "content": "v2=921", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3126, + "content": "v2=922", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3127, + "content": "v2=923", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3128, + "content": "v2=924", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3129, + "content": "v2=925", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3130, + "content": "v2=926", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3131, + "content": "v2=927", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3132, + "content": "v2=928", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3133, + "content": "v2=929", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3134, + "content": "v2=930", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3135, + "content": "v2=931", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3136, + "content": "v2=932", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3137, + "content": "v2=933", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3138, + "content": "v2=934", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3139, + "content": "v2=935", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3140, + "content": "v2=936", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3141, + "content": "v2=937", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3142, + "content": "v2=938", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3143, + "content": "v2=939", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3144, + "content": "v2=940", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3145, + "content": "v2=941", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3146, + "content": "v2=942", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3147, + "content": "v2=943", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3148, + "content": "v2=944", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3149, + "content": "v2=945", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3150, + "content": "v2=946", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3151, + "content": "v2=947", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3152, + "content": "v2=948", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3153, + "content": "v2=949", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3154, + "content": "v2=950", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3155, + "content": "v2=951", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3156, + "content": "v2=952", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3157, + "content": "v2=953", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3158, + "content": "v2=954", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3159, + "content": "v2=955", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3160, + "content": "v2=956", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3161, + "content": "v2=957", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3162, + "content": "v2=958", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3163, + "content": "v2=959", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3164, + "content": "v2=960", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3165, + "content": "v2=961", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3166, + "content": "v2=962", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3167, + "content": "v2=963", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3168, + "content": "v2=964", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3169, + "content": "v2=965", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3170, + "content": "v2=966", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3171, + "content": "v2=967", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3172, + "content": "v2=968", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3173, + "content": "v2=969", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3174, + "content": "v2=970", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3175, + "content": "v2=971", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3176, + "content": "v2=972", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3177, + "content": "v2=973", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3178, + "content": "v2=974", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3179, + "content": "v2=975", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3180, + "content": "v2=976", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3181, + "content": "v2=977", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3182, + "content": "v2=978", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3183, + "content": "v2=979", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3184, + "content": "v2=980", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3185, + "content": "v2=981", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3186, + "content": "v2=982", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3187, + "content": "v2=983", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3188, + "content": "v2=984", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3189, + "content": "v2=985", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3190, + "content": "v2=986", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3191, + "content": "v2=987", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3192, + "content": "v2=988", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3193, + "content": "v2=989", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3194, + "content": "v2=990", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3195, + "content": "v2=991", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3196, + "content": "v2=992", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3197, + "content": "v2=993", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3198, + "content": "v2=994", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3199, + "content": "v2=995", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3200, + "content": "v2=996", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3201, + "content": "v2=997", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3202, + "content": "v2=998", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3203, + "content": "v2=999", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3204, + "content": "v3=0", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3205, + "content": "v3=1", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3206, + "content": "v3=2", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3207, + "content": "v3=3", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3208, + "content": "v3=4", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3209, + "content": "v3=5", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3210, + "content": "v3=6", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3211, + "content": "v3=7", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3212, + "content": "v3=8", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3213, + "content": "v3=9", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3214, + "content": "v3=10", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3215, + "content": "v3=11", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3216, + "content": "v3=12", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3217, + "content": "v3=13", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3218, + "content": "v3=14", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3219, + "content": "v3=15", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3220, + "content": "v3=16", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3221, + "content": "v3=17", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3222, + "content": "v3=18", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3223, + "content": "v3=19", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3224, + "content": "v3=20", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3225, + "content": "v3=21", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3226, + "content": "v3=22", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3227, + "content": "v3=23", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3228, + "content": "v3=24", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3229, + "content": "v3=25", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3230, + "content": "v3=26", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3231, + "content": "v3=27", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3232, + "content": "v3=28", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3233, + "content": "v3=29", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3234, + "content": "v3=30", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3235, + "content": "v3=31", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3236, + "content": "v3=32", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3237, + "content": "v3=33", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3238, + "content": "v3=34", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3239, + "content": "v3=35", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3240, + "content": "v3=36", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3241, + "content": "v3=37", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3242, + "content": "v3=38", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3243, + "content": "v3=39", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3244, + "content": "v3=40", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3245, + "content": "v3=41", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3246, + "content": "v3=42", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3247, + "content": "v3=43", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3248, + "content": "v3=44", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3249, + "content": "v3=45", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3250, + "content": "v3=46", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3251, + "content": "v3=47", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3252, + "content": "v3=48", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3253, + "content": "v3=49", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3254, + "content": "v3=50", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3255, + "content": "v3=51", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3256, + "content": "v3=52", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3257, + "content": "v3=53", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3258, + "content": "v3=54", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3259, + "content": "v3=55", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3260, + "content": "v3=56", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3261, + "content": "v3=57", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3262, + "content": "v3=58", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3263, + "content": "v3=59", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3264, + "content": "v3=60", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3265, + "content": "v3=61", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3266, + "content": "v3=62", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3267, + "content": "v3=63", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3268, + "content": "v3=64", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3269, + "content": "v3=65", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3270, + "content": "v3=66", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3271, + "content": "v3=67", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3272, + "content": "v3=68", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3273, + "content": "v3=69", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3274, + "content": "v3=70", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3275, + "content": "v3=71", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3276, + "content": "v3=72", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3277, + "content": "v3=73", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3278, + "content": "v3=74", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3279, + "content": "v3=75", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3280, + "content": "v3=76", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3281, + "content": "v3=77", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3282, + "content": "v3=78", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3283, + "content": "v3=79", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3284, + "content": "v3=80", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3285, + "content": "v3=81", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3286, + "content": "v3=82", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3287, + "content": "v3=83", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3288, + "content": "v3=84", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3289, + "content": "v3=85", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3290, + "content": "v3=86", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3291, + "content": "v3=87", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3292, + "content": "v3=88", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3293, + "content": "v3=89", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3294, + "content": "v3=90", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3295, + "content": "v3=91", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3296, + "content": "v3=92", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3297, + "content": "v3=93", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3298, + "content": "v3=94", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3299, + "content": "v3=95", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3300, + "content": "v3=96", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3301, + "content": "v3=97", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3302, + "content": "v3=98", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3303, + "content": "v3=99", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3304, + "content": "v3=100", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3305, + "content": "v3=101", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3306, + "content": "v3=102", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3307, + "content": "v3=103", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3308, + "content": "v3=104", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3309, + "content": "v3=105", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3310, + "content": "v3=106", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3311, + "content": "v3=107", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3312, + "content": "v3=108", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3313, + "content": "v3=109", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3314, + "content": "v3=110", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3315, + "content": "v3=111", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3316, + "content": "v3=112", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3317, + "content": "v3=113", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3318, + "content": "v3=114", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3319, + "content": "v3=115", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3320, + "content": "v3=116", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3321, + "content": "v3=117", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3322, + "content": "v3=118", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3323, + "content": "v3=119", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3324, + "content": "v3=120", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3325, + "content": "v3=121", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3326, + "content": "v3=122", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3327, + "content": "v3=123", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3328, + "content": "v3=124", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3329, + "content": "v3=125", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3330, + "content": "v3=126", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3331, + "content": "v3=127", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3332, + "content": "v3=128", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3333, + "content": "v3=129", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3334, + "content": "v3=130", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3335, + "content": "v3=131", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3336, + "content": "v3=132", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3337, + "content": "v3=133", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3338, + "content": "v3=134", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3339, + "content": "v3=135", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3340, + "content": "v3=136", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3341, + "content": "v3=137", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3342, + "content": "v3=138", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3343, + "content": "v3=139", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3344, + "content": "v3=140", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3345, + "content": "v3=141", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3346, + "content": "v3=142", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3347, + "content": "v3=143", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3348, + "content": "v3=144", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3349, + "content": "v3=145", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3350, + "content": "v3=146", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3351, + "content": "v3=147", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3352, + "content": "v3=148", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3353, + "content": "v3=149", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3354, + "content": "v3=150", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3355, + "content": "v3=151", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3356, + "content": "v3=152", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3357, + "content": "v3=153", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3358, + "content": "v3=154", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3359, + "content": "v3=155", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3360, + "content": "v3=156", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3361, + "content": "v3=157", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3362, + "content": "v3=158", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3363, + "content": "v3=159", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3364, + "content": "v3=160", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3365, + "content": "v3=161", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3366, + "content": "v3=162", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3367, + "content": "v3=163", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3368, + "content": "v3=164", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3369, + "content": "v3=165", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3370, + "content": "v3=166", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3371, + "content": "v3=167", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3372, + "content": "v3=168", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3373, + "content": "v3=169", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3374, + "content": "v3=170", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3375, + "content": "v3=171", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3376, + "content": "v3=172", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3377, + "content": "v3=173", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3378, + "content": "v3=174", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3379, + "content": "v3=175", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3380, + "content": "v3=176", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3381, + "content": "v3=177", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3382, + "content": "v3=178", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3383, + "content": "v3=179", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3384, + "content": "v3=180", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3385, + "content": "v3=181", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3386, + "content": "v3=182", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3387, + "content": "v3=183", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3388, + "content": "v3=184", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3389, + "content": "v3=185", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3390, + "content": "v3=186", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3391, + "content": "v3=187", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3392, + "content": "v3=188", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3393, + "content": "v3=189", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3394, + "content": "v3=190", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3395, + "content": "v3=191", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3396, + "content": "v3=192", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3397, + "content": "v3=193", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3398, + "content": "v3=194", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3399, + "content": "v3=195", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3400, + "content": "v3=196", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3401, + "content": "v3=197", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3402, + "content": "v3=198", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3403, + "content": "v3=199", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3404, + "content": "v3=200", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3405, + "content": "v3=201", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3406, + "content": "v3=202", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3407, + "content": "v3=203", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3408, + "content": "v3=204", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3409, + "content": "v3=205", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3410, + "content": "v3=206", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3411, + "content": "v3=207", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3412, + "content": "v3=208", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3413, + "content": "v3=209", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3414, + "content": "v3=210", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3415, + "content": "v3=211", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3416, + "content": "v3=212", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3417, + "content": "v3=213", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3418, + "content": "v3=214", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3419, + "content": "v3=215", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3420, + "content": "v3=216", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3421, + "content": "v3=217", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3422, + "content": "v3=218", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3423, + "content": "v3=219", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3424, + "content": "v3=220", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3425, + "content": "v3=221", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3426, + "content": "v3=222", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3427, + "content": "v3=223", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3428, + "content": "v3=224", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3429, + "content": "v3=225", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3430, + "content": "v3=226", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3431, + "content": "v3=227", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3432, + "content": "v3=228", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3433, + "content": "v3=229", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3434, + "content": "v3=230", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3435, + "content": "v3=231", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3436, + "content": "v3=232", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3437, + "content": "v3=233", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3438, + "content": "v3=234", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3439, + "content": "v3=235", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3440, + "content": "v3=236", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3441, + "content": "v3=237", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3442, + "content": "v3=238", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3443, + "content": "v3=239", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3444, + "content": "v3=240", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3445, + "content": "v3=241", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3446, + "content": "v3=242", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3447, + "content": "v3=243", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3448, + "content": "v3=244", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3449, + "content": "v3=245", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3450, + "content": "v3=246", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3451, + "content": "v3=247", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3452, + "content": "v3=248", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3453, + "content": "v3=249", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3454, + "content": "v3=250", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3455, + "content": "v3=251", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3456, + "content": "v3=252", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3457, + "content": "v3=253", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3458, + "content": "v3=254", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3459, + "content": "v3=255", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3460, + "content": "v3=256", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3461, + "content": "v3=257", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3462, + "content": "v3=258", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3463, + "content": "v3=259", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3464, + "content": "v3=260", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3465, + "content": "v3=261", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3466, + "content": "v3=262", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3467, + "content": "v3=263", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3468, + "content": "v3=264", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3469, + "content": "v3=265", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3470, + "content": "v3=266", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3471, + "content": "v3=267", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3472, + "content": "v3=268", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3473, + "content": "v3=269", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3474, + "content": "v3=270", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3475, + "content": "v3=271", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3476, + "content": "v3=272", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3477, + "content": "v3=273", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3478, + "content": "v3=274", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3479, + "content": "v3=275", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3480, + "content": "v3=276", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3481, + "content": "v3=277", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3482, + "content": "v3=278", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3483, + "content": "v3=279", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3484, + "content": "v3=280", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3485, + "content": "v3=281", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3486, + "content": "v3=282", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3487, + "content": "v3=283", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3488, + "content": "v3=284", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3489, + "content": "v3=285", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3490, + "content": "v3=286", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3491, + "content": "v3=287", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3492, + "content": "v3=288", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3493, + "content": "v3=289", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3494, + "content": "v3=290", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3495, + "content": "v3=291", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3496, + "content": "v3=292", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3497, + "content": "v3=293", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3498, + "content": "v3=294", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3499, + "content": "v3=295", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3500, + "content": "v3=296", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3501, + "content": "v3=297", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3502, + "content": "v3=298", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3503, + "content": "v3=299", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3504, + "content": "v3=300", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3505, + "content": "v3=301", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3506, + "content": "v3=302", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3507, + "content": "v3=303", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3508, + "content": "v3=304", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3509, + "content": "v3=305", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3510, + "content": "v3=306", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3511, + "content": "v3=307", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3512, + "content": "v3=308", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3513, + "content": "v3=309", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3514, + "content": "v3=310", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3515, + "content": "v3=311", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3516, + "content": "v3=312", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3517, + "content": "v3=313", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3518, + "content": "v3=314", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3519, + "content": "v3=315", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3520, + "content": "v3=316", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3521, + "content": "v3=317", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3522, + "content": "v3=318", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3523, + "content": "v3=319", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3524, + "content": "v3=320", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3525, + "content": "v3=321", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3526, + "content": "v3=322", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3527, + "content": "v3=323", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3528, + "content": "v3=324", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3529, + "content": "v3=325", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3530, + "content": "v3=326", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3531, + "content": "v3=327", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3532, + "content": "v3=328", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3533, + "content": "v3=329", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3534, + "content": "v3=330", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3535, + "content": "v3=331", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3536, + "content": "v3=332", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3537, + "content": "v3=333", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3538, + "content": "v3=334", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3539, + "content": "v3=335", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3540, + "content": "v3=336", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3541, + "content": "v3=337", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3542, + "content": "v3=338", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3543, + "content": "v3=339", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3544, + "content": "v3=340", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3545, + "content": "v3=341", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3546, + "content": "v3=342", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3547, + "content": "v3=343", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3548, + "content": "v3=344", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3549, + "content": "v3=345", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3550, + "content": "v3=346", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3551, + "content": "v3=347", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3552, + "content": "v3=348", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3553, + "content": "v3=349", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3554, + "content": "v3=350", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3555, + "content": "v3=351", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3556, + "content": "v3=352", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3557, + "content": "v3=353", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3558, + "content": "v3=354", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3559, + "content": "v3=355", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3560, + "content": "v3=356", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3561, + "content": "v3=357", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3562, + "content": "v3=358", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3563, + "content": "v3=359", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3564, + "content": "v3=360", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3565, + "content": "v3=361", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3566, + "content": "v3=362", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3567, + "content": "v3=363", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3568, + "content": "v3=364", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3569, + "content": "v3=365", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3570, + "content": "v3=366", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3571, + "content": "v3=367", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3572, + "content": "v3=368", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3573, + "content": "v3=369", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3574, + "content": "v3=370", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3575, + "content": "v3=371", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3576, + "content": "v3=372", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3577, + "content": "v3=373", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3578, + "content": "v3=374", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3579, + "content": "v3=375", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3580, + "content": "v3=376", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3581, + "content": "v3=377", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3582, + "content": "v3=378", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3583, + "content": "v3=379", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3584, + "content": "v3=380", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3585, + "content": "v3=381", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3586, + "content": "v3=382", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3587, + "content": "v3=383", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3588, + "content": "v3=384", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3589, + "content": "v3=385", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3590, + "content": "v3=386", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3591, + "content": "v3=387", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3592, + "content": "v3=388", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3593, + "content": "v3=389", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3594, + "content": "v3=390", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3595, + "content": "v3=391", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3596, + "content": "v3=392", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3597, + "content": "v3=393", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3598, + "content": "v3=394", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3599, + "content": "v3=395", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3600, + "content": "v3=396", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3601, + "content": "v3=397", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3602, + "content": "v3=398", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3603, + "content": "v3=399", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3604, + "content": "v3=400", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3605, + "content": "v3=401", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3606, + "content": "v3=402", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3607, + "content": "v3=403", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3608, + "content": "v3=404", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3609, + "content": "v3=405", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3610, + "content": "v3=406", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3611, + "content": "v3=407", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3612, + "content": "v3=408", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3613, + "content": "v3=409", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3614, + "content": "v3=410", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3615, + "content": "v3=411", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3616, + "content": "v3=412", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3617, + "content": "v3=413", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3618, + "content": "v3=414", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3619, + "content": "v3=415", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3620, + "content": "v3=416", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3621, + "content": "v3=417", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3622, + "content": "v3=418", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3623, + "content": "v3=419", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3624, + "content": "v3=420", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3625, + "content": "v3=421", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3626, + "content": "v3=422", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3627, + "content": "v3=423", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3628, + "content": "v3=424", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3629, + "content": "v3=425", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3630, + "content": "v3=426", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3631, + "content": "v3=427", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3632, + "content": "v3=428", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3633, + "content": "v3=429", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3634, + "content": "v3=430", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3635, + "content": "v3=431", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3636, + "content": "v3=432", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3637, + "content": "v3=433", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3638, + "content": "v3=434", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3639, + "content": "v3=435", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3640, + "content": "v3=436", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3641, + "content": "v3=437", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3642, + "content": "v3=438", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3643, + "content": "v3=439", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3644, + "content": "v3=440", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3645, + "content": "v3=441", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3646, + "content": "v3=442", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3647, + "content": "v3=443", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3648, + "content": "v3=444", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3649, + "content": "v3=445", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3650, + "content": "v3=446", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3651, + "content": "v3=447", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3652, + "content": "v3=448", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3653, + "content": "v3=449", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3654, + "content": "v3=450", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3655, + "content": "v3=451", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3656, + "content": "v3=452", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3657, + "content": "v3=453", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3658, + "content": "v3=454", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3659, + "content": "v3=455", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3660, + "content": "v3=456", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3661, + "content": "v3=457", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3662, + "content": "v3=458", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3663, + "content": "v3=459", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3664, + "content": "v3=460", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3665, + "content": "v3=461", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3666, + "content": "v3=462", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3667, + "content": "v3=463", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3668, + "content": "v3=464", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3669, + "content": "v3=465", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3670, + "content": "v3=466", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3671, + "content": "v3=467", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3672, + "content": "v3=468", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3673, + "content": "v3=469", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3674, + "content": "v3=470", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3675, + "content": "v3=471", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3676, + "content": "v3=472", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3677, + "content": "v3=473", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3678, + "content": "v3=474", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3679, + "content": "v3=475", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3680, + "content": "v3=476", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3681, + "content": "v3=477", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3682, + "content": "v3=478", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3683, + "content": "v3=479", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3684, + "content": "v3=480", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3685, + "content": "v3=481", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3686, + "content": "v3=482", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3687, + "content": "v3=483", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3688, + "content": "v3=484", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3689, + "content": "v3=485", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3690, + "content": "v3=486", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3691, + "content": "v3=487", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3692, + "content": "v3=488", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3693, + "content": "v3=489", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3694, + "content": "v3=490", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3695, + "content": "v3=491", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3696, + "content": "v3=492", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3697, + "content": "v3=493", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3698, + "content": "v3=494", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3699, + "content": "v3=495", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3700, + "content": "v3=496", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3701, + "content": "v3=497", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3702, + "content": "v3=498", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3703, + "content": "v3=499", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3704, + "content": "v3=500", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3705, + "content": "v3=501", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3706, + "content": "v3=502", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3707, + "content": "v3=503", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3708, + "content": "v3=504", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3709, + "content": "v3=505", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3710, + "content": "v3=506", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3711, + "content": "v3=507", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3712, + "content": "v3=508", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3713, + "content": "v3=509", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3714, + "content": "v3=510", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3715, + "content": "v3=511", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3716, + "content": "v3=512", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3717, + "content": "v3=513", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3718, + "content": "v3=514", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3719, + "content": "v3=515", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3720, + "content": "v3=516", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3721, + "content": "v3=517", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3722, + "content": "v3=518", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3723, + "content": "v3=519", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3724, + "content": "v3=520", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3725, + "content": "v3=521", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3726, + "content": "v3=522", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3727, + "content": "v3=523", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3728, + "content": "v3=524", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3729, + "content": "v3=525", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3730, + "content": "v3=526", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3731, + "content": "v3=527", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3732, + "content": "v3=528", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3733, + "content": "v3=529", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3734, + "content": "v3=530", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3735, + "content": "v3=531", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3736, + "content": "v3=532", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3737, + "content": "v3=533", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3738, + "content": "v3=534", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3739, + "content": "v3=535", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3740, + "content": "v3=536", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3741, + "content": "v3=537", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3742, + "content": "v3=538", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3743, + "content": "v3=539", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3744, + "content": "v3=540", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3745, + "content": "v3=541", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3746, + "content": "v3=542", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3747, + "content": "v3=543", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3748, + "content": "v3=544", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3749, + "content": "v3=545", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3750, + "content": "v3=546", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3751, + "content": "v3=547", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3752, + "content": "v3=548", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3753, + "content": "v3=549", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3754, + "content": "v3=550", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3755, + "content": "v3=551", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3756, + "content": "v3=552", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3757, + "content": "v3=553", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3758, + "content": "v3=554", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3759, + "content": "v3=555", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3760, + "content": "v3=556", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3761, + "content": "v3=557", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3762, + "content": "v3=558", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3763, + "content": "v3=559", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3764, + "content": "v3=560", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3765, + "content": "v3=561", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3766, + "content": "v3=562", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3767, + "content": "v3=563", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3768, + "content": "v3=564", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3769, + "content": "v3=565", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3770, + "content": "v3=566", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3771, + "content": "v3=567", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3772, + "content": "v3=568", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3773, + "content": "v3=569", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3774, + "content": "v3=570", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3775, + "content": "v3=571", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3776, + "content": "v3=572", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3777, + "content": "v3=573", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3778, + "content": "v3=574", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3779, + "content": "v3=575", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3780, + "content": "v3=576", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3781, + "content": "v3=577", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3782, + "content": "v3=578", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3783, + "content": "v3=579", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3784, + "content": "v3=580", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3785, + "content": "v3=581", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3786, + "content": "v3=582", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3787, + "content": "v3=583", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3788, + "content": "v3=584", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3789, + "content": "v3=585", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3790, + "content": "v3=586", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3791, + "content": "v3=587", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3792, + "content": "v3=588", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3793, + "content": "v3=589", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3794, + "content": "v3=590", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3795, + "content": "v3=591", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3796, + "content": "v3=592", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3797, + "content": "v3=593", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3798, + "content": "v3=594", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3799, + "content": "v3=595", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3800, + "content": "v3=596", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3801, + "content": "v3=597", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3802, + "content": "v3=598", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3803, + "content": "v3=599", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3804, + "content": "v3=600", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3805, + "content": "v3=601", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3806, + "content": "v3=602", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3807, + "content": "v3=603", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3808, + "content": "v3=604", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3809, + "content": "v3=605", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3810, + "content": "v3=606", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3811, + "content": "v3=607", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3812, + "content": "v3=608", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3813, + "content": "v3=609", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3814, + "content": "v3=610", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3815, + "content": "v3=611", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3816, + "content": "v3=612", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3817, + "content": "v3=613", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3818, + "content": "v3=614", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3819, + "content": "v3=615", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3820, + "content": "v3=616", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3821, + "content": "v3=617", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3822, + "content": "v3=618", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3823, + "content": "v3=619", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3824, + "content": "v3=620", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3825, + "content": "v3=621", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3826, + "content": "v3=622", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3827, + "content": "v3=623", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3828, + "content": "v3=624", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3829, + "content": "v3=625", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3830, + "content": "v3=626", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3831, + "content": "v3=627", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3832, + "content": "v3=628", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3833, + "content": "v3=629", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3834, + "content": "v3=630", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3835, + "content": "v3=631", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3836, + "content": "v3=632", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3837, + "content": "v3=633", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3838, + "content": "v3=634", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3839, + "content": "v3=635", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3840, + "content": "v3=636", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3841, + "content": "v3=637", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3842, + "content": "v3=638", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3843, + "content": "v3=639", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3844, + "content": "v3=640", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3845, + "content": "v3=641", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3846, + "content": "v3=642", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3847, + "content": "v3=643", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3848, + "content": "v3=644", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3849, + "content": "v3=645", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3850, + "content": "v3=646", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3851, + "content": "v3=647", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3852, + "content": "v3=648", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3853, + "content": "v3=649", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3854, + "content": "v3=650", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3855, + "content": "v3=651", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3856, + "content": "v3=652", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3857, + "content": "v3=653", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3858, + "content": "v3=654", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3859, + "content": "v3=655", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3860, + "content": "v3=656", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3861, + "content": "v3=657", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3862, + "content": "v3=658", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3863, + "content": "v3=659", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3864, + "content": "v3=660", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3865, + "content": "v3=661", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3866, + "content": "v3=662", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3867, + "content": "v3=663", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3868, + "content": "v3=664", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3869, + "content": "v3=665", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3870, + "content": "v3=666", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3871, + "content": "v3=667", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3872, + "content": "v3=668", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3873, + "content": "v3=669", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3874, + "content": "v3=670", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3875, + "content": "v3=671", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3876, + "content": "v3=672", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3877, + "content": "v3=673", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3878, + "content": "v3=674", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3879, + "content": "v3=675", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3880, + "content": "v3=676", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3881, + "content": "v3=677", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3882, + "content": "v3=678", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3883, + "content": "v3=679", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3884, + "content": "v3=680", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3885, + "content": "v3=681", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3886, + "content": "v3=682", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3887, + "content": "v3=683", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3888, + "content": "v3=684", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3889, + "content": "v3=685", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3890, + "content": "v3=686", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3891, + "content": "v3=687", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3892, + "content": "v3=688", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3893, + "content": "v3=689", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3894, + "content": "v3=690", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3895, + "content": "v3=691", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3896, + "content": "v3=692", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3897, + "content": "v3=693", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3898, + "content": "v3=694", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3899, + "content": "v3=695", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3900, + "content": "v3=696", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3901, + "content": "v3=697", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3902, + "content": "v3=698", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3903, + "content": "v3=699", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3904, + "content": "v3=700", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3905, + "content": "v3=701", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3906, + "content": "v3=702", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3907, + "content": "v3=703", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3908, + "content": "v3=704", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3909, + "content": "v3=705", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3910, + "content": "v3=706", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3911, + "content": "v3=707", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3912, + "content": "v3=708", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3913, + "content": "v3=709", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3914, + "content": "v3=710", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3915, + "content": "v3=711", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3916, + "content": "v3=712", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3917, + "content": "v3=713", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3918, + "content": "v3=714", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3919, + "content": "v3=715", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3920, + "content": "v3=716", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3921, + "content": "v3=717", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3922, + "content": "v3=718", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3923, + "content": "v3=719", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3924, + "content": "v3=720", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3925, + "content": "v3=721", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3926, + "content": "v3=722", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3927, + "content": "v3=723", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3928, + "content": "v3=724", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3929, + "content": "v3=725", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3930, + "content": "v3=726", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3931, + "content": "v3=727", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3932, + "content": "v3=728", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3933, + "content": "v3=729", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3934, + "content": "v3=730", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3935, + "content": "v3=731", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3936, + "content": "v3=732", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3937, + "content": "v3=733", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3938, + "content": "v3=734", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3939, + "content": "v3=735", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3940, + "content": "v3=736", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3941, + "content": "v3=737", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3942, + "content": "v3=738", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3943, + "content": "v3=739", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3944, + "content": "v3=740", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3945, + "content": "v3=741", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3946, + "content": "v3=742", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3947, + "content": "v3=743", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3948, + "content": "v3=744", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3949, + "content": "v3=745", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3950, + "content": "v3=746", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3951, + "content": "v3=747", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3952, + "content": "v3=748", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3953, + "content": "v3=749", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3954, + "content": "v3=750", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3955, + "content": "v3=751", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3956, + "content": "v3=752", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3957, + "content": "v3=753", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3958, + "content": "v3=754", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3959, + "content": "v3=755", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3960, + "content": "v3=756", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3961, + "content": "v3=757", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3962, + "content": "v3=758", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3963, + "content": "v3=759", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3964, + "content": "v3=760", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3965, + "content": "v3=761", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3966, + "content": "v3=762", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3967, + "content": "v3=763", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3968, + "content": "v3=764", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3969, + "content": "v3=765", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3970, + "content": "v3=766", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3971, + "content": "v3=767", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3972, + "content": "v3=768", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3973, + "content": "v3=769", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3974, + "content": "v3=770", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3975, + "content": "v3=771", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3976, + "content": "v3=772", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3977, + "content": "v3=773", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3978, + "content": "v3=774", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3979, + "content": "v3=775", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3980, + "content": "v3=776", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3981, + "content": "v3=777", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3982, + "content": "v3=778", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3983, + "content": "v3=779", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3984, + "content": "v3=780", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3985, + "content": "v3=781", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3986, + "content": "v3=782", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3987, + "content": "v3=783", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3988, + "content": "v3=784", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3989, + "content": "v3=785", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3990, + "content": "v3=786", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3991, + "content": "v3=787", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3992, + "content": "v3=788", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3993, + "content": "v3=789", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3994, + "content": "v3=790", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3995, + "content": "v3=791", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3996, + "content": "v3=792", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3997, + "content": "v3=793", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3998, + "content": "v3=794", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3999, + "content": "v3=795", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4000, + "content": "v3=796", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4001, + "content": "v3=797", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4002, + "content": "v3=798", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4003, + "content": "v3=799", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4004, + "content": "v3=800", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4005, + "content": "v3=801", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4006, + "content": "v3=802", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4007, + "content": "v3=803", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4008, + "content": "v3=804", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4009, + "content": "v3=805", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4010, + "content": "v3=806", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4011, + "content": "v3=807", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4012, + "content": "v3=808", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4013, + "content": "v3=809", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4014, + "content": "v3=810", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4015, + "content": "v3=811", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4016, + "content": "v3=812", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4017, + "content": "v3=813", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4018, + "content": "v3=814", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4019, + "content": "v3=815", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4020, + "content": "v3=816", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4021, + "content": "v3=817", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4022, + "content": "v3=818", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4023, + "content": "v3=819", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4024, + "content": "v3=820", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4025, + "content": "v3=821", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4026, + "content": "v3=822", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4027, + "content": "v3=823", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4028, + "content": "v3=824", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4029, + "content": "v3=825", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4030, + "content": "v3=826", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4031, + "content": "v3=827", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4032, + "content": "v3=828", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4033, + "content": "v3=829", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4034, + "content": "v3=830", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4035, + "content": "v3=831", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4036, + "content": "v3=832", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4037, + "content": "v3=833", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4038, + "content": "v3=834", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4039, + "content": "v3=835", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4040, + "content": "v3=836", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4041, + "content": "v3=837", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4042, + "content": "v3=838", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4043, + "content": "v3=839", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4044, + "content": "v3=840", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4045, + "content": "v3=841", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4046, + "content": "v3=842", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4047, + "content": "v3=843", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4048, + "content": "v3=844", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4049, + "content": "v3=845", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4050, + "content": "v3=846", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4051, + "content": "v3=847", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4052, + "content": "v3=848", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4053, + "content": "v3=849", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4054, + "content": "v3=850", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4055, + "content": "v3=851", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4056, + "content": "v3=852", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4057, + "content": "v3=853", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4058, + "content": "v3=854", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4059, + "content": "v3=855", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4060, + "content": "v3=856", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4061, + "content": "v3=857", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4062, + "content": "v3=858", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4063, + "content": "v3=859", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4064, + "content": "v3=860", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4065, + "content": "v3=861", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4066, + "content": "v3=862", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4067, + "content": "v3=863", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4068, + "content": "v3=864", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4069, + "content": "v3=865", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4070, + "content": "v3=866", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4071, + "content": "v3=867", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4072, + "content": "v3=868", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4073, + "content": "v3=869", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4074, + "content": "v3=870", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4075, + "content": "v3=871", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4076, + "content": "v3=872", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4077, + "content": "v3=873", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4078, + "content": "v3=874", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4079, + "content": "v3=875", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4080, + "content": "v3=876", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4081, + "content": "v3=877", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4082, + "content": "v3=878", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4083, + "content": "v3=879", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4084, + "content": "v3=880", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4085, + "content": "v3=881", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4086, + "content": "v3=882", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4087, + "content": "v3=883", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4088, + "content": "v3=884", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4089, + "content": "v3=885", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4090, + "content": "v3=886", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4091, + "content": "v3=887", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4092, + "content": "v3=888", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4093, + "content": "v3=889", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4094, + "content": "v3=890", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4095, + "content": "v3=891", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4096, + "content": "v3=892", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4097, + "content": "v3=893", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4098, + "content": "v3=894", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4099, + "content": "v3=895", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4100, + "content": "v3=896", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4101, + "content": "v3=897", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4102, + "content": "v3=898", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4103, + "content": "v3=899", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4104, + "content": "v3=900", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4105, + "content": "v3=901", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4106, + "content": "v3=902", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4107, + "content": "v3=903", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4108, + "content": "v3=904", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4109, + "content": "v3=905", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4110, + "content": "v3=906", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4111, + "content": "v3=907", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4112, + "content": "v3=908", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4113, + "content": "v3=909", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4114, + "content": "v3=910", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4115, + "content": "v3=911", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4116, + "content": "v3=912", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4117, + "content": "v3=913", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4118, + "content": "v3=914", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4119, + "content": "v3=915", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4120, + "content": "v3=916", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4121, + "content": "v3=917", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4122, + "content": "v3=918", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4123, + "content": "v3=919", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4124, + "content": "v3=920", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4125, + "content": "v3=921", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4126, + "content": "v3=922", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4127, + "content": "v3=923", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4128, + "content": "v3=924", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4129, + "content": "v3=925", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4130, + "content": "v3=926", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4131, + "content": "v3=927", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4132, + "content": "v3=928", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4133, + "content": "v3=929", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4134, + "content": "v3=930", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4135, + "content": "v3=931", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4136, + "content": "v3=932", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4137, + "content": "v3=933", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4138, + "content": "v3=934", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4139, + "content": "v3=935", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4140, + "content": "v3=936", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4141, + "content": "v3=937", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4142, + "content": "v3=938", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4143, + "content": "v3=939", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4144, + "content": "v3=940", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4145, + "content": "v3=941", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4146, + "content": "v3=942", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4147, + "content": "v3=943", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4148, + "content": "v3=944", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4149, + "content": "v3=945", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4150, + "content": "v3=946", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4151, + "content": "v3=947", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4152, + "content": "v3=948", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4153, + "content": "v3=949", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4154, + "content": "v3=950", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4155, + "content": "v3=951", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4156, + "content": "v3=952", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4157, + "content": "v3=953", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4158, + "content": "v3=954", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4159, + "content": "v3=955", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4160, + "content": "v3=956", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4161, + "content": "v3=957", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4162, + "content": "v3=958", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4163, + "content": "v3=959", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4164, + "content": "v3=960", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4165, + "content": "v3=961", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4166, + "content": "v3=962", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4167, + "content": "v3=963", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4168, + "content": "v3=964", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4169, + "content": "v3=965", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4170, + "content": "v3=966", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4171, + "content": "v3=967", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4172, + "content": "v3=968", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4173, + "content": "v3=969", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4174, + "content": "v3=970", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4175, + "content": "v3=971", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4176, + "content": "v3=972", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4177, + "content": "v3=973", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4178, + "content": "v3=974", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4179, + "content": "v3=975", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4180, + "content": "v3=976", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4181, + "content": "v3=977", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4182, + "content": "v3=978", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4183, + "content": "v3=979", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4184, + "content": "v3=980", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4185, + "content": "v3=981", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4186, + "content": "v3=982", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4187, + "content": "v3=983", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4188, + "content": "v3=984", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4189, + "content": "v3=985", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4190, + "content": "v3=986", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4191, + "content": "v3=987", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4192, + "content": "v3=988", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4193, + "content": "v3=989", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4194, + "content": "v3=990", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4195, + "content": "v3=991", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4196, + "content": "v3=992", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4197, + "content": "v3=993", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4198, + "content": "v3=994", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4199, + "content": "v3=995", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4200, + "content": "v3=996", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4201, + "content": "v3=997", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4202, + "content": "v3=998", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4203, + "content": "v3=999", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4204, + "content": "airplane", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4205, + "content": "apple", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4206, + "content": "backpack", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4207, + "content": "banana", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4208, + "content": "baseball bat", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4209, + "content": "baseball glove", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4210, + "content": "bear", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4211, + "content": "bed", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4212, + "content": "bench", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4213, + "content": "bicycle", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4214, + "content": "bird", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4215, + "content": "boat", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4216, + "content": "book", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4217, + "content": "bottle", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4218, + "content": "bowl", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4219, + "content": "broccoli", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4220, + "content": "bus", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4221, + "content": "cake", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4222, + "content": "car", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4223, + "content": "carrot", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4224, + "content": "cat", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4225, + "content": "cell phone", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4226, + "content": "chair", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4227, + "content": "clock", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4228, + "content": "couch", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4229, + "content": "cow", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4230, + "content": "cup", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4231, + "content": "dining table", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4232, + "content": "dog", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4233, + "content": "donut", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4234, + "content": "elephant", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4235, + "content": "fire hydrant", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4236, + "content": "fork", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4237, + "content": "frisbee", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4238, + "content": "giraffe", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4239, + "content": "hair drier", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4240, + "content": "handbag", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4241, + "content": "horse", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4242, + "content": "hot dog", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4243, + "content": "keyboard", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4244, + "content": "kite", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4245, + "content": "knife", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4246, + "content": "laptop", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4247, + "content": "microwave", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4248, + "content": "motorcycle", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4249, + "content": "mouse", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4250, + "content": "orange", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4251, + "content": "oven", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4252, + "content": "parking meter", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4253, + "content": "person", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4254, + "content": "pizza", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4255, + "content": "potted plant", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4256, + "content": "refrigerator", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4257, + "content": "remote", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4258, + "content": "sandwich", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4259, + "content": "scissors", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4260, + "content": "sheep", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4261, + "content": "sink", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4262, + "content": "skateboard", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4263, + "content": "skis", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4264, + "content": "snowboard", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4265, + "content": "spoon", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4266, + "content": "sports ball", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4267, + "content": "stop sign", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4268, + "content": "suitcase", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4269, + "content": "surfboard", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4270, + "content": "teddy bear", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4271, + "content": "tennis racket", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4272, + "content": "tie", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4273, + "content": "toaster", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4274, + "content": "toilet", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4275, + "content": "toothbrush", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4276, + "content": "traffic light", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4277, + "content": "train", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4278, + "content": "truck", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4279, + "content": "tv", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4280, + "content": "umbrella", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4281, + "content": "vase", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4282, + "content": "wine glass", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 4283, + "content": "zebra", + "single_word": true, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + } + ], + "normalizer": { + "type": "BertNormalizer", + "clean_text": true, + "handle_chinese_chars": true, + "strip_accents": null, + "lowercase": true + }, + "pre_tokenizer": { + "type": "BertPreTokenizer" + }, + "post_processor": null, + "decoder": { + "type": "WordPiece", + "prefix": "##", + "cleanup": true + }, + "model": { + "type": "WordPiece", + "unk_token": "[UNK]", + "continuing_subword_prefix": "##", + "max_input_chars_per_word": 100, + "vocab": { + "[PAD]": 0, + "[UNK]": 1, + "[SOS]": 2, + "[EOS]": 3, + "[S_0]": 4, + "[S_1]": 5, + "[S_2]": 6, + "[S_3]": 7, + "[S_4]": 8, + "[S_5]": 9, + "[S_6]": 10, + "[S_7]": 11, + "[S_8]": 12, + "[S_9]": 13, + "[S_10]": 14, + "[S_11]": 15, + "[S_12]": 16, + "[S_13]": 17, + "[S_14]": 18, + "[S_15]": 19, + "[S_16]": 20, + "[S_17]": 21, + "[S_18]": 22, + "[S_19]": 23, + "[S_20]": 24, + "[S_21]": 25, + "[S_22]": 26, + "[S_23]": 27, + "[S_24]": 28, + "[S_25]": 29, + "[S_26]": 30, + "[S_27]": 31, + "[S_28]": 32, + "[S_29]": 33, + "[S_30]": 34, + "[S_31]": 35, + "[S_32]": 36, + "[S_33]": 37, + "[S_34]": 38, + "[S_35]": 39, + "[S_36]": 40, + "[S_37]": 41, + "[S_38]": 42, + "[S_39]": 43, + "[S_40]": 44, + "[S_41]": 45, + "[S_42]": 46, + "[S_43]": 47, + "[S_44]": 48, + "[S_45]": 49, + "[S_46]": 50, + "[S_47]": 51, + "[S_48]": 52, + "[S_49]": 53, + "[S_50]": 54, + "[S_51]": 55, + "[S_52]": 56, + "[S_53]": 57, + "[S_54]": 58, + "[S_55]": 59, + "[S_56]": 60, + "[S_57]": 61, + "[S_58]": 62, + "[S_59]": 63, + "[S_60]": 64, + "[S_61]": 65, + "[S_62]": 66, + "[S_63]": 67, + "[S_64]": 68, + "[S_65]": 69, + "[S_66]": 70, + "[S_67]": 71, + "[S_68]": 72, + "[S_69]": 73, + "[S_70]": 74, + "[S_71]": 75, + "[S_72]": 76, + "[S_73]": 77, + "[S_74]": 78, + "[S_75]": 79, + "[S_76]": 80, + "[S_77]": 81, + "[S_78]": 82, + "[S_79]": 83, + "[S_80]": 84, + "[S_81]": 85, + "[S_82]": 86, + "[S_83]": 87, + "[S_84]": 88, + "[S_85]": 89, + "[S_86]": 90, + "[S_87]": 91, + "[S_88]": 92, + "[S_89]": 93, + "[S_90]": 94, + "[S_91]": 95, + "[S_92]": 96, + "[S_93]": 97, + "[S_94]": 98, + "[S_95]": 99, + "[S_96]": 100, + "[S_97]": 101, + "[S_98]": 102, + "[S_99]": 103, + "[S_100]": 104, + "[S_101]": 105, + "[S_102]": 106, + "[S_103]": 107, + "[S_104]": 108, + "[S_105]": 109, + "[S_106]": 110, + "[S_107]": 111, + "[S_108]": 112, + "[S_109]": 113, + "[S_110]": 114, + "[S_111]": 115, + "[S_112]": 116, + "[S_113]": 117, + "[S_114]": 118, + "[S_115]": 119, + "[S_116]": 120, + "[S_117]": 121, + "[S_118]": 122, + "[S_119]": 123, + "[S_120]": 124, + "[S_121]": 125, + "[S_122]": 126, + "[S_123]": 127, + "[S_124]": 128, + "[S_125]": 129, + "[S_126]": 130, + "[S_127]": 131, + "[S_128]": 132, + "[S_129]": 133, + "[S_130]": 134, + "[S_131]": 135, + "[S_132]": 136, + "[S_133]": 137, + "[S_134]": 138, + "[S_135]": 139, + "[S_136]": 140, + "[S_137]": 141, + "[S_138]": 142, + "[S_139]": 143, + "[S_140]": 144, + "[S_141]": 145, + "[S_142]": 146, + "[S_143]": 147, + "[S_144]": 148, + "[S_145]": 149, + "[S_146]": 150, + "[S_147]": 151, + "[S_148]": 152, + "[S_149]": 153, + "[S_150]": 154, + "[S_151]": 155, + "[S_152]": 156, + "[S_153]": 157, + "[S_154]": 158, + "[S_155]": 159, + "[S_156]": 160, + "[S_157]": 161, + "[S_158]": 162, + "[S_159]": 163, + "[S_160]": 164, + "[S_161]": 165, + "[S_162]": 166, + "[S_163]": 167, + "[S_164]": 168, + "[S_165]": 169, + "[S_166]": 170, + "[S_167]": 171, + "[S_168]": 172, + "[S_169]": 173, + "[S_170]": 174, + "[S_171]": 175, + "[S_172]": 176, + "[S_173]": 177, + "[S_174]": 178, + "[S_175]": 179, + "[S_176]": 180, + "[S_177]": 181, + "[S_178]": 182, + "[S_179]": 183, + "[S_180]": 184, + "[S_181]": 185, + "[S_182]": 186, + "[S_183]": 187, + "[S_184]": 188, + "[S_185]": 189, + "[S_186]": 190, + "[S_187]": 191, + "[S_188]": 192, + "[S_189]": 193, + "[S_190]": 194, + "[S_191]": 195, + "[S_192]": 196, + "[S_193]": 197, + "[S_194]": 198, + "[S_195]": 199, + "[S_196]": 200, + "[S_197]": 201, + "[S_198]": 202, + "[S_199]": 203, + "v0=0": 204, + "v0=1": 205, + "v0=2": 206, + "v0=3": 207, + "v0=4": 208, + "v0=5": 209, + "v0=6": 210, + "v0=7": 211, + "v0=8": 212, + "v0=9": 213, + "v0=10": 214, + "v0=11": 215, + "v0=12": 216, + "v0=13": 217, + "v0=14": 218, + "v0=15": 219, + "v0=16": 220, + "v0=17": 221, + "v0=18": 222, + "v0=19": 223, + "v0=20": 224, + "v0=21": 225, + "v0=22": 226, + "v0=23": 227, + "v0=24": 228, + "v0=25": 229, + "v0=26": 230, + "v0=27": 231, + "v0=28": 232, + "v0=29": 233, + "v0=30": 234, + "v0=31": 235, + "v0=32": 236, + "v0=33": 237, + "v0=34": 238, + "v0=35": 239, + "v0=36": 240, + "v0=37": 241, + "v0=38": 242, + "v0=39": 243, + "v0=40": 244, + "v0=41": 245, + "v0=42": 246, + "v0=43": 247, + "v0=44": 248, + "v0=45": 249, + "v0=46": 250, + "v0=47": 251, + "v0=48": 252, + "v0=49": 253, + "v0=50": 254, + "v0=51": 255, + "v0=52": 256, + "v0=53": 257, + "v0=54": 258, + "v0=55": 259, + "v0=56": 260, + "v0=57": 261, + "v0=58": 262, + "v0=59": 263, + "v0=60": 264, + "v0=61": 265, + "v0=62": 266, + "v0=63": 267, + "v0=64": 268, + "v0=65": 269, + "v0=66": 270, + "v0=67": 271, + "v0=68": 272, + "v0=69": 273, + "v0=70": 274, + "v0=71": 275, + "v0=72": 276, + "v0=73": 277, + "v0=74": 278, + "v0=75": 279, + "v0=76": 280, + "v0=77": 281, + "v0=78": 282, + "v0=79": 283, + "v0=80": 284, + "v0=81": 285, + "v0=82": 286, + "v0=83": 287, + "v0=84": 288, + "v0=85": 289, + "v0=86": 290, + "v0=87": 291, + "v0=88": 292, + "v0=89": 293, + "v0=90": 294, + "v0=91": 295, + "v0=92": 296, + "v0=93": 297, + "v0=94": 298, + "v0=95": 299, + "v0=96": 300, + "v0=97": 301, + "v0=98": 302, + "v0=99": 303, + "v0=100": 304, + "v0=101": 305, + "v0=102": 306, + "v0=103": 307, + "v0=104": 308, + "v0=105": 309, + "v0=106": 310, + "v0=107": 311, + "v0=108": 312, + "v0=109": 313, + "v0=110": 314, + "v0=111": 315, + "v0=112": 316, + "v0=113": 317, + "v0=114": 318, + "v0=115": 319, + "v0=116": 320, + "v0=117": 321, + "v0=118": 322, + "v0=119": 323, + "v0=120": 324, + "v0=121": 325, + "v0=122": 326, + "v0=123": 327, + "v0=124": 328, + "v0=125": 329, + "v0=126": 330, + "v0=127": 331, + "v0=128": 332, + "v0=129": 333, + "v0=130": 334, + "v0=131": 335, + "v0=132": 336, + "v0=133": 337, + "v0=134": 338, + "v0=135": 339, + "v0=136": 340, + "v0=137": 341, + "v0=138": 342, + "v0=139": 343, + "v0=140": 344, + "v0=141": 345, + "v0=142": 346, + "v0=143": 347, + "v0=144": 348, + "v0=145": 349, + "v0=146": 350, + "v0=147": 351, + "v0=148": 352, + "v0=149": 353, + "v0=150": 354, + "v0=151": 355, + "v0=152": 356, + "v0=153": 357, + "v0=154": 358, + "v0=155": 359, + "v0=156": 360, + "v0=157": 361, + "v0=158": 362, + "v0=159": 363, + "v0=160": 364, + "v0=161": 365, + "v0=162": 366, + "v0=163": 367, + "v0=164": 368, + "v0=165": 369, + "v0=166": 370, + "v0=167": 371, + "v0=168": 372, + "v0=169": 373, + "v0=170": 374, + "v0=171": 375, + "v0=172": 376, + "v0=173": 377, + "v0=174": 378, + "v0=175": 379, + "v0=176": 380, + "v0=177": 381, + "v0=178": 382, + "v0=179": 383, + "v0=180": 384, + "v0=181": 385, + "v0=182": 386, + "v0=183": 387, + "v0=184": 388, + "v0=185": 389, + "v0=186": 390, + "v0=187": 391, + "v0=188": 392, + "v0=189": 393, + "v0=190": 394, + "v0=191": 395, + "v0=192": 396, + "v0=193": 397, + "v0=194": 398, + "v0=195": 399, + "v0=196": 400, + "v0=197": 401, + "v0=198": 402, + "v0=199": 403, + "v0=200": 404, + "v0=201": 405, + "v0=202": 406, + "v0=203": 407, + "v0=204": 408, + "v0=205": 409, + "v0=206": 410, + "v0=207": 411, + "v0=208": 412, + "v0=209": 413, + "v0=210": 414, + "v0=211": 415, + "v0=212": 416, + "v0=213": 417, + "v0=214": 418, + "v0=215": 419, + "v0=216": 420, + "v0=217": 421, + "v0=218": 422, + "v0=219": 423, + "v0=220": 424, + "v0=221": 425, + "v0=222": 426, + "v0=223": 427, + "v0=224": 428, + "v0=225": 429, + "v0=226": 430, + "v0=227": 431, + "v0=228": 432, + "v0=229": 433, + "v0=230": 434, + "v0=231": 435, + "v0=232": 436, + "v0=233": 437, + "v0=234": 438, + "v0=235": 439, + "v0=236": 440, + "v0=237": 441, + "v0=238": 442, + "v0=239": 443, + "v0=240": 444, + "v0=241": 445, + "v0=242": 446, + "v0=243": 447, + "v0=244": 448, + "v0=245": 449, + "v0=246": 450, + "v0=247": 451, + "v0=248": 452, + "v0=249": 453, + "v0=250": 454, + "v0=251": 455, + "v0=252": 456, + "v0=253": 457, + "v0=254": 458, + "v0=255": 459, + "v0=256": 460, + "v0=257": 461, + "v0=258": 462, + "v0=259": 463, + "v0=260": 464, + "v0=261": 465, + "v0=262": 466, + "v0=263": 467, + "v0=264": 468, + "v0=265": 469, + "v0=266": 470, + "v0=267": 471, + "v0=268": 472, + "v0=269": 473, + "v0=270": 474, + "v0=271": 475, + "v0=272": 476, + "v0=273": 477, + "v0=274": 478, + "v0=275": 479, + "v0=276": 480, + "v0=277": 481, + "v0=278": 482, + "v0=279": 483, + "v0=280": 484, + "v0=281": 485, + "v0=282": 486, + "v0=283": 487, + "v0=284": 488, + "v0=285": 489, + "v0=286": 490, + "v0=287": 491, + "v0=288": 492, + "v0=289": 493, + "v0=290": 494, + "v0=291": 495, + "v0=292": 496, + "v0=293": 497, + "v0=294": 498, + "v0=295": 499, + "v0=296": 500, + "v0=297": 501, + "v0=298": 502, + "v0=299": 503, + "v0=300": 504, + "v0=301": 505, + "v0=302": 506, + "v0=303": 507, + "v0=304": 508, + "v0=305": 509, + "v0=306": 510, + "v0=307": 511, + "v0=308": 512, + "v0=309": 513, + "v0=310": 514, + "v0=311": 515, + "v0=312": 516, + "v0=313": 517, + "v0=314": 518, + "v0=315": 519, + "v0=316": 520, + "v0=317": 521, + "v0=318": 522, + "v0=319": 523, + "v0=320": 524, + "v0=321": 525, + "v0=322": 526, + "v0=323": 527, + "v0=324": 528, + "v0=325": 529, + "v0=326": 530, + "v0=327": 531, + "v0=328": 532, + "v0=329": 533, + "v0=330": 534, + "v0=331": 535, + "v0=332": 536, + "v0=333": 537, + "v0=334": 538, + "v0=335": 539, + "v0=336": 540, + "v0=337": 541, + "v0=338": 542, + "v0=339": 543, + "v0=340": 544, + "v0=341": 545, + "v0=342": 546, + "v0=343": 547, + "v0=344": 548, + "v0=345": 549, + "v0=346": 550, + "v0=347": 551, + "v0=348": 552, + "v0=349": 553, + "v0=350": 554, + "v0=351": 555, + "v0=352": 556, + "v0=353": 557, + "v0=354": 558, + "v0=355": 559, + "v0=356": 560, + "v0=357": 561, + "v0=358": 562, + "v0=359": 563, + "v0=360": 564, + "v0=361": 565, + "v0=362": 566, + "v0=363": 567, + "v0=364": 568, + "v0=365": 569, + "v0=366": 570, + "v0=367": 571, + "v0=368": 572, + "v0=369": 573, + "v0=370": 574, + "v0=371": 575, + "v0=372": 576, + "v0=373": 577, + "v0=374": 578, + "v0=375": 579, + "v0=376": 580, + "v0=377": 581, + "v0=378": 582, + "v0=379": 583, + "v0=380": 584, + "v0=381": 585, + "v0=382": 586, + "v0=383": 587, + "v0=384": 588, + "v0=385": 589, + "v0=386": 590, + "v0=387": 591, + "v0=388": 592, + "v0=389": 593, + "v0=390": 594, + "v0=391": 595, + "v0=392": 596, + "v0=393": 597, + "v0=394": 598, + "v0=395": 599, + "v0=396": 600, + "v0=397": 601, + "v0=398": 602, + "v0=399": 603, + "v0=400": 604, + "v0=401": 605, + "v0=402": 606, + "v0=403": 607, + "v0=404": 608, + "v0=405": 609, + "v0=406": 610, + "v0=407": 611, + "v0=408": 612, + "v0=409": 613, + "v0=410": 614, + "v0=411": 615, + "v0=412": 616, + "v0=413": 617, + "v0=414": 618, + "v0=415": 619, + "v0=416": 620, + "v0=417": 621, + "v0=418": 622, + "v0=419": 623, + "v0=420": 624, + "v0=421": 625, + "v0=422": 626, + "v0=423": 627, + "v0=424": 628, + "v0=425": 629, + "v0=426": 630, + "v0=427": 631, + "v0=428": 632, + "v0=429": 633, + "v0=430": 634, + "v0=431": 635, + "v0=432": 636, + "v0=433": 637, + "v0=434": 638, + "v0=435": 639, + "v0=436": 640, + "v0=437": 641, + "v0=438": 642, + "v0=439": 643, + "v0=440": 644, + "v0=441": 645, + "v0=442": 646, + "v0=443": 647, + "v0=444": 648, + "v0=445": 649, + "v0=446": 650, + "v0=447": 651, + "v0=448": 652, + "v0=449": 653, + "v0=450": 654, + "v0=451": 655, + "v0=452": 656, + "v0=453": 657, + "v0=454": 658, + "v0=455": 659, + "v0=456": 660, + "v0=457": 661, + "v0=458": 662, + "v0=459": 663, + "v0=460": 664, + "v0=461": 665, + "v0=462": 666, + "v0=463": 667, + "v0=464": 668, + "v0=465": 669, + "v0=466": 670, + "v0=467": 671, + "v0=468": 672, + "v0=469": 673, + "v0=470": 674, + "v0=471": 675, + "v0=472": 676, + "v0=473": 677, + "v0=474": 678, + "v0=475": 679, + "v0=476": 680, + "v0=477": 681, + "v0=478": 682, + "v0=479": 683, + "v0=480": 684, + "v0=481": 685, + "v0=482": 686, + "v0=483": 687, + "v0=484": 688, + "v0=485": 689, + "v0=486": 690, + "v0=487": 691, + "v0=488": 692, + "v0=489": 693, + "v0=490": 694, + "v0=491": 695, + "v0=492": 696, + "v0=493": 697, + "v0=494": 698, + "v0=495": 699, + "v0=496": 700, + "v0=497": 701, + "v0=498": 702, + "v0=499": 703, + "v0=500": 704, + "v0=501": 705, + "v0=502": 706, + "v0=503": 707, + "v0=504": 708, + "v0=505": 709, + "v0=506": 710, + "v0=507": 711, + "v0=508": 712, + "v0=509": 713, + "v0=510": 714, + "v0=511": 715, + "v0=512": 716, + "v0=513": 717, + "v0=514": 718, + "v0=515": 719, + "v0=516": 720, + "v0=517": 721, + "v0=518": 722, + "v0=519": 723, + "v0=520": 724, + "v0=521": 725, + "v0=522": 726, + "v0=523": 727, + "v0=524": 728, + "v0=525": 729, + "v0=526": 730, + "v0=527": 731, + "v0=528": 732, + "v0=529": 733, + "v0=530": 734, + "v0=531": 735, + "v0=532": 736, + "v0=533": 737, + "v0=534": 738, + "v0=535": 739, + "v0=536": 740, + "v0=537": 741, + "v0=538": 742, + "v0=539": 743, + "v0=540": 744, + "v0=541": 745, + "v0=542": 746, + "v0=543": 747, + "v0=544": 748, + "v0=545": 749, + "v0=546": 750, + "v0=547": 751, + "v0=548": 752, + "v0=549": 753, + "v0=550": 754, + "v0=551": 755, + "v0=552": 756, + "v0=553": 757, + "v0=554": 758, + "v0=555": 759, + "v0=556": 760, + "v0=557": 761, + "v0=558": 762, + "v0=559": 763, + "v0=560": 764, + "v0=561": 765, + "v0=562": 766, + "v0=563": 767, + "v0=564": 768, + "v0=565": 769, + "v0=566": 770, + "v0=567": 771, + "v0=568": 772, + "v0=569": 773, + "v0=570": 774, + "v0=571": 775, + "v0=572": 776, + "v0=573": 777, + "v0=574": 778, + "v0=575": 779, + "v0=576": 780, + "v0=577": 781, + "v0=578": 782, + "v0=579": 783, + "v0=580": 784, + "v0=581": 785, + "v0=582": 786, + "v0=583": 787, + "v0=584": 788, + "v0=585": 789, + "v0=586": 790, + "v0=587": 791, + "v0=588": 792, + "v0=589": 793, + "v0=590": 794, + "v0=591": 795, + "v0=592": 796, + "v0=593": 797, + "v0=594": 798, + "v0=595": 799, + "v0=596": 800, + "v0=597": 801, + "v0=598": 802, + "v0=599": 803, + "v0=600": 804, + "v0=601": 805, + "v0=602": 806, + "v0=603": 807, + "v0=604": 808, + "v0=605": 809, + "v0=606": 810, + "v0=607": 811, + "v0=608": 812, + "v0=609": 813, + "v0=610": 814, + "v0=611": 815, + "v0=612": 816, + "v0=613": 817, + "v0=614": 818, + "v0=615": 819, + "v0=616": 820, + "v0=617": 821, + "v0=618": 822, + "v0=619": 823, + "v0=620": 824, + "v0=621": 825, + "v0=622": 826, + "v0=623": 827, + "v0=624": 828, + "v0=625": 829, + "v0=626": 830, + "v0=627": 831, + "v0=628": 832, + "v0=629": 833, + "v0=630": 834, + "v0=631": 835, + "v0=632": 836, + "v0=633": 837, + "v0=634": 838, + "v0=635": 839, + "v0=636": 840, + "v0=637": 841, + "v0=638": 842, + "v0=639": 843, + "v0=640": 844, + "v0=641": 845, + "v0=642": 846, + "v0=643": 847, + "v0=644": 848, + "v0=645": 849, + "v0=646": 850, + "v0=647": 851, + "v0=648": 852, + "v0=649": 853, + "v0=650": 854, + "v0=651": 855, + "v0=652": 856, + "v0=653": 857, + "v0=654": 858, + "v0=655": 859, + "v0=656": 860, + "v0=657": 861, + "v0=658": 862, + "v0=659": 863, + "v0=660": 864, + "v0=661": 865, + "v0=662": 866, + "v0=663": 867, + "v0=664": 868, + "v0=665": 869, + "v0=666": 870, + "v0=667": 871, + "v0=668": 872, + "v0=669": 873, + "v0=670": 874, + "v0=671": 875, + "v0=672": 876, + "v0=673": 877, + "v0=674": 878, + "v0=675": 879, + "v0=676": 880, + "v0=677": 881, + "v0=678": 882, + "v0=679": 883, + "v0=680": 884, + "v0=681": 885, + "v0=682": 886, + "v0=683": 887, + "v0=684": 888, + "v0=685": 889, + "v0=686": 890, + "v0=687": 891, + "v0=688": 892, + "v0=689": 893, + "v0=690": 894, + "v0=691": 895, + "v0=692": 896, + "v0=693": 897, + "v0=694": 898, + "v0=695": 899, + "v0=696": 900, + "v0=697": 901, + "v0=698": 902, + "v0=699": 903, + "v0=700": 904, + "v0=701": 905, + "v0=702": 906, + "v0=703": 907, + "v0=704": 908, + "v0=705": 909, + "v0=706": 910, + "v0=707": 911, + "v0=708": 912, + "v0=709": 913, + "v0=710": 914, + "v0=711": 915, + "v0=712": 916, + "v0=713": 917, + "v0=714": 918, + "v0=715": 919, + "v0=716": 920, + "v0=717": 921, + "v0=718": 922, + "v0=719": 923, + "v0=720": 924, + "v0=721": 925, + "v0=722": 926, + "v0=723": 927, + "v0=724": 928, + "v0=725": 929, + "v0=726": 930, + "v0=727": 931, + "v0=728": 932, + "v0=729": 933, + "v0=730": 934, + "v0=731": 935, + "v0=732": 936, + "v0=733": 937, + "v0=734": 938, + "v0=735": 939, + "v0=736": 940, + "v0=737": 941, + "v0=738": 942, + "v0=739": 943, + "v0=740": 944, + "v0=741": 945, + "v0=742": 946, + "v0=743": 947, + "v0=744": 948, + "v0=745": 949, + "v0=746": 950, + "v0=747": 951, + "v0=748": 952, + "v0=749": 953, + "v0=750": 954, + "v0=751": 955, + "v0=752": 956, + "v0=753": 957, + "v0=754": 958, + "v0=755": 959, + "v0=756": 960, + "v0=757": 961, + "v0=758": 962, + "v0=759": 963, + "v0=760": 964, + "v0=761": 965, + "v0=762": 966, + "v0=763": 967, + "v0=764": 968, + "v0=765": 969, + "v0=766": 970, + "v0=767": 971, + "v0=768": 972, + "v0=769": 973, + "v0=770": 974, + "v0=771": 975, + "v0=772": 976, + "v0=773": 977, + "v0=774": 978, + "v0=775": 979, + "v0=776": 980, + "v0=777": 981, + "v0=778": 982, + "v0=779": 983, + "v0=780": 984, + "v0=781": 985, + "v0=782": 986, + "v0=783": 987, + "v0=784": 988, + "v0=785": 989, + "v0=786": 990, + "v0=787": 991, + "v0=788": 992, + "v0=789": 993, + "v0=790": 994, + "v0=791": 995, + "v0=792": 996, + "v0=793": 997, + "v0=794": 998, + "v0=795": 999, + "v0=796": 1000, + "v0=797": 1001, + "v0=798": 1002, + "v0=799": 1003, + "v0=800": 1004, + "v0=801": 1005, + "v0=802": 1006, + "v0=803": 1007, + "v0=804": 1008, + "v0=805": 1009, + "v0=806": 1010, + "v0=807": 1011, + "v0=808": 1012, + "v0=809": 1013, + "v0=810": 1014, + "v0=811": 1015, + "v0=812": 1016, + "v0=813": 1017, + "v0=814": 1018, + "v0=815": 1019, + "v0=816": 1020, + "v0=817": 1021, + "v0=818": 1022, + "v0=819": 1023, + "v0=820": 1024, + "v0=821": 1025, + "v0=822": 1026, + "v0=823": 1027, + "v0=824": 1028, + "v0=825": 1029, + "v0=826": 1030, + "v0=827": 1031, + "v0=828": 1032, + "v0=829": 1033, + "v0=830": 1034, + "v0=831": 1035, + "v0=832": 1036, + "v0=833": 1037, + "v0=834": 1038, + "v0=835": 1039, + "v0=836": 1040, + "v0=837": 1041, + "v0=838": 1042, + "v0=839": 1043, + "v0=840": 1044, + "v0=841": 1045, + "v0=842": 1046, + "v0=843": 1047, + "v0=844": 1048, + "v0=845": 1049, + "v0=846": 1050, + "v0=847": 1051, + "v0=848": 1052, + "v0=849": 1053, + "v0=850": 1054, + "v0=851": 1055, + "v0=852": 1056, + "v0=853": 1057, + "v0=854": 1058, + "v0=855": 1059, + "v0=856": 1060, + "v0=857": 1061, + "v0=858": 1062, + "v0=859": 1063, + "v0=860": 1064, + "v0=861": 1065, + "v0=862": 1066, + "v0=863": 1067, + "v0=864": 1068, + "v0=865": 1069, + "v0=866": 1070, + "v0=867": 1071, + "v0=868": 1072, + "v0=869": 1073, + "v0=870": 1074, + "v0=871": 1075, + "v0=872": 1076, + "v0=873": 1077, + "v0=874": 1078, + "v0=875": 1079, + "v0=876": 1080, + "v0=877": 1081, + "v0=878": 1082, + "v0=879": 1083, + "v0=880": 1084, + "v0=881": 1085, + "v0=882": 1086, + "v0=883": 1087, + "v0=884": 1088, + "v0=885": 1089, + "v0=886": 1090, + "v0=887": 1091, + "v0=888": 1092, + "v0=889": 1093, + "v0=890": 1094, + "v0=891": 1095, + "v0=892": 1096, + "v0=893": 1097, + "v0=894": 1098, + "v0=895": 1099, + "v0=896": 1100, + "v0=897": 1101, + "v0=898": 1102, + "v0=899": 1103, + "v0=900": 1104, + "v0=901": 1105, + "v0=902": 1106, + "v0=903": 1107, + "v0=904": 1108, + "v0=905": 1109, + "v0=906": 1110, + "v0=907": 1111, + "v0=908": 1112, + "v0=909": 1113, + "v0=910": 1114, + "v0=911": 1115, + "v0=912": 1116, + "v0=913": 1117, + "v0=914": 1118, + "v0=915": 1119, + "v0=916": 1120, + "v0=917": 1121, + "v0=918": 1122, + "v0=919": 1123, + "v0=920": 1124, + "v0=921": 1125, + "v0=922": 1126, + "v0=923": 1127, + "v0=924": 1128, + "v0=925": 1129, + "v0=926": 1130, + "v0=927": 1131, + "v0=928": 1132, + "v0=929": 1133, + "v0=930": 1134, + "v0=931": 1135, + "v0=932": 1136, + "v0=933": 1137, + "v0=934": 1138, + "v0=935": 1139, + "v0=936": 1140, + "v0=937": 1141, + "v0=938": 1142, + "v0=939": 1143, + "v0=940": 1144, + "v0=941": 1145, + "v0=942": 1146, + "v0=943": 1147, + "v0=944": 1148, + "v0=945": 1149, + "v0=946": 1150, + "v0=947": 1151, + "v0=948": 1152, + "v0=949": 1153, + "v0=950": 1154, + "v0=951": 1155, + "v0=952": 1156, + "v0=953": 1157, + "v0=954": 1158, + "v0=955": 1159, + "v0=956": 1160, + "v0=957": 1161, + "v0=958": 1162, + "v0=959": 1163, + "v0=960": 1164, + "v0=961": 1165, + "v0=962": 1166, + "v0=963": 1167, + "v0=964": 1168, + "v0=965": 1169, + "v0=966": 1170, + "v0=967": 1171, + "v0=968": 1172, + "v0=969": 1173, + "v0=970": 1174, + "v0=971": 1175, + "v0=972": 1176, + "v0=973": 1177, + "v0=974": 1178, + "v0=975": 1179, + "v0=976": 1180, + "v0=977": 1181, + "v0=978": 1182, + "v0=979": 1183, + "v0=980": 1184, + "v0=981": 1185, + "v0=982": 1186, + "v0=983": 1187, + "v0=984": 1188, + "v0=985": 1189, + "v0=986": 1190, + "v0=987": 1191, + "v0=988": 1192, + "v0=989": 1193, + "v0=990": 1194, + "v0=991": 1195, + "v0=992": 1196, + "v0=993": 1197, + "v0=994": 1198, + "v0=995": 1199, + "v0=996": 1200, + "v0=997": 1201, + "v0=998": 1202, + "v0=999": 1203, + "v1=0": 1204, + "v1=1": 1205, + "v1=2": 1206, + "v1=3": 1207, + "v1=4": 1208, + "v1=5": 1209, + "v1=6": 1210, + "v1=7": 1211, + "v1=8": 1212, + "v1=9": 1213, + "v1=10": 1214, + "v1=11": 1215, + "v1=12": 1216, + "v1=13": 1217, + "v1=14": 1218, + "v1=15": 1219, + "v1=16": 1220, + "v1=17": 1221, + "v1=18": 1222, + "v1=19": 1223, + "v1=20": 1224, + "v1=21": 1225, + "v1=22": 1226, + "v1=23": 1227, + "v1=24": 1228, + "v1=25": 1229, + "v1=26": 1230, + "v1=27": 1231, + "v1=28": 1232, + "v1=29": 1233, + "v1=30": 1234, + "v1=31": 1235, + "v1=32": 1236, + "v1=33": 1237, + "v1=34": 1238, + "v1=35": 1239, + "v1=36": 1240, + "v1=37": 1241, + "v1=38": 1242, + "v1=39": 1243, + "v1=40": 1244, + "v1=41": 1245, + "v1=42": 1246, + "v1=43": 1247, + "v1=44": 1248, + "v1=45": 1249, + "v1=46": 1250, + "v1=47": 1251, + "v1=48": 1252, + "v1=49": 1253, + "v1=50": 1254, + "v1=51": 1255, + "v1=52": 1256, + "v1=53": 1257, + "v1=54": 1258, + "v1=55": 1259, + "v1=56": 1260, + "v1=57": 1261, + "v1=58": 1262, + "v1=59": 1263, + "v1=60": 1264, + "v1=61": 1265, + "v1=62": 1266, + "v1=63": 1267, + "v1=64": 1268, + "v1=65": 1269, + "v1=66": 1270, + "v1=67": 1271, + "v1=68": 1272, + "v1=69": 1273, + "v1=70": 1274, + "v1=71": 1275, + "v1=72": 1276, + "v1=73": 1277, + "v1=74": 1278, + "v1=75": 1279, + "v1=76": 1280, + "v1=77": 1281, + "v1=78": 1282, + "v1=79": 1283, + "v1=80": 1284, + "v1=81": 1285, + "v1=82": 1286, + "v1=83": 1287, + "v1=84": 1288, + "v1=85": 1289, + "v1=86": 1290, + "v1=87": 1291, + "v1=88": 1292, + "v1=89": 1293, + "v1=90": 1294, + "v1=91": 1295, + "v1=92": 1296, + "v1=93": 1297, + "v1=94": 1298, + "v1=95": 1299, + "v1=96": 1300, + "v1=97": 1301, + "v1=98": 1302, + "v1=99": 1303, + "v1=100": 1304, + "v1=101": 1305, + "v1=102": 1306, + "v1=103": 1307, + "v1=104": 1308, + "v1=105": 1309, + "v1=106": 1310, + "v1=107": 1311, + "v1=108": 1312, + "v1=109": 1313, + "v1=110": 1314, + "v1=111": 1315, + "v1=112": 1316, + "v1=113": 1317, + "v1=114": 1318, + "v1=115": 1319, + "v1=116": 1320, + "v1=117": 1321, + "v1=118": 1322, + "v1=119": 1323, + "v1=120": 1324, + "v1=121": 1325, + "v1=122": 1326, + "v1=123": 1327, + "v1=124": 1328, + "v1=125": 1329, + "v1=126": 1330, + "v1=127": 1331, + "v1=128": 1332, + "v1=129": 1333, + "v1=130": 1334, + "v1=131": 1335, + "v1=132": 1336, + "v1=133": 1337, + "v1=134": 1338, + "v1=135": 1339, + "v1=136": 1340, + "v1=137": 1341, + "v1=138": 1342, + "v1=139": 1343, + "v1=140": 1344, + "v1=141": 1345, + "v1=142": 1346, + "v1=143": 1347, + "v1=144": 1348, + "v1=145": 1349, + "v1=146": 1350, + "v1=147": 1351, + "v1=148": 1352, + "v1=149": 1353, + "v1=150": 1354, + "v1=151": 1355, + "v1=152": 1356, + "v1=153": 1357, + "v1=154": 1358, + "v1=155": 1359, + "v1=156": 1360, + "v1=157": 1361, + "v1=158": 1362, + "v1=159": 1363, + "v1=160": 1364, + "v1=161": 1365, + "v1=162": 1366, + "v1=163": 1367, + "v1=164": 1368, + "v1=165": 1369, + "v1=166": 1370, + "v1=167": 1371, + "v1=168": 1372, + "v1=169": 1373, + "v1=170": 1374, + "v1=171": 1375, + "v1=172": 1376, + "v1=173": 1377, + "v1=174": 1378, + "v1=175": 1379, + "v1=176": 1380, + "v1=177": 1381, + "v1=178": 1382, + "v1=179": 1383, + "v1=180": 1384, + "v1=181": 1385, + "v1=182": 1386, + "v1=183": 1387, + "v1=184": 1388, + "v1=185": 1389, + "v1=186": 1390, + "v1=187": 1391, + "v1=188": 1392, + "v1=189": 1393, + "v1=190": 1394, + "v1=191": 1395, + "v1=192": 1396, + "v1=193": 1397, + "v1=194": 1398, + "v1=195": 1399, + "v1=196": 1400, + "v1=197": 1401, + "v1=198": 1402, + "v1=199": 1403, + "v1=200": 1404, + "v1=201": 1405, + "v1=202": 1406, + "v1=203": 1407, + "v1=204": 1408, + "v1=205": 1409, + "v1=206": 1410, + "v1=207": 1411, + "v1=208": 1412, + "v1=209": 1413, + "v1=210": 1414, + "v1=211": 1415, + "v1=212": 1416, + "v1=213": 1417, + "v1=214": 1418, + "v1=215": 1419, + "v1=216": 1420, + "v1=217": 1421, + "v1=218": 1422, + "v1=219": 1423, + "v1=220": 1424, + "v1=221": 1425, + "v1=222": 1426, + "v1=223": 1427, + "v1=224": 1428, + "v1=225": 1429, + "v1=226": 1430, + "v1=227": 1431, + "v1=228": 1432, + "v1=229": 1433, + "v1=230": 1434, + "v1=231": 1435, + "v1=232": 1436, + "v1=233": 1437, + "v1=234": 1438, + "v1=235": 1439, + "v1=236": 1440, + "v1=237": 1441, + "v1=238": 1442, + "v1=239": 1443, + "v1=240": 1444, + "v1=241": 1445, + "v1=242": 1446, + "v1=243": 1447, + "v1=244": 1448, + "v1=245": 1449, + "v1=246": 1450, + "v1=247": 1451, + "v1=248": 1452, + "v1=249": 1453, + "v1=250": 1454, + "v1=251": 1455, + "v1=252": 1456, + "v1=253": 1457, + "v1=254": 1458, + "v1=255": 1459, + "v1=256": 1460, + "v1=257": 1461, + "v1=258": 1462, + "v1=259": 1463, + "v1=260": 1464, + "v1=261": 1465, + "v1=262": 1466, + "v1=263": 1467, + "v1=264": 1468, + "v1=265": 1469, + "v1=266": 1470, + "v1=267": 1471, + "v1=268": 1472, + "v1=269": 1473, + "v1=270": 1474, + "v1=271": 1475, + "v1=272": 1476, + "v1=273": 1477, + "v1=274": 1478, + "v1=275": 1479, + "v1=276": 1480, + "v1=277": 1481, + "v1=278": 1482, + "v1=279": 1483, + "v1=280": 1484, + "v1=281": 1485, + "v1=282": 1486, + "v1=283": 1487, + "v1=284": 1488, + "v1=285": 1489, + "v1=286": 1490, + "v1=287": 1491, + "v1=288": 1492, + "v1=289": 1493, + "v1=290": 1494, + "v1=291": 1495, + "v1=292": 1496, + "v1=293": 1497, + "v1=294": 1498, + "v1=295": 1499, + "v1=296": 1500, + "v1=297": 1501, + "v1=298": 1502, + "v1=299": 1503, + "v1=300": 1504, + "v1=301": 1505, + "v1=302": 1506, + "v1=303": 1507, + "v1=304": 1508, + "v1=305": 1509, + "v1=306": 1510, + "v1=307": 1511, + "v1=308": 1512, + "v1=309": 1513, + "v1=310": 1514, + "v1=311": 1515, + "v1=312": 1516, + "v1=313": 1517, + "v1=314": 1518, + "v1=315": 1519, + "v1=316": 1520, + "v1=317": 1521, + "v1=318": 1522, + "v1=319": 1523, + "v1=320": 1524, + "v1=321": 1525, + "v1=322": 1526, + "v1=323": 1527, + "v1=324": 1528, + "v1=325": 1529, + "v1=326": 1530, + "v1=327": 1531, + "v1=328": 1532, + "v1=329": 1533, + "v1=330": 1534, + "v1=331": 1535, + "v1=332": 1536, + "v1=333": 1537, + "v1=334": 1538, + "v1=335": 1539, + "v1=336": 1540, + "v1=337": 1541, + "v1=338": 1542, + "v1=339": 1543, + "v1=340": 1544, + "v1=341": 1545, + "v1=342": 1546, + "v1=343": 1547, + "v1=344": 1548, + "v1=345": 1549, + "v1=346": 1550, + "v1=347": 1551, + "v1=348": 1552, + "v1=349": 1553, + "v1=350": 1554, + "v1=351": 1555, + "v1=352": 1556, + "v1=353": 1557, + "v1=354": 1558, + "v1=355": 1559, + "v1=356": 1560, + "v1=357": 1561, + "v1=358": 1562, + "v1=359": 1563, + "v1=360": 1564, + "v1=361": 1565, + "v1=362": 1566, + "v1=363": 1567, + "v1=364": 1568, + "v1=365": 1569, + "v1=366": 1570, + "v1=367": 1571, + "v1=368": 1572, + "v1=369": 1573, + "v1=370": 1574, + "v1=371": 1575, + "v1=372": 1576, + "v1=373": 1577, + "v1=374": 1578, + "v1=375": 1579, + "v1=376": 1580, + "v1=377": 1581, + "v1=378": 1582, + "v1=379": 1583, + "v1=380": 1584, + "v1=381": 1585, + "v1=382": 1586, + "v1=383": 1587, + "v1=384": 1588, + "v1=385": 1589, + "v1=386": 1590, + "v1=387": 1591, + "v1=388": 1592, + "v1=389": 1593, + "v1=390": 1594, + "v1=391": 1595, + "v1=392": 1596, + "v1=393": 1597, + "v1=394": 1598, + "v1=395": 1599, + "v1=396": 1600, + "v1=397": 1601, + "v1=398": 1602, + "v1=399": 1603, + "v1=400": 1604, + "v1=401": 1605, + "v1=402": 1606, + "v1=403": 1607, + "v1=404": 1608, + "v1=405": 1609, + "v1=406": 1610, + "v1=407": 1611, + "v1=408": 1612, + "v1=409": 1613, + "v1=410": 1614, + "v1=411": 1615, + "v1=412": 1616, + "v1=413": 1617, + "v1=414": 1618, + "v1=415": 1619, + "v1=416": 1620, + "v1=417": 1621, + "v1=418": 1622, + "v1=419": 1623, + "v1=420": 1624, + "v1=421": 1625, + "v1=422": 1626, + "v1=423": 1627, + "v1=424": 1628, + "v1=425": 1629, + "v1=426": 1630, + "v1=427": 1631, + "v1=428": 1632, + "v1=429": 1633, + "v1=430": 1634, + "v1=431": 1635, + "v1=432": 1636, + "v1=433": 1637, + "v1=434": 1638, + "v1=435": 1639, + "v1=436": 1640, + "v1=437": 1641, + "v1=438": 1642, + "v1=439": 1643, + "v1=440": 1644, + "v1=441": 1645, + "v1=442": 1646, + "v1=443": 1647, + "v1=444": 1648, + "v1=445": 1649, + "v1=446": 1650, + "v1=447": 1651, + "v1=448": 1652, + "v1=449": 1653, + "v1=450": 1654, + "v1=451": 1655, + "v1=452": 1656, + "v1=453": 1657, + "v1=454": 1658, + "v1=455": 1659, + "v1=456": 1660, + "v1=457": 1661, + "v1=458": 1662, + "v1=459": 1663, + "v1=460": 1664, + "v1=461": 1665, + "v1=462": 1666, + "v1=463": 1667, + "v1=464": 1668, + "v1=465": 1669, + "v1=466": 1670, + "v1=467": 1671, + "v1=468": 1672, + "v1=469": 1673, + "v1=470": 1674, + "v1=471": 1675, + "v1=472": 1676, + "v1=473": 1677, + "v1=474": 1678, + "v1=475": 1679, + "v1=476": 1680, + "v1=477": 1681, + "v1=478": 1682, + "v1=479": 1683, + "v1=480": 1684, + "v1=481": 1685, + "v1=482": 1686, + "v1=483": 1687, + "v1=484": 1688, + "v1=485": 1689, + "v1=486": 1690, + "v1=487": 1691, + "v1=488": 1692, + "v1=489": 1693, + "v1=490": 1694, + "v1=491": 1695, + "v1=492": 1696, + "v1=493": 1697, + "v1=494": 1698, + "v1=495": 1699, + "v1=496": 1700, + "v1=497": 1701, + "v1=498": 1702, + "v1=499": 1703, + "v1=500": 1704, + "v1=501": 1705, + "v1=502": 1706, + "v1=503": 1707, + "v1=504": 1708, + "v1=505": 1709, + "v1=506": 1710, + "v1=507": 1711, + "v1=508": 1712, + "v1=509": 1713, + "v1=510": 1714, + "v1=511": 1715, + "v1=512": 1716, + "v1=513": 1717, + "v1=514": 1718, + "v1=515": 1719, + "v1=516": 1720, + "v1=517": 1721, + "v1=518": 1722, + "v1=519": 1723, + "v1=520": 1724, + "v1=521": 1725, + "v1=522": 1726, + "v1=523": 1727, + "v1=524": 1728, + "v1=525": 1729, + "v1=526": 1730, + "v1=527": 1731, + "v1=528": 1732, + "v1=529": 1733, + "v1=530": 1734, + "v1=531": 1735, + "v1=532": 1736, + "v1=533": 1737, + "v1=534": 1738, + "v1=535": 1739, + "v1=536": 1740, + "v1=537": 1741, + "v1=538": 1742, + "v1=539": 1743, + "v1=540": 1744, + "v1=541": 1745, + "v1=542": 1746, + "v1=543": 1747, + "v1=544": 1748, + "v1=545": 1749, + "v1=546": 1750, + "v1=547": 1751, + "v1=548": 1752, + "v1=549": 1753, + "v1=550": 1754, + "v1=551": 1755, + "v1=552": 1756, + "v1=553": 1757, + "v1=554": 1758, + "v1=555": 1759, + "v1=556": 1760, + "v1=557": 1761, + "v1=558": 1762, + "v1=559": 1763, + "v1=560": 1764, + "v1=561": 1765, + "v1=562": 1766, + "v1=563": 1767, + "v1=564": 1768, + "v1=565": 1769, + "v1=566": 1770, + "v1=567": 1771, + "v1=568": 1772, + "v1=569": 1773, + "v1=570": 1774, + "v1=571": 1775, + "v1=572": 1776, + "v1=573": 1777, + "v1=574": 1778, + "v1=575": 1779, + "v1=576": 1780, + "v1=577": 1781, + "v1=578": 1782, + "v1=579": 1783, + "v1=580": 1784, + "v1=581": 1785, + "v1=582": 1786, + "v1=583": 1787, + "v1=584": 1788, + "v1=585": 1789, + "v1=586": 1790, + "v1=587": 1791, + "v1=588": 1792, + "v1=589": 1793, + "v1=590": 1794, + "v1=591": 1795, + "v1=592": 1796, + "v1=593": 1797, + "v1=594": 1798, + "v1=595": 1799, + "v1=596": 1800, + "v1=597": 1801, + "v1=598": 1802, + "v1=599": 1803, + "v1=600": 1804, + "v1=601": 1805, + "v1=602": 1806, + "v1=603": 1807, + "v1=604": 1808, + "v1=605": 1809, + "v1=606": 1810, + "v1=607": 1811, + "v1=608": 1812, + "v1=609": 1813, + "v1=610": 1814, + "v1=611": 1815, + "v1=612": 1816, + "v1=613": 1817, + "v1=614": 1818, + "v1=615": 1819, + "v1=616": 1820, + "v1=617": 1821, + "v1=618": 1822, + "v1=619": 1823, + "v1=620": 1824, + "v1=621": 1825, + "v1=622": 1826, + "v1=623": 1827, + "v1=624": 1828, + "v1=625": 1829, + "v1=626": 1830, + "v1=627": 1831, + "v1=628": 1832, + "v1=629": 1833, + "v1=630": 1834, + "v1=631": 1835, + "v1=632": 1836, + "v1=633": 1837, + "v1=634": 1838, + "v1=635": 1839, + "v1=636": 1840, + "v1=637": 1841, + "v1=638": 1842, + "v1=639": 1843, + "v1=640": 1844, + "v1=641": 1845, + "v1=642": 1846, + "v1=643": 1847, + "v1=644": 1848, + "v1=645": 1849, + "v1=646": 1850, + "v1=647": 1851, + "v1=648": 1852, + "v1=649": 1853, + "v1=650": 1854, + "v1=651": 1855, + "v1=652": 1856, + "v1=653": 1857, + "v1=654": 1858, + "v1=655": 1859, + "v1=656": 1860, + "v1=657": 1861, + "v1=658": 1862, + "v1=659": 1863, + "v1=660": 1864, + "v1=661": 1865, + "v1=662": 1866, + "v1=663": 1867, + "v1=664": 1868, + "v1=665": 1869, + "v1=666": 1870, + "v1=667": 1871, + "v1=668": 1872, + "v1=669": 1873, + "v1=670": 1874, + "v1=671": 1875, + "v1=672": 1876, + "v1=673": 1877, + "v1=674": 1878, + "v1=675": 1879, + "v1=676": 1880, + "v1=677": 1881, + "v1=678": 1882, + "v1=679": 1883, + "v1=680": 1884, + "v1=681": 1885, + "v1=682": 1886, + "v1=683": 1887, + "v1=684": 1888, + "v1=685": 1889, + "v1=686": 1890, + "v1=687": 1891, + "v1=688": 1892, + "v1=689": 1893, + "v1=690": 1894, + "v1=691": 1895, + "v1=692": 1896, + "v1=693": 1897, + "v1=694": 1898, + "v1=695": 1899, + "v1=696": 1900, + "v1=697": 1901, + "v1=698": 1902, + "v1=699": 1903, + "v1=700": 1904, + "v1=701": 1905, + "v1=702": 1906, + "v1=703": 1907, + "v1=704": 1908, + "v1=705": 1909, + "v1=706": 1910, + "v1=707": 1911, + "v1=708": 1912, + "v1=709": 1913, + "v1=710": 1914, + "v1=711": 1915, + "v1=712": 1916, + "v1=713": 1917, + "v1=714": 1918, + "v1=715": 1919, + "v1=716": 1920, + "v1=717": 1921, + "v1=718": 1922, + "v1=719": 1923, + "v1=720": 1924, + "v1=721": 1925, + "v1=722": 1926, + "v1=723": 1927, + "v1=724": 1928, + "v1=725": 1929, + "v1=726": 1930, + "v1=727": 1931, + "v1=728": 1932, + "v1=729": 1933, + "v1=730": 1934, + "v1=731": 1935, + "v1=732": 1936, + "v1=733": 1937, + "v1=734": 1938, + "v1=735": 1939, + "v1=736": 1940, + "v1=737": 1941, + "v1=738": 1942, + "v1=739": 1943, + "v1=740": 1944, + "v1=741": 1945, + "v1=742": 1946, + "v1=743": 1947, + "v1=744": 1948, + "v1=745": 1949, + "v1=746": 1950, + "v1=747": 1951, + "v1=748": 1952, + "v1=749": 1953, + "v1=750": 1954, + "v1=751": 1955, + "v1=752": 1956, + "v1=753": 1957, + "v1=754": 1958, + "v1=755": 1959, + "v1=756": 1960, + "v1=757": 1961, + "v1=758": 1962, + "v1=759": 1963, + "v1=760": 1964, + "v1=761": 1965, + "v1=762": 1966, + "v1=763": 1967, + "v1=764": 1968, + "v1=765": 1969, + "v1=766": 1970, + "v1=767": 1971, + "v1=768": 1972, + "v1=769": 1973, + "v1=770": 1974, + "v1=771": 1975, + "v1=772": 1976, + "v1=773": 1977, + "v1=774": 1978, + "v1=775": 1979, + "v1=776": 1980, + "v1=777": 1981, + "v1=778": 1982, + "v1=779": 1983, + "v1=780": 1984, + "v1=781": 1985, + "v1=782": 1986, + "v1=783": 1987, + "v1=784": 1988, + "v1=785": 1989, + "v1=786": 1990, + "v1=787": 1991, + "v1=788": 1992, + "v1=789": 1993, + "v1=790": 1994, + "v1=791": 1995, + "v1=792": 1996, + "v1=793": 1997, + "v1=794": 1998, + "v1=795": 1999, + "v1=796": 2000, + "v1=797": 2001, + "v1=798": 2002, + "v1=799": 2003, + "v1=800": 2004, + "v1=801": 2005, + "v1=802": 2006, + "v1=803": 2007, + "v1=804": 2008, + "v1=805": 2009, + "v1=806": 2010, + "v1=807": 2011, + "v1=808": 2012, + "v1=809": 2013, + "v1=810": 2014, + "v1=811": 2015, + "v1=812": 2016, + "v1=813": 2017, + "v1=814": 2018, + "v1=815": 2019, + "v1=816": 2020, + "v1=817": 2021, + "v1=818": 2022, + "v1=819": 2023, + "v1=820": 2024, + "v1=821": 2025, + "v1=822": 2026, + "v1=823": 2027, + "v1=824": 2028, + "v1=825": 2029, + "v1=826": 2030, + "v1=827": 2031, + "v1=828": 2032, + "v1=829": 2033, + "v1=830": 2034, + "v1=831": 2035, + "v1=832": 2036, + "v1=833": 2037, + "v1=834": 2038, + "v1=835": 2039, + "v1=836": 2040, + "v1=837": 2041, + "v1=838": 2042, + "v1=839": 2043, + "v1=840": 2044, + "v1=841": 2045, + "v1=842": 2046, + "v1=843": 2047, + "v1=844": 2048, + "v1=845": 2049, + "v1=846": 2050, + "v1=847": 2051, + "v1=848": 2052, + "v1=849": 2053, + "v1=850": 2054, + "v1=851": 2055, + "v1=852": 2056, + "v1=853": 2057, + "v1=854": 2058, + "v1=855": 2059, + "v1=856": 2060, + "v1=857": 2061, + "v1=858": 2062, + "v1=859": 2063, + "v1=860": 2064, + "v1=861": 2065, + "v1=862": 2066, + "v1=863": 2067, + "v1=864": 2068, + "v1=865": 2069, + "v1=866": 2070, + "v1=867": 2071, + "v1=868": 2072, + "v1=869": 2073, + "v1=870": 2074, + "v1=871": 2075, + "v1=872": 2076, + "v1=873": 2077, + "v1=874": 2078, + "v1=875": 2079, + "v1=876": 2080, + "v1=877": 2081, + "v1=878": 2082, + "v1=879": 2083, + "v1=880": 2084, + "v1=881": 2085, + "v1=882": 2086, + "v1=883": 2087, + "v1=884": 2088, + "v1=885": 2089, + "v1=886": 2090, + "v1=887": 2091, + "v1=888": 2092, + "v1=889": 2093, + "v1=890": 2094, + "v1=891": 2095, + "v1=892": 2096, + "v1=893": 2097, + "v1=894": 2098, + "v1=895": 2099, + "v1=896": 2100, + "v1=897": 2101, + "v1=898": 2102, + "v1=899": 2103, + "v1=900": 2104, + "v1=901": 2105, + "v1=902": 2106, + "v1=903": 2107, + "v1=904": 2108, + "v1=905": 2109, + "v1=906": 2110, + "v1=907": 2111, + "v1=908": 2112, + "v1=909": 2113, + "v1=910": 2114, + "v1=911": 2115, + "v1=912": 2116, + "v1=913": 2117, + "v1=914": 2118, + "v1=915": 2119, + "v1=916": 2120, + "v1=917": 2121, + "v1=918": 2122, + "v1=919": 2123, + "v1=920": 2124, + "v1=921": 2125, + "v1=922": 2126, + "v1=923": 2127, + "v1=924": 2128, + "v1=925": 2129, + "v1=926": 2130, + "v1=927": 2131, + "v1=928": 2132, + "v1=929": 2133, + "v1=930": 2134, + "v1=931": 2135, + "v1=932": 2136, + "v1=933": 2137, + "v1=934": 2138, + "v1=935": 2139, + "v1=936": 2140, + "v1=937": 2141, + "v1=938": 2142, + "v1=939": 2143, + "v1=940": 2144, + "v1=941": 2145, + "v1=942": 2146, + "v1=943": 2147, + "v1=944": 2148, + "v1=945": 2149, + "v1=946": 2150, + "v1=947": 2151, + "v1=948": 2152, + "v1=949": 2153, + "v1=950": 2154, + "v1=951": 2155, + "v1=952": 2156, + "v1=953": 2157, + "v1=954": 2158, + "v1=955": 2159, + "v1=956": 2160, + "v1=957": 2161, + "v1=958": 2162, + "v1=959": 2163, + "v1=960": 2164, + "v1=961": 2165, + "v1=962": 2166, + "v1=963": 2167, + "v1=964": 2168, + "v1=965": 2169, + "v1=966": 2170, + "v1=967": 2171, + "v1=968": 2172, + "v1=969": 2173, + "v1=970": 2174, + "v1=971": 2175, + "v1=972": 2176, + "v1=973": 2177, + "v1=974": 2178, + "v1=975": 2179, + "v1=976": 2180, + "v1=977": 2181, + "v1=978": 2182, + "v1=979": 2183, + "v1=980": 2184, + "v1=981": 2185, + "v1=982": 2186, + "v1=983": 2187, + "v1=984": 2188, + "v1=985": 2189, + "v1=986": 2190, + "v1=987": 2191, + "v1=988": 2192, + "v1=989": 2193, + "v1=990": 2194, + "v1=991": 2195, + "v1=992": 2196, + "v1=993": 2197, + "v1=994": 2198, + "v1=995": 2199, + "v1=996": 2200, + "v1=997": 2201, + "v1=998": 2202, + "v1=999": 2203, + "v2=0": 2204, + "v2=1": 2205, + "v2=2": 2206, + "v2=3": 2207, + "v2=4": 2208, + "v2=5": 2209, + "v2=6": 2210, + "v2=7": 2211, + "v2=8": 2212, + "v2=9": 2213, + "v2=10": 2214, + "v2=11": 2215, + "v2=12": 2216, + "v2=13": 2217, + "v2=14": 2218, + "v2=15": 2219, + "v2=16": 2220, + "v2=17": 2221, + "v2=18": 2222, + "v2=19": 2223, + "v2=20": 2224, + "v2=21": 2225, + "v2=22": 2226, + "v2=23": 2227, + "v2=24": 2228, + "v2=25": 2229, + "v2=26": 2230, + "v2=27": 2231, + "v2=28": 2232, + "v2=29": 2233, + "v2=30": 2234, + "v2=31": 2235, + "v2=32": 2236, + "v2=33": 2237, + "v2=34": 2238, + "v2=35": 2239, + "v2=36": 2240, + "v2=37": 2241, + "v2=38": 2242, + "v2=39": 2243, + "v2=40": 2244, + "v2=41": 2245, + "v2=42": 2246, + "v2=43": 2247, + "v2=44": 2248, + "v2=45": 2249, + "v2=46": 2250, + "v2=47": 2251, + "v2=48": 2252, + "v2=49": 2253, + "v2=50": 2254, + "v2=51": 2255, + "v2=52": 2256, + "v2=53": 2257, + "v2=54": 2258, + "v2=55": 2259, + "v2=56": 2260, + "v2=57": 2261, + "v2=58": 2262, + "v2=59": 2263, + "v2=60": 2264, + "v2=61": 2265, + "v2=62": 2266, + "v2=63": 2267, + "v2=64": 2268, + "v2=65": 2269, + "v2=66": 2270, + "v2=67": 2271, + "v2=68": 2272, + "v2=69": 2273, + "v2=70": 2274, + "v2=71": 2275, + "v2=72": 2276, + "v2=73": 2277, + "v2=74": 2278, + "v2=75": 2279, + "v2=76": 2280, + "v2=77": 2281, + "v2=78": 2282, + "v2=79": 2283, + "v2=80": 2284, + "v2=81": 2285, + "v2=82": 2286, + "v2=83": 2287, + "v2=84": 2288, + "v2=85": 2289, + "v2=86": 2290, + "v2=87": 2291, + "v2=88": 2292, + "v2=89": 2293, + "v2=90": 2294, + "v2=91": 2295, + "v2=92": 2296, + "v2=93": 2297, + "v2=94": 2298, + "v2=95": 2299, + "v2=96": 2300, + "v2=97": 2301, + "v2=98": 2302, + "v2=99": 2303, + "v2=100": 2304, + "v2=101": 2305, + "v2=102": 2306, + "v2=103": 2307, + "v2=104": 2308, + "v2=105": 2309, + "v2=106": 2310, + "v2=107": 2311, + "v2=108": 2312, + "v2=109": 2313, + "v2=110": 2314, + "v2=111": 2315, + "v2=112": 2316, + "v2=113": 2317, + "v2=114": 2318, + "v2=115": 2319, + "v2=116": 2320, + "v2=117": 2321, + "v2=118": 2322, + "v2=119": 2323, + "v2=120": 2324, + "v2=121": 2325, + "v2=122": 2326, + "v2=123": 2327, + "v2=124": 2328, + "v2=125": 2329, + "v2=126": 2330, + "v2=127": 2331, + "v2=128": 2332, + "v2=129": 2333, + "v2=130": 2334, + "v2=131": 2335, + "v2=132": 2336, + "v2=133": 2337, + "v2=134": 2338, + "v2=135": 2339, + "v2=136": 2340, + "v2=137": 2341, + "v2=138": 2342, + "v2=139": 2343, + "v2=140": 2344, + "v2=141": 2345, + "v2=142": 2346, + "v2=143": 2347, + "v2=144": 2348, + "v2=145": 2349, + "v2=146": 2350, + "v2=147": 2351, + "v2=148": 2352, + "v2=149": 2353, + "v2=150": 2354, + "v2=151": 2355, + "v2=152": 2356, + "v2=153": 2357, + "v2=154": 2358, + "v2=155": 2359, + "v2=156": 2360, + "v2=157": 2361, + "v2=158": 2362, + "v2=159": 2363, + "v2=160": 2364, + "v2=161": 2365, + "v2=162": 2366, + "v2=163": 2367, + "v2=164": 2368, + "v2=165": 2369, + "v2=166": 2370, + "v2=167": 2371, + "v2=168": 2372, + "v2=169": 2373, + "v2=170": 2374, + "v2=171": 2375, + "v2=172": 2376, + "v2=173": 2377, + "v2=174": 2378, + "v2=175": 2379, + "v2=176": 2380, + "v2=177": 2381, + "v2=178": 2382, + "v2=179": 2383, + "v2=180": 2384, + "v2=181": 2385, + "v2=182": 2386, + "v2=183": 2387, + "v2=184": 2388, + "v2=185": 2389, + "v2=186": 2390, + "v2=187": 2391, + "v2=188": 2392, + "v2=189": 2393, + "v2=190": 2394, + "v2=191": 2395, + "v2=192": 2396, + "v2=193": 2397, + "v2=194": 2398, + "v2=195": 2399, + "v2=196": 2400, + "v2=197": 2401, + "v2=198": 2402, + "v2=199": 2403, + "v2=200": 2404, + "v2=201": 2405, + "v2=202": 2406, + "v2=203": 2407, + "v2=204": 2408, + "v2=205": 2409, + "v2=206": 2410, + "v2=207": 2411, + "v2=208": 2412, + "v2=209": 2413, + "v2=210": 2414, + "v2=211": 2415, + "v2=212": 2416, + "v2=213": 2417, + "v2=214": 2418, + "v2=215": 2419, + "v2=216": 2420, + "v2=217": 2421, + "v2=218": 2422, + "v2=219": 2423, + "v2=220": 2424, + "v2=221": 2425, + "v2=222": 2426, + "v2=223": 2427, + "v2=224": 2428, + "v2=225": 2429, + "v2=226": 2430, + "v2=227": 2431, + "v2=228": 2432, + "v2=229": 2433, + "v2=230": 2434, + "v2=231": 2435, + "v2=232": 2436, + "v2=233": 2437, + "v2=234": 2438, + "v2=235": 2439, + "v2=236": 2440, + "v2=237": 2441, + "v2=238": 2442, + "v2=239": 2443, + "v2=240": 2444, + "v2=241": 2445, + "v2=242": 2446, + "v2=243": 2447, + "v2=244": 2448, + "v2=245": 2449, + "v2=246": 2450, + "v2=247": 2451, + "v2=248": 2452, + "v2=249": 2453, + "v2=250": 2454, + "v2=251": 2455, + "v2=252": 2456, + "v2=253": 2457, + "v2=254": 2458, + "v2=255": 2459, + "v2=256": 2460, + "v2=257": 2461, + "v2=258": 2462, + "v2=259": 2463, + "v2=260": 2464, + "v2=261": 2465, + "v2=262": 2466, + "v2=263": 2467, + "v2=264": 2468, + "v2=265": 2469, + "v2=266": 2470, + "v2=267": 2471, + "v2=268": 2472, + "v2=269": 2473, + "v2=270": 2474, + "v2=271": 2475, + "v2=272": 2476, + "v2=273": 2477, + "v2=274": 2478, + "v2=275": 2479, + "v2=276": 2480, + "v2=277": 2481, + "v2=278": 2482, + "v2=279": 2483, + "v2=280": 2484, + "v2=281": 2485, + "v2=282": 2486, + "v2=283": 2487, + "v2=284": 2488, + "v2=285": 2489, + "v2=286": 2490, + "v2=287": 2491, + "v2=288": 2492, + "v2=289": 2493, + "v2=290": 2494, + "v2=291": 2495, + "v2=292": 2496, + "v2=293": 2497, + "v2=294": 2498, + "v2=295": 2499, + "v2=296": 2500, + "v2=297": 2501, + "v2=298": 2502, + "v2=299": 2503, + "v2=300": 2504, + "v2=301": 2505, + "v2=302": 2506, + "v2=303": 2507, + "v2=304": 2508, + "v2=305": 2509, + "v2=306": 2510, + "v2=307": 2511, + "v2=308": 2512, + "v2=309": 2513, + "v2=310": 2514, + "v2=311": 2515, + "v2=312": 2516, + "v2=313": 2517, + "v2=314": 2518, + "v2=315": 2519, + "v2=316": 2520, + "v2=317": 2521, + "v2=318": 2522, + "v2=319": 2523, + "v2=320": 2524, + "v2=321": 2525, + "v2=322": 2526, + "v2=323": 2527, + "v2=324": 2528, + "v2=325": 2529, + "v2=326": 2530, + "v2=327": 2531, + "v2=328": 2532, + "v2=329": 2533, + "v2=330": 2534, + "v2=331": 2535, + "v2=332": 2536, + "v2=333": 2537, + "v2=334": 2538, + "v2=335": 2539, + "v2=336": 2540, + "v2=337": 2541, + "v2=338": 2542, + "v2=339": 2543, + "v2=340": 2544, + "v2=341": 2545, + "v2=342": 2546, + "v2=343": 2547, + "v2=344": 2548, + "v2=345": 2549, + "v2=346": 2550, + "v2=347": 2551, + "v2=348": 2552, + "v2=349": 2553, + "v2=350": 2554, + "v2=351": 2555, + "v2=352": 2556, + "v2=353": 2557, + "v2=354": 2558, + "v2=355": 2559, + "v2=356": 2560, + "v2=357": 2561, + "v2=358": 2562, + "v2=359": 2563, + "v2=360": 2564, + "v2=361": 2565, + "v2=362": 2566, + "v2=363": 2567, + "v2=364": 2568, + "v2=365": 2569, + "v2=366": 2570, + "v2=367": 2571, + "v2=368": 2572, + "v2=369": 2573, + "v2=370": 2574, + "v2=371": 2575, + "v2=372": 2576, + "v2=373": 2577, + "v2=374": 2578, + "v2=375": 2579, + "v2=376": 2580, + "v2=377": 2581, + "v2=378": 2582, + "v2=379": 2583, + "v2=380": 2584, + "v2=381": 2585, + "v2=382": 2586, + "v2=383": 2587, + "v2=384": 2588, + "v2=385": 2589, + "v2=386": 2590, + "v2=387": 2591, + "v2=388": 2592, + "v2=389": 2593, + "v2=390": 2594, + "v2=391": 2595, + "v2=392": 2596, + "v2=393": 2597, + "v2=394": 2598, + "v2=395": 2599, + "v2=396": 2600, + "v2=397": 2601, + "v2=398": 2602, + "v2=399": 2603, + "v2=400": 2604, + "v2=401": 2605, + "v2=402": 2606, + "v2=403": 2607, + "v2=404": 2608, + "v2=405": 2609, + "v2=406": 2610, + "v2=407": 2611, + "v2=408": 2612, + "v2=409": 2613, + "v2=410": 2614, + "v2=411": 2615, + "v2=412": 2616, + "v2=413": 2617, + "v2=414": 2618, + "v2=415": 2619, + "v2=416": 2620, + "v2=417": 2621, + "v2=418": 2622, + "v2=419": 2623, + "v2=420": 2624, + "v2=421": 2625, + "v2=422": 2626, + "v2=423": 2627, + "v2=424": 2628, + "v2=425": 2629, + "v2=426": 2630, + "v2=427": 2631, + "v2=428": 2632, + "v2=429": 2633, + "v2=430": 2634, + "v2=431": 2635, + "v2=432": 2636, + "v2=433": 2637, + "v2=434": 2638, + "v2=435": 2639, + "v2=436": 2640, + "v2=437": 2641, + "v2=438": 2642, + "v2=439": 2643, + "v2=440": 2644, + "v2=441": 2645, + "v2=442": 2646, + "v2=443": 2647, + "v2=444": 2648, + "v2=445": 2649, + "v2=446": 2650, + "v2=447": 2651, + "v2=448": 2652, + "v2=449": 2653, + "v2=450": 2654, + "v2=451": 2655, + "v2=452": 2656, + "v2=453": 2657, + "v2=454": 2658, + "v2=455": 2659, + "v2=456": 2660, + "v2=457": 2661, + "v2=458": 2662, + "v2=459": 2663, + "v2=460": 2664, + "v2=461": 2665, + "v2=462": 2666, + "v2=463": 2667, + "v2=464": 2668, + "v2=465": 2669, + "v2=466": 2670, + "v2=467": 2671, + "v2=468": 2672, + "v2=469": 2673, + "v2=470": 2674, + "v2=471": 2675, + "v2=472": 2676, + "v2=473": 2677, + "v2=474": 2678, + "v2=475": 2679, + "v2=476": 2680, + "v2=477": 2681, + "v2=478": 2682, + "v2=479": 2683, + "v2=480": 2684, + "v2=481": 2685, + "v2=482": 2686, + "v2=483": 2687, + "v2=484": 2688, + "v2=485": 2689, + "v2=486": 2690, + "v2=487": 2691, + "v2=488": 2692, + "v2=489": 2693, + "v2=490": 2694, + "v2=491": 2695, + "v2=492": 2696, + "v2=493": 2697, + "v2=494": 2698, + "v2=495": 2699, + "v2=496": 2700, + "v2=497": 2701, + "v2=498": 2702, + "v2=499": 2703, + "v2=500": 2704, + "v2=501": 2705, + "v2=502": 2706, + "v2=503": 2707, + "v2=504": 2708, + "v2=505": 2709, + "v2=506": 2710, + "v2=507": 2711, + "v2=508": 2712, + "v2=509": 2713, + "v2=510": 2714, + "v2=511": 2715, + "v2=512": 2716, + "v2=513": 2717, + "v2=514": 2718, + "v2=515": 2719, + "v2=516": 2720, + "v2=517": 2721, + "v2=518": 2722, + "v2=519": 2723, + "v2=520": 2724, + "v2=521": 2725, + "v2=522": 2726, + "v2=523": 2727, + "v2=524": 2728, + "v2=525": 2729, + "v2=526": 2730, + "v2=527": 2731, + "v2=528": 2732, + "v2=529": 2733, + "v2=530": 2734, + "v2=531": 2735, + "v2=532": 2736, + "v2=533": 2737, + "v2=534": 2738, + "v2=535": 2739, + "v2=536": 2740, + "v2=537": 2741, + "v2=538": 2742, + "v2=539": 2743, + "v2=540": 2744, + "v2=541": 2745, + "v2=542": 2746, + "v2=543": 2747, + "v2=544": 2748, + "v2=545": 2749, + "v2=546": 2750, + "v2=547": 2751, + "v2=548": 2752, + "v2=549": 2753, + "v2=550": 2754, + "v2=551": 2755, + "v2=552": 2756, + "v2=553": 2757, + "v2=554": 2758, + "v2=555": 2759, + "v2=556": 2760, + "v2=557": 2761, + "v2=558": 2762, + "v2=559": 2763, + "v2=560": 2764, + "v2=561": 2765, + "v2=562": 2766, + "v2=563": 2767, + "v2=564": 2768, + "v2=565": 2769, + "v2=566": 2770, + "v2=567": 2771, + "v2=568": 2772, + "v2=569": 2773, + "v2=570": 2774, + "v2=571": 2775, + "v2=572": 2776, + "v2=573": 2777, + "v2=574": 2778, + "v2=575": 2779, + "v2=576": 2780, + "v2=577": 2781, + "v2=578": 2782, + "v2=579": 2783, + "v2=580": 2784, + "v2=581": 2785, + "v2=582": 2786, + "v2=583": 2787, + "v2=584": 2788, + "v2=585": 2789, + "v2=586": 2790, + "v2=587": 2791, + "v2=588": 2792, + "v2=589": 2793, + "v2=590": 2794, + "v2=591": 2795, + "v2=592": 2796, + "v2=593": 2797, + "v2=594": 2798, + "v2=595": 2799, + "v2=596": 2800, + "v2=597": 2801, + "v2=598": 2802, + "v2=599": 2803, + "v2=600": 2804, + "v2=601": 2805, + "v2=602": 2806, + "v2=603": 2807, + "v2=604": 2808, + "v2=605": 2809, + "v2=606": 2810, + "v2=607": 2811, + "v2=608": 2812, + "v2=609": 2813, + "v2=610": 2814, + "v2=611": 2815, + "v2=612": 2816, + "v2=613": 2817, + "v2=614": 2818, + "v2=615": 2819, + "v2=616": 2820, + "v2=617": 2821, + "v2=618": 2822, + "v2=619": 2823, + "v2=620": 2824, + "v2=621": 2825, + "v2=622": 2826, + "v2=623": 2827, + "v2=624": 2828, + "v2=625": 2829, + "v2=626": 2830, + "v2=627": 2831, + "v2=628": 2832, + "v2=629": 2833, + "v2=630": 2834, + "v2=631": 2835, + "v2=632": 2836, + "v2=633": 2837, + "v2=634": 2838, + "v2=635": 2839, + "v2=636": 2840, + "v2=637": 2841, + "v2=638": 2842, + "v2=639": 2843, + "v2=640": 2844, + "v2=641": 2845, + "v2=642": 2846, + "v2=643": 2847, + "v2=644": 2848, + "v2=645": 2849, + "v2=646": 2850, + "v2=647": 2851, + "v2=648": 2852, + "v2=649": 2853, + "v2=650": 2854, + "v2=651": 2855, + "v2=652": 2856, + "v2=653": 2857, + "v2=654": 2858, + "v2=655": 2859, + "v2=656": 2860, + "v2=657": 2861, + "v2=658": 2862, + "v2=659": 2863, + "v2=660": 2864, + "v2=661": 2865, + "v2=662": 2866, + "v2=663": 2867, + "v2=664": 2868, + "v2=665": 2869, + "v2=666": 2870, + "v2=667": 2871, + "v2=668": 2872, + "v2=669": 2873, + "v2=670": 2874, + "v2=671": 2875, + "v2=672": 2876, + "v2=673": 2877, + "v2=674": 2878, + "v2=675": 2879, + "v2=676": 2880, + "v2=677": 2881, + "v2=678": 2882, + "v2=679": 2883, + "v2=680": 2884, + "v2=681": 2885, + "v2=682": 2886, + "v2=683": 2887, + "v2=684": 2888, + "v2=685": 2889, + "v2=686": 2890, + "v2=687": 2891, + "v2=688": 2892, + "v2=689": 2893, + "v2=690": 2894, + "v2=691": 2895, + "v2=692": 2896, + "v2=693": 2897, + "v2=694": 2898, + "v2=695": 2899, + "v2=696": 2900, + "v2=697": 2901, + "v2=698": 2902, + "v2=699": 2903, + "v2=700": 2904, + "v2=701": 2905, + "v2=702": 2906, + "v2=703": 2907, + "v2=704": 2908, + "v2=705": 2909, + "v2=706": 2910, + "v2=707": 2911, + "v2=708": 2912, + "v2=709": 2913, + "v2=710": 2914, + "v2=711": 2915, + "v2=712": 2916, + "v2=713": 2917, + "v2=714": 2918, + "v2=715": 2919, + "v2=716": 2920, + "v2=717": 2921, + "v2=718": 2922, + "v2=719": 2923, + "v2=720": 2924, + "v2=721": 2925, + "v2=722": 2926, + "v2=723": 2927, + "v2=724": 2928, + "v2=725": 2929, + "v2=726": 2930, + "v2=727": 2931, + "v2=728": 2932, + "v2=729": 2933, + "v2=730": 2934, + "v2=731": 2935, + "v2=732": 2936, + "v2=733": 2937, + "v2=734": 2938, + "v2=735": 2939, + "v2=736": 2940, + "v2=737": 2941, + "v2=738": 2942, + "v2=739": 2943, + "v2=740": 2944, + "v2=741": 2945, + "v2=742": 2946, + "v2=743": 2947, + "v2=744": 2948, + "v2=745": 2949, + "v2=746": 2950, + "v2=747": 2951, + "v2=748": 2952, + "v2=749": 2953, + "v2=750": 2954, + "v2=751": 2955, + "v2=752": 2956, + "v2=753": 2957, + "v2=754": 2958, + "v2=755": 2959, + "v2=756": 2960, + "v2=757": 2961, + "v2=758": 2962, + "v2=759": 2963, + "v2=760": 2964, + "v2=761": 2965, + "v2=762": 2966, + "v2=763": 2967, + "v2=764": 2968, + "v2=765": 2969, + "v2=766": 2970, + "v2=767": 2971, + "v2=768": 2972, + "v2=769": 2973, + "v2=770": 2974, + "v2=771": 2975, + "v2=772": 2976, + "v2=773": 2977, + "v2=774": 2978, + "v2=775": 2979, + "v2=776": 2980, + "v2=777": 2981, + "v2=778": 2982, + "v2=779": 2983, + "v2=780": 2984, + "v2=781": 2985, + "v2=782": 2986, + "v2=783": 2987, + "v2=784": 2988, + "v2=785": 2989, + "v2=786": 2990, + "v2=787": 2991, + "v2=788": 2992, + "v2=789": 2993, + "v2=790": 2994, + "v2=791": 2995, + "v2=792": 2996, + "v2=793": 2997, + "v2=794": 2998, + "v2=795": 2999, + "v2=796": 3000, + "v2=797": 3001, + "v2=798": 3002, + "v2=799": 3003, + "v2=800": 3004, + "v2=801": 3005, + "v2=802": 3006, + "v2=803": 3007, + "v2=804": 3008, + "v2=805": 3009, + "v2=806": 3010, + "v2=807": 3011, + "v2=808": 3012, + "v2=809": 3013, + "v2=810": 3014, + "v2=811": 3015, + "v2=812": 3016, + "v2=813": 3017, + "v2=814": 3018, + "v2=815": 3019, + "v2=816": 3020, + "v2=817": 3021, + "v2=818": 3022, + "v2=819": 3023, + "v2=820": 3024, + "v2=821": 3025, + "v2=822": 3026, + "v2=823": 3027, + "v2=824": 3028, + "v2=825": 3029, + "v2=826": 3030, + "v2=827": 3031, + "v2=828": 3032, + "v2=829": 3033, + "v2=830": 3034, + "v2=831": 3035, + "v2=832": 3036, + "v2=833": 3037, + "v2=834": 3038, + "v2=835": 3039, + "v2=836": 3040, + "v2=837": 3041, + "v2=838": 3042, + "v2=839": 3043, + "v2=840": 3044, + "v2=841": 3045, + "v2=842": 3046, + "v2=843": 3047, + "v2=844": 3048, + "v2=845": 3049, + "v2=846": 3050, + "v2=847": 3051, + "v2=848": 3052, + "v2=849": 3053, + "v2=850": 3054, + "v2=851": 3055, + "v2=852": 3056, + "v2=853": 3057, + "v2=854": 3058, + "v2=855": 3059, + "v2=856": 3060, + "v2=857": 3061, + "v2=858": 3062, + "v2=859": 3063, + "v2=860": 3064, + "v2=861": 3065, + "v2=862": 3066, + "v2=863": 3067, + "v2=864": 3068, + "v2=865": 3069, + "v2=866": 3070, + "v2=867": 3071, + "v2=868": 3072, + "v2=869": 3073, + "v2=870": 3074, + "v2=871": 3075, + "v2=872": 3076, + "v2=873": 3077, + "v2=874": 3078, + "v2=875": 3079, + "v2=876": 3080, + "v2=877": 3081, + "v2=878": 3082, + "v2=879": 3083, + "v2=880": 3084, + "v2=881": 3085, + "v2=882": 3086, + "v2=883": 3087, + "v2=884": 3088, + "v2=885": 3089, + "v2=886": 3090, + "v2=887": 3091, + "v2=888": 3092, + "v2=889": 3093, + "v2=890": 3094, + "v2=891": 3095, + "v2=892": 3096, + "v2=893": 3097, + "v2=894": 3098, + "v2=895": 3099, + "v2=896": 3100, + "v2=897": 3101, + "v2=898": 3102, + "v2=899": 3103, + "v2=900": 3104, + "v2=901": 3105, + "v2=902": 3106, + "v2=903": 3107, + "v2=904": 3108, + "v2=905": 3109, + "v2=906": 3110, + "v2=907": 3111, + "v2=908": 3112, + "v2=909": 3113, + "v2=910": 3114, + "v2=911": 3115, + "v2=912": 3116, + "v2=913": 3117, + "v2=914": 3118, + "v2=915": 3119, + "v2=916": 3120, + "v2=917": 3121, + "v2=918": 3122, + "v2=919": 3123, + "v2=920": 3124, + "v2=921": 3125, + "v2=922": 3126, + "v2=923": 3127, + "v2=924": 3128, + "v2=925": 3129, + "v2=926": 3130, + "v2=927": 3131, + "v2=928": 3132, + "v2=929": 3133, + "v2=930": 3134, + "v2=931": 3135, + "v2=932": 3136, + "v2=933": 3137, + "v2=934": 3138, + "v2=935": 3139, + "v2=936": 3140, + "v2=937": 3141, + "v2=938": 3142, + "v2=939": 3143, + "v2=940": 3144, + "v2=941": 3145, + "v2=942": 3146, + "v2=943": 3147, + "v2=944": 3148, + "v2=945": 3149, + "v2=946": 3150, + "v2=947": 3151, + "v2=948": 3152, + "v2=949": 3153, + "v2=950": 3154, + "v2=951": 3155, + "v2=952": 3156, + "v2=953": 3157, + "v2=954": 3158, + "v2=955": 3159, + "v2=956": 3160, + "v2=957": 3161, + "v2=958": 3162, + "v2=959": 3163, + "v2=960": 3164, + "v2=961": 3165, + "v2=962": 3166, + "v2=963": 3167, + "v2=964": 3168, + "v2=965": 3169, + "v2=966": 3170, + "v2=967": 3171, + "v2=968": 3172, + "v2=969": 3173, + "v2=970": 3174, + "v2=971": 3175, + "v2=972": 3176, + "v2=973": 3177, + "v2=974": 3178, + "v2=975": 3179, + "v2=976": 3180, + "v2=977": 3181, + "v2=978": 3182, + "v2=979": 3183, + "v2=980": 3184, + "v2=981": 3185, + "v2=982": 3186, + "v2=983": 3187, + "v2=984": 3188, + "v2=985": 3189, + "v2=986": 3190, + "v2=987": 3191, + "v2=988": 3192, + "v2=989": 3193, + "v2=990": 3194, + "v2=991": 3195, + "v2=992": 3196, + "v2=993": 3197, + "v2=994": 3198, + "v2=995": 3199, + "v2=996": 3200, + "v2=997": 3201, + "v2=998": 3202, + "v2=999": 3203, + "v3=0": 3204, + "v3=1": 3205, + "v3=2": 3206, + "v3=3": 3207, + "v3=4": 3208, + "v3=5": 3209, + "v3=6": 3210, + "v3=7": 3211, + "v3=8": 3212, + "v3=9": 3213, + "v3=10": 3214, + "v3=11": 3215, + "v3=12": 3216, + "v3=13": 3217, + "v3=14": 3218, + "v3=15": 3219, + "v3=16": 3220, + "v3=17": 3221, + "v3=18": 3222, + "v3=19": 3223, + "v3=20": 3224, + "v3=21": 3225, + "v3=22": 3226, + "v3=23": 3227, + "v3=24": 3228, + "v3=25": 3229, + "v3=26": 3230, + "v3=27": 3231, + "v3=28": 3232, + "v3=29": 3233, + "v3=30": 3234, + "v3=31": 3235, + "v3=32": 3236, + "v3=33": 3237, + "v3=34": 3238, + "v3=35": 3239, + "v3=36": 3240, + "v3=37": 3241, + "v3=38": 3242, + "v3=39": 3243, + "v3=40": 3244, + "v3=41": 3245, + "v3=42": 3246, + "v3=43": 3247, + "v3=44": 3248, + "v3=45": 3249, + "v3=46": 3250, + "v3=47": 3251, + "v3=48": 3252, + "v3=49": 3253, + "v3=50": 3254, + "v3=51": 3255, + "v3=52": 3256, + "v3=53": 3257, + "v3=54": 3258, + "v3=55": 3259, + "v3=56": 3260, + "v3=57": 3261, + "v3=58": 3262, + "v3=59": 3263, + "v3=60": 3264, + "v3=61": 3265, + "v3=62": 3266, + "v3=63": 3267, + "v3=64": 3268, + "v3=65": 3269, + "v3=66": 3270, + "v3=67": 3271, + "v3=68": 3272, + "v3=69": 3273, + "v3=70": 3274, + "v3=71": 3275, + "v3=72": 3276, + "v3=73": 3277, + "v3=74": 3278, + "v3=75": 3279, + "v3=76": 3280, + "v3=77": 3281, + "v3=78": 3282, + "v3=79": 3283, + "v3=80": 3284, + "v3=81": 3285, + "v3=82": 3286, + "v3=83": 3287, + "v3=84": 3288, + "v3=85": 3289, + "v3=86": 3290, + "v3=87": 3291, + "v3=88": 3292, + "v3=89": 3293, + "v3=90": 3294, + "v3=91": 3295, + "v3=92": 3296, + "v3=93": 3297, + "v3=94": 3298, + "v3=95": 3299, + "v3=96": 3300, + "v3=97": 3301, + "v3=98": 3302, + "v3=99": 3303, + "v3=100": 3304, + "v3=101": 3305, + "v3=102": 3306, + "v3=103": 3307, + "v3=104": 3308, + "v3=105": 3309, + "v3=106": 3310, + "v3=107": 3311, + "v3=108": 3312, + "v3=109": 3313, + "v3=110": 3314, + "v3=111": 3315, + "v3=112": 3316, + "v3=113": 3317, + "v3=114": 3318, + "v3=115": 3319, + "v3=116": 3320, + "v3=117": 3321, + "v3=118": 3322, + "v3=119": 3323, + "v3=120": 3324, + "v3=121": 3325, + "v3=122": 3326, + "v3=123": 3327, + "v3=124": 3328, + "v3=125": 3329, + "v3=126": 3330, + "v3=127": 3331, + "v3=128": 3332, + "v3=129": 3333, + "v3=130": 3334, + "v3=131": 3335, + "v3=132": 3336, + "v3=133": 3337, + "v3=134": 3338, + "v3=135": 3339, + "v3=136": 3340, + "v3=137": 3341, + "v3=138": 3342, + "v3=139": 3343, + "v3=140": 3344, + "v3=141": 3345, + "v3=142": 3346, + "v3=143": 3347, + "v3=144": 3348, + "v3=145": 3349, + "v3=146": 3350, + "v3=147": 3351, + "v3=148": 3352, + "v3=149": 3353, + "v3=150": 3354, + "v3=151": 3355, + "v3=152": 3356, + "v3=153": 3357, + "v3=154": 3358, + "v3=155": 3359, + "v3=156": 3360, + "v3=157": 3361, + "v3=158": 3362, + "v3=159": 3363, + "v3=160": 3364, + "v3=161": 3365, + "v3=162": 3366, + "v3=163": 3367, + "v3=164": 3368, + "v3=165": 3369, + "v3=166": 3370, + "v3=167": 3371, + "v3=168": 3372, + "v3=169": 3373, + "v3=170": 3374, + "v3=171": 3375, + "v3=172": 3376, + "v3=173": 3377, + "v3=174": 3378, + "v3=175": 3379, + "v3=176": 3380, + "v3=177": 3381, + "v3=178": 3382, + "v3=179": 3383, + "v3=180": 3384, + "v3=181": 3385, + "v3=182": 3386, + "v3=183": 3387, + "v3=184": 3388, + "v3=185": 3389, + "v3=186": 3390, + "v3=187": 3391, + "v3=188": 3392, + "v3=189": 3393, + "v3=190": 3394, + "v3=191": 3395, + "v3=192": 3396, + "v3=193": 3397, + "v3=194": 3398, + "v3=195": 3399, + "v3=196": 3400, + "v3=197": 3401, + "v3=198": 3402, + "v3=199": 3403, + "v3=200": 3404, + "v3=201": 3405, + "v3=202": 3406, + "v3=203": 3407, + "v3=204": 3408, + "v3=205": 3409, + "v3=206": 3410, + "v3=207": 3411, + "v3=208": 3412, + "v3=209": 3413, + "v3=210": 3414, + "v3=211": 3415, + "v3=212": 3416, + "v3=213": 3417, + "v3=214": 3418, + "v3=215": 3419, + "v3=216": 3420, + "v3=217": 3421, + "v3=218": 3422, + "v3=219": 3423, + "v3=220": 3424, + "v3=221": 3425, + "v3=222": 3426, + "v3=223": 3427, + "v3=224": 3428, + "v3=225": 3429, + "v3=226": 3430, + "v3=227": 3431, + "v3=228": 3432, + "v3=229": 3433, + "v3=230": 3434, + "v3=231": 3435, + "v3=232": 3436, + "v3=233": 3437, + "v3=234": 3438, + "v3=235": 3439, + "v3=236": 3440, + "v3=237": 3441, + "v3=238": 3442, + "v3=239": 3443, + "v3=240": 3444, + "v3=241": 3445, + "v3=242": 3446, + "v3=243": 3447, + "v3=244": 3448, + "v3=245": 3449, + "v3=246": 3450, + "v3=247": 3451, + "v3=248": 3452, + "v3=249": 3453, + "v3=250": 3454, + "v3=251": 3455, + "v3=252": 3456, + "v3=253": 3457, + "v3=254": 3458, + "v3=255": 3459, + "v3=256": 3460, + "v3=257": 3461, + "v3=258": 3462, + "v3=259": 3463, + "v3=260": 3464, + "v3=261": 3465, + "v3=262": 3466, + "v3=263": 3467, + "v3=264": 3468, + "v3=265": 3469, + "v3=266": 3470, + "v3=267": 3471, + "v3=268": 3472, + "v3=269": 3473, + "v3=270": 3474, + "v3=271": 3475, + "v3=272": 3476, + "v3=273": 3477, + "v3=274": 3478, + "v3=275": 3479, + "v3=276": 3480, + "v3=277": 3481, + "v3=278": 3482, + "v3=279": 3483, + "v3=280": 3484, + "v3=281": 3485, + "v3=282": 3486, + "v3=283": 3487, + "v3=284": 3488, + "v3=285": 3489, + "v3=286": 3490, + "v3=287": 3491, + "v3=288": 3492, + "v3=289": 3493, + "v3=290": 3494, + "v3=291": 3495, + "v3=292": 3496, + "v3=293": 3497, + "v3=294": 3498, + "v3=295": 3499, + "v3=296": 3500, + "v3=297": 3501, + "v3=298": 3502, + "v3=299": 3503, + "v3=300": 3504, + "v3=301": 3505, + "v3=302": 3506, + "v3=303": 3507, + "v3=304": 3508, + "v3=305": 3509, + "v3=306": 3510, + "v3=307": 3511, + "v3=308": 3512, + "v3=309": 3513, + "v3=310": 3514, + "v3=311": 3515, + "v3=312": 3516, + "v3=313": 3517, + "v3=314": 3518, + "v3=315": 3519, + "v3=316": 3520, + "v3=317": 3521, + "v3=318": 3522, + "v3=319": 3523, + "v3=320": 3524, + "v3=321": 3525, + "v3=322": 3526, + "v3=323": 3527, + "v3=324": 3528, + "v3=325": 3529, + "v3=326": 3530, + "v3=327": 3531, + "v3=328": 3532, + "v3=329": 3533, + "v3=330": 3534, + "v3=331": 3535, + "v3=332": 3536, + "v3=333": 3537, + "v3=334": 3538, + "v3=335": 3539, + "v3=336": 3540, + "v3=337": 3541, + "v3=338": 3542, + "v3=339": 3543, + "v3=340": 3544, + "v3=341": 3545, + "v3=342": 3546, + "v3=343": 3547, + "v3=344": 3548, + "v3=345": 3549, + "v3=346": 3550, + "v3=347": 3551, + "v3=348": 3552, + "v3=349": 3553, + "v3=350": 3554, + "v3=351": 3555, + "v3=352": 3556, + "v3=353": 3557, + "v3=354": 3558, + "v3=355": 3559, + "v3=356": 3560, + "v3=357": 3561, + "v3=358": 3562, + "v3=359": 3563, + "v3=360": 3564, + "v3=361": 3565, + "v3=362": 3566, + "v3=363": 3567, + "v3=364": 3568, + "v3=365": 3569, + "v3=366": 3570, + "v3=367": 3571, + "v3=368": 3572, + "v3=369": 3573, + "v3=370": 3574, + "v3=371": 3575, + "v3=372": 3576, + "v3=373": 3577, + "v3=374": 3578, + "v3=375": 3579, + "v3=376": 3580, + "v3=377": 3581, + "v3=378": 3582, + "v3=379": 3583, + "v3=380": 3584, + "v3=381": 3585, + "v3=382": 3586, + "v3=383": 3587, + "v3=384": 3588, + "v3=385": 3589, + "v3=386": 3590, + "v3=387": 3591, + "v3=388": 3592, + "v3=389": 3593, + "v3=390": 3594, + "v3=391": 3595, + "v3=392": 3596, + "v3=393": 3597, + "v3=394": 3598, + "v3=395": 3599, + "v3=396": 3600, + "v3=397": 3601, + "v3=398": 3602, + "v3=399": 3603, + "v3=400": 3604, + "v3=401": 3605, + "v3=402": 3606, + "v3=403": 3607, + "v3=404": 3608, + "v3=405": 3609, + "v3=406": 3610, + "v3=407": 3611, + "v3=408": 3612, + "v3=409": 3613, + "v3=410": 3614, + "v3=411": 3615, + "v3=412": 3616, + "v3=413": 3617, + "v3=414": 3618, + "v3=415": 3619, + "v3=416": 3620, + "v3=417": 3621, + "v3=418": 3622, + "v3=419": 3623, + "v3=420": 3624, + "v3=421": 3625, + "v3=422": 3626, + "v3=423": 3627, + "v3=424": 3628, + "v3=425": 3629, + "v3=426": 3630, + "v3=427": 3631, + "v3=428": 3632, + "v3=429": 3633, + "v3=430": 3634, + "v3=431": 3635, + "v3=432": 3636, + "v3=433": 3637, + "v3=434": 3638, + "v3=435": 3639, + "v3=436": 3640, + "v3=437": 3641, + "v3=438": 3642, + "v3=439": 3643, + "v3=440": 3644, + "v3=441": 3645, + "v3=442": 3646, + "v3=443": 3647, + "v3=444": 3648, + "v3=445": 3649, + "v3=446": 3650, + "v3=447": 3651, + "v3=448": 3652, + "v3=449": 3653, + "v3=450": 3654, + "v3=451": 3655, + "v3=452": 3656, + "v3=453": 3657, + "v3=454": 3658, + "v3=455": 3659, + "v3=456": 3660, + "v3=457": 3661, + "v3=458": 3662, + "v3=459": 3663, + "v3=460": 3664, + "v3=461": 3665, + "v3=462": 3666, + "v3=463": 3667, + "v3=464": 3668, + "v3=465": 3669, + "v3=466": 3670, + "v3=467": 3671, + "v3=468": 3672, + "v3=469": 3673, + "v3=470": 3674, + "v3=471": 3675, + "v3=472": 3676, + "v3=473": 3677, + "v3=474": 3678, + "v3=475": 3679, + "v3=476": 3680, + "v3=477": 3681, + "v3=478": 3682, + "v3=479": 3683, + "v3=480": 3684, + "v3=481": 3685, + "v3=482": 3686, + "v3=483": 3687, + "v3=484": 3688, + "v3=485": 3689, + "v3=486": 3690, + "v3=487": 3691, + "v3=488": 3692, + "v3=489": 3693, + "v3=490": 3694, + "v3=491": 3695, + "v3=492": 3696, + "v3=493": 3697, + "v3=494": 3698, + "v3=495": 3699, + "v3=496": 3700, + "v3=497": 3701, + "v3=498": 3702, + "v3=499": 3703, + "v3=500": 3704, + "v3=501": 3705, + "v3=502": 3706, + "v3=503": 3707, + "v3=504": 3708, + "v3=505": 3709, + "v3=506": 3710, + "v3=507": 3711, + "v3=508": 3712, + "v3=509": 3713, + "v3=510": 3714, + "v3=511": 3715, + "v3=512": 3716, + "v3=513": 3717, + "v3=514": 3718, + "v3=515": 3719, + "v3=516": 3720, + "v3=517": 3721, + "v3=518": 3722, + "v3=519": 3723, + "v3=520": 3724, + "v3=521": 3725, + "v3=522": 3726, + "v3=523": 3727, + "v3=524": 3728, + "v3=525": 3729, + "v3=526": 3730, + "v3=527": 3731, + "v3=528": 3732, + "v3=529": 3733, + "v3=530": 3734, + "v3=531": 3735, + "v3=532": 3736, + "v3=533": 3737, + "v3=534": 3738, + "v3=535": 3739, + "v3=536": 3740, + "v3=537": 3741, + "v3=538": 3742, + "v3=539": 3743, + "v3=540": 3744, + "v3=541": 3745, + "v3=542": 3746, + "v3=543": 3747, + "v3=544": 3748, + "v3=545": 3749, + "v3=546": 3750, + "v3=547": 3751, + "v3=548": 3752, + "v3=549": 3753, + "v3=550": 3754, + "v3=551": 3755, + "v3=552": 3756, + "v3=553": 3757, + "v3=554": 3758, + "v3=555": 3759, + "v3=556": 3760, + "v3=557": 3761, + "v3=558": 3762, + "v3=559": 3763, + "v3=560": 3764, + "v3=561": 3765, + "v3=562": 3766, + "v3=563": 3767, + "v3=564": 3768, + "v3=565": 3769, + "v3=566": 3770, + "v3=567": 3771, + "v3=568": 3772, + "v3=569": 3773, + "v3=570": 3774, + "v3=571": 3775, + "v3=572": 3776, + "v3=573": 3777, + "v3=574": 3778, + "v3=575": 3779, + "v3=576": 3780, + "v3=577": 3781, + "v3=578": 3782, + "v3=579": 3783, + "v3=580": 3784, + "v3=581": 3785, + "v3=582": 3786, + "v3=583": 3787, + "v3=584": 3788, + "v3=585": 3789, + "v3=586": 3790, + "v3=587": 3791, + "v3=588": 3792, + "v3=589": 3793, + "v3=590": 3794, + "v3=591": 3795, + "v3=592": 3796, + "v3=593": 3797, + "v3=594": 3798, + "v3=595": 3799, + "v3=596": 3800, + "v3=597": 3801, + "v3=598": 3802, + "v3=599": 3803, + "v3=600": 3804, + "v3=601": 3805, + "v3=602": 3806, + "v3=603": 3807, + "v3=604": 3808, + "v3=605": 3809, + "v3=606": 3810, + "v3=607": 3811, + "v3=608": 3812, + "v3=609": 3813, + "v3=610": 3814, + "v3=611": 3815, + "v3=612": 3816, + "v3=613": 3817, + "v3=614": 3818, + "v3=615": 3819, + "v3=616": 3820, + "v3=617": 3821, + "v3=618": 3822, + "v3=619": 3823, + "v3=620": 3824, + "v3=621": 3825, + "v3=622": 3826, + "v3=623": 3827, + "v3=624": 3828, + "v3=625": 3829, + "v3=626": 3830, + "v3=627": 3831, + "v3=628": 3832, + "v3=629": 3833, + "v3=630": 3834, + "v3=631": 3835, + "v3=632": 3836, + "v3=633": 3837, + "v3=634": 3838, + "v3=635": 3839, + "v3=636": 3840, + "v3=637": 3841, + "v3=638": 3842, + "v3=639": 3843, + "v3=640": 3844, + "v3=641": 3845, + "v3=642": 3846, + "v3=643": 3847, + "v3=644": 3848, + "v3=645": 3849, + "v3=646": 3850, + "v3=647": 3851, + "v3=648": 3852, + "v3=649": 3853, + "v3=650": 3854, + "v3=651": 3855, + "v3=652": 3856, + "v3=653": 3857, + "v3=654": 3858, + "v3=655": 3859, + "v3=656": 3860, + "v3=657": 3861, + "v3=658": 3862, + "v3=659": 3863, + "v3=660": 3864, + "v3=661": 3865, + "v3=662": 3866, + "v3=663": 3867, + "v3=664": 3868, + "v3=665": 3869, + "v3=666": 3870, + "v3=667": 3871, + "v3=668": 3872, + "v3=669": 3873, + "v3=670": 3874, + "v3=671": 3875, + "v3=672": 3876, + "v3=673": 3877, + "v3=674": 3878, + "v3=675": 3879, + "v3=676": 3880, + "v3=677": 3881, + "v3=678": 3882, + "v3=679": 3883, + "v3=680": 3884, + "v3=681": 3885, + "v3=682": 3886, + "v3=683": 3887, + "v3=684": 3888, + "v3=685": 3889, + "v3=686": 3890, + "v3=687": 3891, + "v3=688": 3892, + "v3=689": 3893, + "v3=690": 3894, + "v3=691": 3895, + "v3=692": 3896, + "v3=693": 3897, + "v3=694": 3898, + "v3=695": 3899, + "v3=696": 3900, + "v3=697": 3901, + "v3=698": 3902, + "v3=699": 3903, + "v3=700": 3904, + "v3=701": 3905, + "v3=702": 3906, + "v3=703": 3907, + "v3=704": 3908, + "v3=705": 3909, + "v3=706": 3910, + "v3=707": 3911, + "v3=708": 3912, + "v3=709": 3913, + "v3=710": 3914, + "v3=711": 3915, + "v3=712": 3916, + "v3=713": 3917, + "v3=714": 3918, + "v3=715": 3919, + "v3=716": 3920, + "v3=717": 3921, + "v3=718": 3922, + "v3=719": 3923, + "v3=720": 3924, + "v3=721": 3925, + "v3=722": 3926, + "v3=723": 3927, + "v3=724": 3928, + "v3=725": 3929, + "v3=726": 3930, + "v3=727": 3931, + "v3=728": 3932, + "v3=729": 3933, + "v3=730": 3934, + "v3=731": 3935, + "v3=732": 3936, + "v3=733": 3937, + "v3=734": 3938, + "v3=735": 3939, + "v3=736": 3940, + "v3=737": 3941, + "v3=738": 3942, + "v3=739": 3943, + "v3=740": 3944, + "v3=741": 3945, + "v3=742": 3946, + "v3=743": 3947, + "v3=744": 3948, + "v3=745": 3949, + "v3=746": 3950, + "v3=747": 3951, + "v3=748": 3952, + "v3=749": 3953, + "v3=750": 3954, + "v3=751": 3955, + "v3=752": 3956, + "v3=753": 3957, + "v3=754": 3958, + "v3=755": 3959, + "v3=756": 3960, + "v3=757": 3961, + "v3=758": 3962, + "v3=759": 3963, + "v3=760": 3964, + "v3=761": 3965, + "v3=762": 3966, + "v3=763": 3967, + "v3=764": 3968, + "v3=765": 3969, + "v3=766": 3970, + "v3=767": 3971, + "v3=768": 3972, + "v3=769": 3973, + "v3=770": 3974, + "v3=771": 3975, + "v3=772": 3976, + "v3=773": 3977, + "v3=774": 3978, + "v3=775": 3979, + "v3=776": 3980, + "v3=777": 3981, + "v3=778": 3982, + "v3=779": 3983, + "v3=780": 3984, + "v3=781": 3985, + "v3=782": 3986, + "v3=783": 3987, + "v3=784": 3988, + "v3=785": 3989, + "v3=786": 3990, + "v3=787": 3991, + "v3=788": 3992, + "v3=789": 3993, + "v3=790": 3994, + "v3=791": 3995, + "v3=792": 3996, + "v3=793": 3997, + "v3=794": 3998, + "v3=795": 3999, + "v3=796": 4000, + "v3=797": 4001, + "v3=798": 4002, + "v3=799": 4003, + "v3=800": 4004, + "v3=801": 4005, + "v3=802": 4006, + "v3=803": 4007, + "v3=804": 4008, + "v3=805": 4009, + "v3=806": 4010, + "v3=807": 4011, + "v3=808": 4012, + "v3=809": 4013, + "v3=810": 4014, + "v3=811": 4015, + "v3=812": 4016, + "v3=813": 4017, + "v3=814": 4018, + "v3=815": 4019, + "v3=816": 4020, + "v3=817": 4021, + "v3=818": 4022, + "v3=819": 4023, + "v3=820": 4024, + "v3=821": 4025, + "v3=822": 4026, + "v3=823": 4027, + "v3=824": 4028, + "v3=825": 4029, + "v3=826": 4030, + "v3=827": 4031, + "v3=828": 4032, + "v3=829": 4033, + "v3=830": 4034, + "v3=831": 4035, + "v3=832": 4036, + "v3=833": 4037, + "v3=834": 4038, + "v3=835": 4039, + "v3=836": 4040, + "v3=837": 4041, + "v3=838": 4042, + "v3=839": 4043, + "v3=840": 4044, + "v3=841": 4045, + "v3=842": 4046, + "v3=843": 4047, + "v3=844": 4048, + "v3=845": 4049, + "v3=846": 4050, + "v3=847": 4051, + "v3=848": 4052, + "v3=849": 4053, + "v3=850": 4054, + "v3=851": 4055, + "v3=852": 4056, + "v3=853": 4057, + "v3=854": 4058, + "v3=855": 4059, + "v3=856": 4060, + "v3=857": 4061, + "v3=858": 4062, + "v3=859": 4063, + "v3=860": 4064, + "v3=861": 4065, + "v3=862": 4066, + "v3=863": 4067, + "v3=864": 4068, + "v3=865": 4069, + "v3=866": 4070, + "v3=867": 4071, + "v3=868": 4072, + "v3=869": 4073, + "v3=870": 4074, + "v3=871": 4075, + "v3=872": 4076, + "v3=873": 4077, + "v3=874": 4078, + "v3=875": 4079, + "v3=876": 4080, + "v3=877": 4081, + "v3=878": 4082, + "v3=879": 4083, + "v3=880": 4084, + "v3=881": 4085, + "v3=882": 4086, + "v3=883": 4087, + "v3=884": 4088, + "v3=885": 4089, + "v3=886": 4090, + "v3=887": 4091, + "v3=888": 4092, + "v3=889": 4093, + "v3=890": 4094, + "v3=891": 4095, + "v3=892": 4096, + "v3=893": 4097, + "v3=894": 4098, + "v3=895": 4099, + "v3=896": 4100, + "v3=897": 4101, + "v3=898": 4102, + "v3=899": 4103, + "v3=900": 4104, + "v3=901": 4105, + "v3=902": 4106, + "v3=903": 4107, + "v3=904": 4108, + "v3=905": 4109, + "v3=906": 4110, + "v3=907": 4111, + "v3=908": 4112, + "v3=909": 4113, + "v3=910": 4114, + "v3=911": 4115, + "v3=912": 4116, + "v3=913": 4117, + "v3=914": 4118, + "v3=915": 4119, + "v3=916": 4120, + "v3=917": 4121, + "v3=918": 4122, + "v3=919": 4123, + "v3=920": 4124, + "v3=921": 4125, + "v3=922": 4126, + "v3=923": 4127, + "v3=924": 4128, + "v3=925": 4129, + "v3=926": 4130, + "v3=927": 4131, + "v3=928": 4132, + "v3=929": 4133, + "v3=930": 4134, + "v3=931": 4135, + "v3=932": 4136, + "v3=933": 4137, + "v3=934": 4138, + "v3=935": 4139, + "v3=936": 4140, + "v3=937": 4141, + "v3=938": 4142, + "v3=939": 4143, + "v3=940": 4144, + "v3=941": 4145, + "v3=942": 4146, + "v3=943": 4147, + "v3=944": 4148, + "v3=945": 4149, + "v3=946": 4150, + "v3=947": 4151, + "v3=948": 4152, + "v3=949": 4153, + "v3=950": 4154, + "v3=951": 4155, + "v3=952": 4156, + "v3=953": 4157, + "v3=954": 4158, + "v3=955": 4159, + "v3=956": 4160, + "v3=957": 4161, + "v3=958": 4162, + "v3=959": 4163, + "v3=960": 4164, + "v3=961": 4165, + "v3=962": 4166, + "v3=963": 4167, + "v3=964": 4168, + "v3=965": 4169, + "v3=966": 4170, + "v3=967": 4171, + "v3=968": 4172, + "v3=969": 4173, + "v3=970": 4174, + "v3=971": 4175, + "v3=972": 4176, + "v3=973": 4177, + "v3=974": 4178, + "v3=975": 4179, + "v3=976": 4180, + "v3=977": 4181, + "v3=978": 4182, + "v3=979": 4183, + "v3=980": 4184, + "v3=981": 4185, + "v3=982": 4186, + "v3=983": 4187, + "v3=984": 4188, + "v3=985": 4189, + "v3=986": 4190, + "v3=987": 4191, + "v3=988": 4192, + "v3=989": 4193, + "v3=990": 4194, + "v3=991": 4195, + "v3=992": 4196, + "v3=993": 4197, + "v3=994": 4198, + "v3=995": 4199, + "v3=996": 4200, + "v3=997": 4201, + "v3=998": 4202, + "v3=999": 4203, + "airplane": 4204, + "apple": 4205, + "backpack": 4206, + "banana": 4207, + "baseball bat": 4208, + "baseball glove": 4209, + "bear": 4210, + "bed": 4211, + "bench": 4212, + "bicycle": 4213, + "bird": 4214, + "boat": 4215, + "book": 4216, + "bottle": 4217, + "bowl": 4218, + "broccoli": 4219, + "bus": 4220, + "cake": 4221, + "car": 4222, + "carrot": 4223, + "cat": 4224, + "cell phone": 4225, + "chair": 4226, + "clock": 4227, + "couch": 4228, + "cow": 4229, + "cup": 4230, + "dining table": 4231, + "dog": 4232, + "donut": 4233, + "elephant": 4234, + "fire hydrant": 4235, + "fork": 4236, + "frisbee": 4237, + "giraffe": 4238, + "hair drier": 4239, + "handbag": 4240, + "horse": 4241, + "hot dog": 4242, + "keyboard": 4243, + "kite": 4244, + "knife": 4245, + "laptop": 4246, + "microwave": 4247, + "motorcycle": 4248, + "mouse": 4249, + "orange": 4250, + "oven": 4251, + "parking meter": 4252, + "person": 4253, + "pizza": 4254, + "potted plant": 4255, + "refrigerator": 4256, + "remote": 4257, + "sandwich": 4258, + "scissors": 4259, + "sheep": 4260, + "sink": 4261, + "skateboard": 4262, + "skis": 4263, + "snowboard": 4264, + "spoon": 4265, + "sports ball": 4266, + "stop sign": 4267, + "suitcase": 4268, + "surfboard": 4269, + "teddy bear": 4270, + "tennis racket": 4271, + "tie": 4272, + "toaster": 4273, + "toilet": 4274, + "toothbrush": 4275, + "traffic light": 4276, + "train": 4277, + "truck": 4278, + "tv": 4279, + "umbrella": 4280, + "vase": 4281, + "wine glass": 4282, + "zebra": 4283, + "!": 4284, + "#": 4285, + "$": 4286, + "%": 4287, + "&": 4288, + "'": 4289, + "(": 4290, + ")": 4291, + "*": 4292, + "+": 4293, + ",": 4294, + "-": 4295, + ".": 4296, + "/": 4297, + "0": 4298, + "1": 4299, + "2": 4300, + "3": 4301, + "4": 4302, + "5": 4303, + "6": 4304, + "7": 4305, + "8": 4306, + "9": 4307, + ":": 4308, + ";": 4309, + "<": 4310, + "=": 4311, + ">": 4312, + "?": 4313, + "@": 4314, + "[": 4315, + "\\": 4316, + "]": 4317, + "^": 4318, + "_": 4319, + "`": 4320, + "a": 4321, + "b": 4322, + "c": 4323, + "d": 4324, + "e": 4325, + "f": 4326, + "g": 4327, + "h": 4328, + "i": 4329, + "j": 4330, + "k": 4331, + "l": 4332, + "m": 4333, + "n": 4334, + "o": 4335, + "p": 4336, + "q": 4337, + "r": 4338, + "s": 4339, + "t": 4340, + "u": 4341, + "v": 4342, + "w": 4343, + "x": 4344, + "y": 4345, + "z": 4346, + "{": 4347, + "|": 4348, + "}": 4349, + "~": 4350, + "¡": 4351, + "¢": 4352, + "£": 4353, + "¤": 4354, + "¥": 4355, + "§": 4356, + "©": 4357, + "«": 4358, + "¬": 4359, + "®": 4360, + "°": 4361, + "±": 4362, + "´": 4363, + "¶": 4364, + "º": 4365, + "¼": 4366, + "½": 4367, + "¿": 4368, + "×": 4369, + "ß": 4370, + "æ": 4371, + "ð": 4372, + "÷": 4373, + "ø": 4374, + "þ": 4375, + "đ": 4376, + "ħ": 4377, + "ı": 4378, + "ł": 4379, + "œ": 4380, + "ƒ": 4381, + "ǁ": 4382, + "ɑ": 4383, + "ə": 4384, + "α": 4385, + "β": 4386, + "γ": 4387, + "δ": 4388, + "ε": 4389, + "ζ": 4390, + "η": 4391, + "θ": 4392, + "ι": 4393, + "κ": 4394, + "λ": 4395, + "μ": 4396, + "ν": 4397, + "ξ": 4398, + "ο": 4399, + "π": 4400, + "ρ": 4401, + "σ": 4402, + "τ": 4403, + "υ": 4404, + "φ": 4405, + "χ": 4406, + "ψ": 4407, + "ω": 4408, + "а": 4409, + "б": 4410, + "в": 4411, + "г": 4412, + "д": 4413, + "е": 4414, + "ж": 4415, + "з": 4416, + "и": 4417, + "к": 4418, + "л": 4419, + "м": 4420, + "н": 4421, + "о": 4422, + "п": 4423, + "р": 4424, + "с": 4425, + "т": 4426, + "у": 4427, + "ф": 4428, + "х": 4429, + "ц": 4430, + "ч": 4431, + "ш": 4432, + "ы": 4433, + "ь": 4434, + "э": 4435, + "ю": 4436, + "я": 4437, + "є": 4438, + "א": 4439, + "ב": 4440, + "ג": 4441, + "ד": 4442, + "ו": 4443, + "י": 4444, + "ל": 4445, + "ם": 4446, + "ץ": 4447, + "ר": 4448, + "ש": 4449, + "ת": 4450, + "ا": 4451, + "ت": 4452, + "ج": 4453, + "ح": 4454, + "د": 4455, + "ر": 4456, + "س": 4457, + "ش": 4458, + "ع": 4459, + "ق": 4460, + "ل": 4461, + "م": 4462, + "ن": 4463, + "ه": 4464, + "و": 4465, + "ي": 4466, + "म": 4467, + "व": 4468, + "ก": 4469, + "ท": 4470, + "ย": 4471, + "ะ": 4472, + "า": 4473, + "฿": 4474, + "เ": 4475, + "ᄀ": 4476, + "ᄂ": 4477, + "ᄃ": 4478, + "ᄆ": 4479, + "ᄉ": 4480, + "ᄋ": 4481, + "ᄒ": 4482, + "ᅡ": 4483, + "ᅢ": 4484, + "ᅥ": 4485, + "ᅭ": 4486, + "ᅮ": 4487, + "ᅵ": 4488, + "ᆨ": 4489, + "ᆫ": 4490, + "ᆯ": 4491, + "–": 4492, + "—": 4493, + "†": 4494, + "‡": 4495, + "•": 4496, + "…": 4497, + "‧": 4498, + "‰": 4499, + "″": 4500, + "※": 4501, + "⁄": 4502, + "₤": 4503, + "₩": 4504, + "₪": 4505, + "€": 4506, + "₱": 4507, + "₵": 4508, + "℘": 4509, + "™": 4510, + "←": 4511, + "↑": 4512, + "→": 4513, + "↓": 4514, + "↔": 4515, + "⇒": 4516, + "∀": 4517, + "∂": 4518, + "∅": 4519, + "∈": 4520, + "∘": 4521, + "∙": 4522, + "√": 4523, + "∞": 4524, + "∠": 4525, + "∧": 4526, + "∩": 4527, + "∪": 4528, + "∫": 4529, + "∴": 4530, + "≅": 4531, + "≈": 4532, + "≡": 4533, + "≤": 4534, + "≥": 4535, + "⊂": 4536, + "⊗": 4537, + "⊙": 4538, + "⊥": 4539, + "⋅": 4540, + "⎯": 4541, + "─": 4542, + "│": 4543, + "└": 4544, + "█": 4545, + "▲": 4546, + "►": 4547, + "◄": 4548, + "◊": 4549, + "○": 4550, + "◦": 4551, + "☆": 4552, + "☉": 4553, + "☠": 4554, + "☮": 4555, + "♀": 4556, + "♂": 4557, + "♠": 4558, + "♣": 4559, + "♥": 4560, + "♪": 4561, + "♭": 4562, + "♯": 4563, + "。": 4564, + "々": 4565, + "》": 4566, + "「": 4567, + "」": 4568, + "』": 4569, + "〖": 4570, + "〗": 4571, + "い": 4572, + "う": 4573, + "お": 4574, + "さ": 4575, + "て": 4576, + "と": 4577, + "に": 4578, + "の": 4579, + "ま": 4580, + "み": 4581, + "り": 4582, + "を": 4583, + "ん": 4584, + "ア": 4585, + "イ": 4586, + "ウ": 4587, + "オ": 4588, + "カ": 4589, + "コ": 4590, + "シ": 4591, + "ス": 4592, + "タ": 4593, + "ッ": 4594, + "テ": 4595, + "ト": 4596, + "フ": 4597, + "ヘ": 4598, + "ホ": 4599, + "ム": 4600, + "ラ": 4601, + "リ": 4602, + "ル": 4603, + "レ": 4604, + "ロ": 4605, + "ン": 4606, + "・": 4607, + "ー": 4608, + "一": 4609, + "上": 4610, + "下": 4611, + "个": 4612, + "中": 4613, + "之": 4614, + "书": 4615, + "亀": 4616, + "争": 4617, + "二": 4618, + "京": 4619, + "人": 4620, + "会": 4621, + "体": 4622, + "八": 4623, + "公": 4624, + "关": 4625, + "出": 4626, + "北": 4627, + "匡": 4628, + "医": 4629, + "博": 4630, + "卦": 4631, + "印": 4632, + "厂": 4633, + "司": 4634, + "吉": 4635, + "味": 4636, + "国": 4637, + "地": 4638, + "場": 4639, + "夜": 4640, + "大": 4641, + "天": 4642, + "奥": 4643, + "好": 4644, + "子": 4645, + "字": 4646, + "小": 4647, + "山": 4648, + "巴": 4649, + "年": 4650, + "幸": 4651, + "心": 4652, + "情": 4653, + "愛": 4654, + "我": 4655, + "才": 4656, + "支": 4657, + "文": 4658, + "新": 4659, + "方": 4660, + "日": 4661, + "易": 4662, + "春": 4663, + "曲": 4664, + "本": 4665, + "東": 4666, + "松": 4667, + "歌": 4668, + "毅": 4669, + "汉": 4670, + "洋": 4671, + "海": 4672, + "港": 4673, + "游": 4674, + "滝": 4675, + "照": 4676, + "爱": 4677, + "父": 4678, + "版": 4679, + "犬": 4680, + "猫": 4681, + "王": 4682, + "琵": 4683, + "琶": 4684, + "田": 4685, + "発": 4686, + "的": 4687, + "真": 4688, + "矢": 4689, + "石": 4690, + "禅": 4691, + "福": 4692, + "秋": 4693, + "程": 4694, + "竜": 4695, + "罗": 4696, + "羊": 4697, + "美": 4698, + "色": 4699, + "花": 4700, + "蓝": 4701, + "虎": 4702, + "行": 4703, + "说": 4704, + "車": 4705, + "车": 4706, + "道": 4707, + "雨": 4708, + "非": 4709, + "音": 4710, + "香": 4711, + "魂": 4712, + "鱼": 4713, + "麒": 4714, + "麟": 4715, + "": 4716, + "##a": 4717, + "##l": 4718, + "##p": 4719, + "##h": 4720, + "##i": 4721, + "##d": 4722, + "##e": 4723, + "##0": 4724, + "##3": 4725, + "##1": 4726, + "##2": 4727, + "##7": 4728, + "##5": 4729, + "##o": 4730, + "##t": 4731, + "##r": 4732, + "##x": 4733, + "##n": 4734, + "##s": 4735, + "##6": 4736, + "##u": 4737, + "##g": 4738, + "##c": 4739, + "##f": 4740, + "##m": 4741, + "##z": 4742, + "##y": 4743, + "##w": 4744, + "##v": 4745, + "##b": 4746, + "##k": 4747, + "##q": 4748, + "##4": 4749, + "##8": 4750, + "##9": 4751, + "##j": 4752, + "##®": 4753, + "##°": 4754, + "##♥": 4755, + "##×": 4756, + "##©": 4757, + "##ø": 4758, + "##♪": 4759, + "##ß": 4760, + "##ł": 4761, + "##♭": 4762, + "##™": 4763, + "##☆": 4764, + "##о": 4765, + "##н": 4766, + "##в": 4767, + "##ю": 4768, + "##б": 4769, + "##ı": 4770, + "##£": 4771, + "##│": 4772, + "##♯": 4773, + "##æ": 4774, + "##ン": 4775, + "##€": 4776, + "##π": 4777, + "##ο": 4778, + "##¢": 4779, + "##ロ": 4780, + "##オ": 4781, + "##α": 4782, + "##→": 4783, + "##": 4784, + "##►": 4785, + "##√": 4786, + "##≥": 4787, + "##○": 4788, + "##ð": 4789, + "##¤": 4790, + "##⊙": 4791, + "##و": 4792, + "##ر": 4793, + "##μ": 4794, + "##œ": 4795, + "##¬": 4796, + "##ר": 4797, + "##ץ": 4798, + "##ラ": 4799, + "##ス": 4800, + "##đ": 4801, + "##⁄": 4802, + "##─": 4803, + "##ω": 4804, + "##☠": 4805, + "##¥": 4806, + "##м": 4807, + "##↓": 4808, + "##ι": 4809, + "##а": 4810, + "##л": 4811, + "##◄": 4812, + "##ф": 4813, + "##я": 4814, + "##г": 4815, + "##↑": 4816, + "##±": 4817, + "##к": 4818, + "##º": 4819, + "##т": 4820, + "##ش": 4821, + "##ق": 4822, + "##ε": 4823, + "##λ": 4824, + "##υ": 4825, + "##δ": 4826, + "##≈": 4827, + "##イ": 4828, + "##テ": 4829, + "##ー": 4830, + "##ッ": 4831, + "##リ": 4832, + "##シ": 4833, + "##п": 4834, + "##∩": 4835, + "##ь": 4836, + "##р": 4837, + "##е": 4838, + "##д": 4839, + "##☉": 4840, + "##█": 4841, + "##ᅥ": 4842, + "##ᄋ": 4843, + "##ᅮ": 4844, + "##ᆯ": 4845, + "##и": 4846, + "##с": 4847, + "##▲": 4848, + "##◦": 4849, + "##ル": 4850, + "##♣": 4851, + "##∙": 4852, + "##タ": 4853, + "##ム": 4854, + "##∞": 4855, + "##₵": 4856, + "##ᅢ": 4857, + "##ᄒ": 4858, + "##ᅡ": 4859, + "##ᆫ": 4860, + "##ᄆ": 4861, + "##ᅵ": 4862, + "##ᄀ": 4863, + "##ᆨ": 4864, + "##э": 4865, + "##ᅭ": 4866, + "##の": 4867, + "##≤": 4868, + "##ש": 4869, + "##א": 4870, + "##ל": 4871, + "##ل": 4872, + "##ه": 4873, + "##♠": 4874, + "##ə": 4875, + "##ي": 4876, + "##ح": 4877, + "##م": 4878, + "##د": 4879, + "##ы": 4880, + "##ħ": 4881, + "##カ": 4882, + "##´": 4883, + "##ウ": 4884, + "##ت": 4885, + "##☮": 4886, + "##∈": 4887, + "##ア": 4888, + "##コ": 4889, + "##θ": 4890, + "##η": 4891, + "##ν": 4892, + "##ו": 4893, + "##ם": 4894, + "##у": 4895, + "##∫": 4896, + "##♂": 4897, + "##ζ": 4898, + "##ж": 4899, + "##х": 4900, + "##♀": 4901, + "##ก": 4902, + "##า": 4903, + "##ะ": 4904, + "##∘": 4905, + "##∴": 4906, + "##ת": 4907, + "##þ": 4908, + "##er": 4909, + "th": 4910, + "##in": 4911, + "the": 4912, + "##on": 4913, + "##ing": 4914, + "##it": 4915, + "##es": 4916, + "##at": 4917, + "##nd": 4918, + "##or": 4919, + "##ar": 4920, + "in": 4921, + "##al": 4922, + "##an": 4923, + "##en": 4924, + "of": 4925, + "on": 4926, + "##ed": 4927, + "##ou": 4928, + "##ers": 4929, + "##om": 4930, + "##re": 4931, + "##le": 4932, + "and": 4933, + "##as": 4934, + "##ck": 4935, + "##ith": 4936, + "with": 4937, + "pers": 4938, + "st": 4939, + "##is": 4940, + "##ic": 4941, + "##ot": 4942, + "##ow": 4943, + "##ion": 4944, + "##ir": 4945, + "an": 4946, + "##il": 4947, + "##et": 4948, + "##ol": 4949, + "##el": 4950, + "##ra": 4951, + "##st": 4952, + "to": 4953, + "##ch": 4954, + "##ig": 4955, + "##oo": 4956, + "##la": 4957, + "##ur": 4958, + "##id": 4959, + "##og": 4960, + "##ct": 4961, + "##ut": 4962, + "##ea": 4963, + "sh": 4964, + "##all": 4965, + "is": 4966, + "##ge": 4967, + "##ent": 4968, + "##ce": 4969, + "for": 4970, + "wh": 4971, + "ch": 4972, + "##ation": 4973, + "at": 4974, + "##am": 4975, + "##ome": 4976, + "##lu": 4977, + "fr": 4978, + "ba": 4979, + "##ro": 4980, + "##rou": 4981, + "ro": 4982, + "##and": 4983, + "##ay": 4984, + "##ter": 4985, + "##im": 4986, + "##se": 4987, + "it": 4988, + "##iv": 4989, + "##ite": 4990, + "##ap": 4991, + "some": 4992, + "ar": 4993, + "ph": 4994, + "##ad": 4995, + "##th": 4996, + "##round": 4997, + "that": 4998, + "##ding": 4999, + "##um": 5000, + "##un": 5001, + "##ting": 5002, + "##ke": 5003, + "##str": 5004, + "##ul": 5005, + "##op": 5006, + "be": 5007, + "##ess": 5008, + "##ab": 5009, + "##ass": 5010, + "##ood": 5011, + "##ain": 5012, + "ne": 5013, + "##ock": 5014, + "##igh": 5015, + "##est": 5016, + "fl": 5017, + "re": 5018, + "phot": 5019, + "white": 5020, + "sp": 5021, + "##ie": 5022, + "##ld": 5023, + "##us": 5024, + "##ree": 5025, + "back": 5026, + "##ard": 5027, + "br": 5028, + "##if": 5029, + "##ver": 5030, + "il": 5031, + "##ide": 5032, + "##ook": 5033, + "illu": 5034, + "##ip": 5035, + "tw": 5036, + "##age": 5037, + "illustr": 5038, + "illustration": 5039, + "bo": 5040, + "by": 5041, + "##os": 5042, + "##ater": 5043, + "you": 5044, + "or": 5045, + "##em": 5046, + "two": 5047, + "##ant": 5048, + "##ign": 5049, + "man": 5050, + "cl": 5051, + "des": 5052, + "##ph": 5053, + "##ist": 5054, + "##ear": 5055, + "##ight": 5056, + "bla": 5057, + "co": 5058, + "##ect": 5059, + "art": 5060, + "from": 5061, + "black": 5062, + "par": 5063, + "con": 5064, + "##ure": 5065, + "##ine": 5066, + "##own": 5067, + "out": 5068, + "al": 5069, + "##ve": 5070, + "col": 5071, + "stock": 5072, + "sm": 5073, + "mo": 5074, + "water": 5075, + "bu": 5076, + "##ag": 5077, + "##ground": 5078, + "pr": 5079, + "##ate": 5080, + "se": 5081, + "pl": 5082, + "wor": 5083, + "20": 5084, + "gre": 5085, + "bir": 5086, + "background": 5087, + "de": 5088, + "##able": 5089, + "sit": 5090, + "gr": 5091, + "##lay": 5092, + "##ish": 5093, + "design": 5094, + "##ty": 5095, + "com": 5096, + "dr": 5097, + "##one": 5098, + "##ell": 5099, + "##ill": 5100, + "##ies": 5101, + "##ake": 5102, + "bea": 5103, + "sitting": 5104, + "are": 5105, + "##air": 5106, + "##ort": 5107, + "im": 5108, + "this": 5109, + "wom": 5110, + "##ast": 5111, + "old": 5112, + "##ly": 5113, + "tr": 5114, + "##oor": 5115, + "##up": 5116, + "tra": 5117, + "##ak": 5118, + "sc": 5119, + "##qu": 5120, + "pe": 5121, + "top": 5122, + "##ath": 5123, + "wall": 5124, + "small": 5125, + "as": 5126, + "room": 5127, + "##ouse": 5128, + "##ind": 5129, + "##ity": 5130, + "##int": 5131, + "##ick": 5132, + "lo": 5133, + "hol": 5134, + "red": 5135, + "##our": 5136, + "##les": 5137, + "po": 5138, + "##raph": 5139, + "##ian": 5140, + "look": 5141, + "anim": 5142, + "wood": 5143, + "##ace": 5144, + "en": 5145, + "##ive": 5146, + "sn": 5147, + "##ash": 5148, + "201": 5149, + "grass": 5150, + "photo": 5151, + "##ead": 5152, + "sk": 5153, + "##ont": 5154, + "table": 5155, + "hand": 5156, + "buil": 5157, + "blu": 5158, + "##oun": 5159, + "up": 5160, + "ex": 5161, + "le": 5162, + "##ail": 5163, + "##room": 5164, + "##king": 5165, + "new": 5166, + "##ere": 5167, + "play": 5168, + "##irt": 5169, + "##ff": 5170, + "##ple": 5171, + "blue": 5172, + "##ong": 5173, + "pro": 5174, + "##oth": 5175, + "its": 5176, + "##to": 5177, + "stand": 5178, + "##ated": 5179, + "##tle": 5180, + "peop": 5181, + "people": 5182, + "##ry": 5183, + "green": 5184, + "woman": 5185, + "animal": 5186, + "can": 5187, + "##ice": 5188, + "##ia": 5189, + "##ograph": 5190, + "fi": 5191, + "one": 5192, + "##ment": 5193, + "ap": 5194, + "##irl": 5195, + "##ee": 5196, + "tree": 5197, + "building": 5198, + "flow": 5199, + "dra": 5200, + "##xt": 5201, + "##side": 5202, + "##ore": 5203, + "##ector": 5204, + "gl": 5205, + "color": 5206, + "##iz": 5207, + "house": 5208, + "##ild": 5209, + "##ac": 5210, + "free": 5211, + "##ang": 5212, + "##tain": 5213, + "dis": 5214, + "photograph": 5215, + "qu": 5216, + "##ib": 5217, + "vector": 5218, + "girl": 5219, + "wed": 5220, + "##gh": 5221, + "mar": 5222, + "##ust": 5223, + "front": 5224, + "un": 5225, + "##ict": 5226, + "te": 5227, + "sw": 5228, + "fe": 5229, + "##ach": 5230, + "me": 5231, + "all": 5232, + "##pt": 5233, + "ad": 5234, + "##ack": 5235, + "his": 5236, + "standing": 5237, + "wedding": 5238, + "ground": 5239, + "##ous": 5240, + "dress": 5241, + "##ial": 5242, + "##ond": 5243, + "##mer": 5244, + "her": 5245, + "ab": 5246, + "shirt": 5247, + "pain": 5248, + "##ber": 5249, + "show": 5250, + "##ink": 5251, + "off": 5252, + "##uit": 5253, + "sty": 5254, + "##iew": 5255, + "##ery": 5256, + "##ame": 5257, + "lar": 5258, + "home": 5259, + "##by": 5260, + "##yal": 5261, + "rock": 5262, + "##de": 5263, + "##ction": 5264, + "##ings": 5265, + "royal": 5266, + "##ile": 5267, + "dec": 5268, + "##tern": 5269, + "wal": 5270, + "##aper": 5271, + "##ub": 5272, + "holding": 5273, + "cre": 5274, + "op": 5275, + "##ts": 5276, + "so": 5277, + "##ey": 5278, + "##ens": 5279, + "us": 5280, + "##ary": 5281, + "park": 5282, + "your": 5283, + "##ds": 5284, + "has": 5285, + "looking": 5286, + "##ale": 5287, + "##av": 5288, + "draw": 5289, + "food": 5290, + "cam": 5291, + "we": 5292, + "stre": 5293, + "image": 5294, + "per": 5295, + "##ans": 5296, + "large": 5297, + "my": 5298, + "bl": 5299, + "no": 5300, + "view": 5301, + "lea": 5302, + "##land": 5303, + "##uring": 5304, + "gra": 5305, + "air": 5306, + "##ud": 5307, + "##ther": 5308, + "##itch": 5309, + "##ade": 5310, + "##att": 5311, + "hot": 5312, + "##ating": 5313, + "##old": 5314, + "##unn": 5315, + "##ike": 5316, + "##vel": 5317, + "beach": 5318, + "day": 5319, + "mon": 5320, + "pict": 5321, + "##ug": 5322, + "##ase": 5323, + "##ave": 5324, + "wear": 5325, + "##eld": 5326, + "but": 5327, + "mod": 5328, + "##ures": 5329, + "cr": 5330, + "##ashion": 5331, + "fashion": 5332, + "ele": 5333, + "royalty": 5334, + "playing": 5335, + "comp": 5336, + "floor": 5337, + "beaut": 5338, + "##ark": 5339, + "##ric": 5340, + "ind": 5341, + "post": 5342, + "tre": 5343, + "res": 5344, + "over": 5345, + "19": 5346, + "rec": 5347, + "ac": 5348, + "street": 5349, + "##art": 5350, + "photography": 5351, + "side": 5352, + "he": 5353, + "world": 5354, + "mus": 5355, + "##orn": 5356, + "##out": 5357, + "style": 5358, + "glass": 5359, + "##ft": 5360, + "##ange": 5361, + "pre": 5362, + "cut": 5363, + "head": 5364, + "##ich": 5365, + "pat": 5366, + "##ance": 5367, + "##ellow": 5368, + "##uck": 5369, + "fish": 5370, + "their": 5371, + "##ages": 5372, + "##ue": 5373, + "baby": 5374, + "##ves": 5375, + "decor": 5376, + "bar": 5377, + "##ior": 5378, + "yellow": 5379, + "##te": 5380, + "sea": 5381, + "##nder": 5382, + "other": 5383, + "##iving": 5384, + "open": 5385, + "down": 5386, + "##act": 5387, + "set": 5388, + "##day": 5389, + "was": 5390, + "wooden": 5391, + "brown": 5392, + "hair": 5393, + "photos": 5394, + "ve": 5395, + "kn": 5396, + "##ical": 5397, + "##per": 5398, + "sun": 5399, + "af": 5400, + "snow": 5401, + "gard": 5402, + "fo": 5403, + "city": 5404, + "do": 5405, + "##ence": 5406, + "lay": 5407, + "light": 5408, + "##ents": 5409, + "##toon": 5410, + "cartoon": 5411, + "##bo": 5412, + "##py": 5413, + "go": 5414, + "##ng": 5415, + "##ank": 5416, + "log": 5417, + "bot": 5418, + "##ying": 5419, + "wearing": 5420, + "##ition": 5421, + "gu": 5422, + "tatt": 5423, + "##pl": 5424, + "##orm": 5425, + "tattoo": 5426, + "cent": 5427, + "flowers": 5428, + "lit": 5429, + "##iff": 5430, + "sl": 5431, + "##ping": 5432, + "chr": 5433, + "##ory": 5434, + "dogs": 5435, + "print": 5436, + "##urn": 5437, + "##ull": 5438, + "##ife": 5439, + "hotel": 5440, + "plant": 5441, + "kitch": 5442, + "ide": 5443, + "ball": 5444, + "cra": 5445, + "##iful": 5446, + "##au": 5447, + "beautiful": 5448, + "##mb": 5449, + "inside": 5450, + "box": 5451, + "##io": 5452, + "ev": 5453, + "moun": 5454, + "##eth": 5455, + "##ys": 5456, + "##oll": 5457, + "mov": 5458, + "cou": 5459, + "not": 5460, + "wind": 5461, + "##play": 5462, + "##ime": 5463, + "tea": 5464, + "kitchen": 5465, + "garden": 5466, + "child": 5467, + "young": 5468, + "painting": 5469, + "class": 5470, + "inter": 5471, + "next": 5472, + "##anch": 5473, + "##ons": 5474, + "ideas": 5475, + "images": 5476, + "##du": 5477, + "##ool": 5478, + "str": 5479, + "eng": 5480, + "gold": 5481, + "christ": 5482, + "##atch": 5483, + "##ire": 5484, + "sign": 5485, + "during": 5486, + "vint": 5487, + "vintage": 5488, + "che": 5489, + "men": 5490, + "met": 5491, + "bath": 5492, + "near": 5493, + "sur": 5494, + "##man": 5495, + "char": 5496, + "bow": 5497, + "star": 5498, + "face": 5499, + "##ern": 5500, + "##way": 5501, + "##ets": 5502, + "##mas": 5503, + "##ful": 5504, + "walking": 5505, + "sky": 5506, + "bre": 5507, + "someone": 5508, + "long": 5509, + "sil": 5510, + "##era": 5511, + "field": 5512, + "pla": 5513, + "laying": 5514, + "##rough": 5515, + "ser": 5516, + "make": 5517, + "text": 5518, + "little": 5519, + "tak": 5520, + "best": 5521, + "##outh": 5522, + "##ss": 5523, + "##iver": 5524, + "sy": 5525, + "branch": 5526, + "##ask": 5527, + "##ound": 5528, + "have": 5529, + "##ines": 5530, + "##ball": 5531, + "wallp": 5532, + "road": 5533, + "##ids": 5534, + "##ots": 5535, + "coun": 5536, + "##ces": 5537, + "##ted": 5538, + "##ming": 5539, + "##sh": 5540, + "##cept": 5541, + "wild": 5542, + "christmas": 5543, + "##aw": 5544, + "mountain": 5545, + "high": 5546, + "game": 5547, + "display": 5548, + "sch": 5549, + "##ord": 5550, + "pink": 5551, + "logo": 5552, + "##ors": 5553, + "##ics": 5554, + "cap": 5555, + "year": 5556, + "##ps": 5557, + "##its": 5558, + "fam": 5559, + "bra": 5560, + "which": 5561, + "loc": 5562, + "am": 5563, + "flower": 5564, + "being": 5565, + "stud": 5566, + "port": 5567, + "into": 5568, + "sho": 5569, + "##ower": 5570, + "work": 5571, + "through": 5572, + "##are": 5573, + "like": 5574, + "##ather": 5575, + "night": 5576, + "animals": 5577, + "ey": 5578, + "cro": 5579, + "three": 5580, + "cont": 5581, + "cover": 5582, + "bag": 5583, + "##br": 5584, + "camera": 5585, + "##other": 5586, + "living": 5587, + "made": 5588, + "##ject": 5589, + "colle": 5590, + "##ram": 5591, + "##az": 5592, + "##con": 5593, + "cute": 5594, + "##dy": 5595, + "##ational": 5596, + "diff": 5597, + "love": 5598, + "##ium": 5599, + "window": 5600, + "under": 5601, + "##ose": 5602, + "##iqu": 5603, + "recip": 5604, + "##unny": 5605, + "will": 5606, + "##een": 5607, + "dark": 5608, + "##lee": 5609, + "plan": 5610, + "##ult": 5611, + "fla": 5612, + "##cess": 5613, + "them": 5614, + "oce": 5615, + "##sc": 5616, + "##sp": 5617, + "trees": 5618, + "win": 5619, + "big": 5620, + "pup": 5621, + "##arm": 5622, + "land": 5623, + "pattern": 5624, + "##els": 5625, + "##col": 5626, + "outside": 5627, + "di": 5628, + "nat": 5629, + "ag": 5630, + "around": 5631, + "arch": 5632, + "sup": 5633, + "poster": 5634, + "ocean": 5635, + "##ape": 5636, + "how": 5637, + "##ily": 5638, + "differ": 5639, + "pol": 5640, + "door": 5641, + "bedroom": 5642, + "##als": 5643, + "very": 5644, + "quot": 5645, + "arm": 5646, + "drawing": 5647, + "perf": 5648, + "##00": 5649, + "ce": 5650, + "wallpaper": 5651, + "em": 5652, + "mot": 5653, + "time": 5654, + "women": 5655, + "fir": 5656, + "eas": 5657, + "##yc": 5658, + "area": 5659, + "##iss": 5660, + "ear": 5661, + "med": 5662, + "##ney": 5663, + "paper": 5664, + "slee": 5665, + "min": 5666, + "##ious": 5667, + "##ider": 5668, + "act": 5669, + "pie": 5670, + "stick": 5671, + "##ise": 5672, + "##key": 5673, + "##pe": 5674, + "different": 5675, + "bathroom": 5676, + "fire": 5677, + "modern": 5678, + "##amp": 5679, + "get": 5680, + "mid": 5681, + "interior": 5682, + "school": 5683, + "inc": 5684, + "kids": 5685, + "pot": 5686, + "leaves": 5687, + "hang": 5688, + "concept": 5689, + "##ions": 5690, + "pictures": 5691, + "##port": 5692, + "##ates": 5693, + "snake": 5694, + "eating": 5695, + "brid": 5696, + "sand": 5697, + "she": 5698, + "##ations": 5699, + "##ative": 5700, + "##co": 5701, + "nor": 5702, + "great": 5703, + "##let": 5704, + "##meric": 5705, + "mu": 5706, + "##ual": 5707, + "feat": 5708, + "sal": 5709, + "who": 5710, + "americ": 5711, + "swim": 5712, + "metal": 5713, + "boy": 5714, + "pur": 5715, + "shop": 5716, + "##aur": 5717, + "##iness": 5718, + "first": 5719, + "wr": 5720, + "##dle": 5721, + "##ips": 5722, + "life": 5723, + "about": 5724, + "##ins": 5725, + "##ri": 5726, + "##ner": 5727, + "river": 5728, + "squ": 5729, + "##ress": 5730, + "cost": 5731, + "rest": 5732, + "hel": 5733, + "cloth": 5734, + "grou": 5735, + "cor": 5736, + "din": 5737, + "part": 5738, + "##ert": 5739, + "charact": 5740, + "our": 5741, + "war": 5742, + "app": 5743, + "rocks": 5744, + "##ron": 5745, + "been": 5746, + "pen": 5747, + "outd": 5748, + "where": 5749, + "cars": 5750, + "outdoor": 5751, + "ty": 5752, + "##dition": 5753, + "plate": 5754, + "hor": 5755, + "line": 5756, + "ant": 5757, + "birth": 5758, + "##ient": 5759, + "##ally": 5760, + "many": 5761, + "su": 5762, + "family": 5763, + "birds": 5764, + "business": 5765, + "cur": 5766, + "##ench": 5767, + "tur": 5768, + "party": 5769, + "butter": 5770, + "hat": 5771, + "forest": 5772, + "may": 5773, + "##ural": 5774, + "picture": 5775, + "ever": 5776, + "space": 5777, + "##ese": 5778, + "isol": 5779, + "coff": 5780, + "music": 5781, + "foot": 5782, + "##ork": 5783, + "##anc": 5784, + "ow": 5785, + "hanging": 5786, + "cook": 5787, + "##ury": 5788, + "outf": 5789, + "##ique": 5790, + "ste": 5791, + "quotes": 5792, + "each": 5793, + "hear": 5794, + "##nds": 5795, + "mil": 5796, + "movie": 5797, + "##ble": 5798, + "##ix": 5799, + "group": 5800, + "birthday": 5801, + "funny": 5802, + "##ould": 5803, + "isolated": 5804, + "hall": 5805, + "more": 5806, + "sum": 5807, + "scre": 5808, + "##board": 5809, + "classic": 5810, + "travel": 5811, + "sim": 5812, + "coffee": 5813, + "spe": 5814, + "2019": 5815, + "insp": 5816, + "fruit": 5817, + "ret": 5818, + "##bs": 5819, + "10": 5820, + "office": 5821, + "veg": 5822, + "##ustr": 5823, + "##ax": 5824, + "num": 5825, + "suit": 5826, + "##inal": 5827, + "mach": 5828, + "monkey": 5829, + "close": 5830, + "je": 5831, + "hum": 5832, + "after": 5833, + "swimming": 5834, + "mag": 5835, + "mem": 5836, + "let": 5837, + "taken": 5838, + "leg": 5839, + "these": 5840, + "elect": 5841, + "##af": 5842, + "fil": 5843, + "middle": 5844, + "##ster": 5845, + "##owing": 5846, + "##ling": 5847, + "##ove": 5848, + "hap": 5849, + "located": 5850, + "watch": 5851, + "hist": 5852, + "##akes": 5853, + "##ton": 5854, + "##cture": 5855, + "aut": 5856, + "lake": 5857, + "##ren": 5858, + "portra": 5859, + "archite": 5860, + "just": 5861, + "rem": 5862, + "cab": 5863, + "good": 5864, + "##ract": 5865, + "##ik": 5866, + "cle": 5867, + "##illa": 5868, + "lond": 5869, + "costum": 5870, + "collection": 5871, + "produ": 5872, + "ang": 5873, + "london": 5874, + "card": 5875, + "comm": 5876, + "##uff": 5877, + "jack": 5878, + "designs": 5879, + "##ven": 5880, + "map": 5881, + "##ier": 5882, + "comput": 5883, + "##ob": 5884, + "puppy": 5885, + "toy": 5886, + "most": 5887, + "showing": 5888, + "hands": 5889, + "inspir": 5890, + "shoes": 5891, + "furn": 5892, + "##ones": 5893, + "##ane": 5894, + "##elf": 5895, + "rain": 5896, + "disney": 5897, + "##ether": 5898, + "##ays": 5899, + "##unch": 5900, + "##line": 5901, + "##fly": 5902, + "fun": 5903, + "##try": 5904, + "pres": 5905, + "spider": 5906, + "##ope": 5907, + "tattoos": 5908, + "american": 5909, + "they": 5910, + "##artment": 5911, + "prin": 5912, + "full": 5913, + "##20": 5914, + "landsc": 5915, + "colorful": 5916, + "cele": 5917, + "north": 5918, + "lad": 5919, + "##ai": 5920, + "team": 5921, + "ref": 5922, + "dresses": 5923, + "trans": 5924, + "emp": 5925, + "summer": 5926, + "##iding": 5927, + "cho": 5928, + "##ails": 5929, + "##esh": 5930, + "##ush": 5931, + "##ley": 5932, + "##ette": 5933, + "surf": 5934, + "##phone": 5935, + "beh": 5936, + "##iting": 5937, + "##ital": 5938, + "swe": 5939, + "##ump": 5940, + "restaur": 5941, + "pos": 5942, + "words": 5943, + "restaurant": 5944, + "pal": 5945, + "##izard": 5946, + "##alth": 5947, + "desk": 5948, + "bel": 5949, + "tem": 5950, + "farm": 5951, + "##olf": 5952, + "##iling": 5953, + "leaf": 5954, + "flag": 5955, + "silver": 5956, + "gif": 5957, + "fit": 5958, + "neck": 5959, + "winter": 5960, + "att": 5961, + "bask": 5962, + "bott": 5963, + "mark": 5964, + "sever": 5965, + "body": 5966, + "center": 5967, + "spr": 5968, + "stra": 5969, + "har": 5970, + "##unning": 5971, + "contain": 5972, + "orig": 5973, + "##ough": 5974, + "children": 5975, + "computer": 5976, + "cour": 5977, + "siz": 5978, + "##ular": 5979, + "prem": 5980, + "##od": 5981, + "##ivers": 5982, + "machine": 5983, + "##iture": 5984, + "chick": 5985, + "kind": 5986, + "heart": 5987, + "ter": 5988, + "whe": 5989, + "aw": 5990, + "furniture": 5991, + "##enn": 5992, + "used": 5993, + "pop": 5994, + "motor": 5995, + "island": 5996, + "happy": 5997, + "event": 5998, + "vis": 5999, + "##so": 6000, + "character": 6001, + "eyes": 6002, + "several": 6003, + "##ved": 6004, + "lizard": 6005, + "celebr": 6006, + "portrait": 6007, + "behind": 6008, + "what": 6009, + "icon": 6010, + "##top": 6011, + "##ene": 6012, + "pan": 6013, + "mouth": 6014, + "inse": 6015, + "##ning": 6016, + "eg": 6017, + "health": 6018, + "bridge": 6019, + "recipes": 6020, + "key": 6021, + "insect": 6022, + "ed": 6023, + "tour": 6024, + "colors": 6025, + "del": 6026, + "right": 6027, + "pool": 6028, + "sec": 6029, + "plants": 6030, + "##ames": 6031, + "##ems": 6032, + "flying": 6033, + "parked": 6034, + "read": 6035, + "purple": 6036, + "##eak": 6037, + "need": 6038, + "ob": 6039, + "inclu": 6040, + "fram": 6041, + "end": 6042, + "there": 6043, + "board": 6044, + "##rel": 6045, + "##ched": 6046, + "tog": 6047, + "base": 6048, + "vide": 6049, + "when": 6050, + "couple": 6051, + "##cial": 6052, + "##ible": 6053, + "bat": 6054, + "##wood": 6055, + "var": 6056, + "##less": 6057, + "sport": 6058, + "national": 6059, + "##oon": 6060, + "corn": 6061, + "cool": 6062, + "##ped": 6063, + "woods": 6064, + "again": 6065, + "butterfly": 6066, + "rel": 6067, + "fall": 6068, + "piece": 6069, + "together": 6070, + "project": 6071, + "ban": 6072, + "south": 6073, + "ice": 6074, + "det": 6075, + "cross": 6076, + "basket": 6077, + "##ger": 6078, + "val": 6079, + "lap": 6080, + "##ux": 6081, + "every": 6082, + "##astic": 6083, + "3d": 6084, + "cir": 6085, + "afric": 6086, + "##emon": 6087, + "store": 6088, + "looks": 6089, + "form": 6090, + "apartment": 6091, + "cust": 6092, + "sale": 6093, + "bal": 6094, + "frie": 6095, + "architecture": 6096, + "##cc": 6097, + "circ": 6098, + "##ities": 6099, + "bet": 6100, + "sing": 6101, + "2018": 6102, + "leather": 6103, + "muse": 6104, + "wine": 6105, + "##lic": 6106, + "any": 6107, + "while": 6108, + "##lish": 6109, + "##ah": 6110, + "##lace": 6111, + "jun": 6112, + "##ina": 6113, + "200": 6114, + "clou": 6115, + "beds": 6116, + "##ival": 6117, + "girls": 6118, + "film": 6119, + "##urch": 6120, + "##ows": 6121, + "museum": 6122, + "##iday": 6123, + "custom": 6124, + "##ator": 6125, + "mas": 6126, + "roo": 6127, + "##ycle": 6128, + "##red": 6129, + "creative": 6130, + "phone": 6131, + "##ats": 6132, + "outfits": 6133, + "##hic": 6134, + "cast": 6135, + "model": 6136, + "vehic": 6137, + "another": 6138, + "pet": 6139, + "books": 6140, + "rep": 6141, + "fin": 6142, + "church": 6143, + "inf": 6144, + "##eep": 6145, + "dist": 6146, + "18": 6147, + "running": 6148, + "super": 6149, + "grey": 6150, + "##ables": 6151, + "empty": 6152, + "fence": 6153, + "guit": 6154, + "guitar": 6155, + "stone": 6156, + "##ited": 6157, + "see": 6158, + "four": 6159, + "ship": 6160, + "##work": 6161, + "##oy": 6162, + "walk": 6163, + "##rop": 6164, + "wee": 6165, + "indian": 6166, + "##set": 6167, + "easy": 6168, + "chocol": 6169, + "lab": 6170, + "cream": 6171, + "lot": 6172, + "av": 6173, + "seas": 6174, + "hill": 6175, + "past": 6176, + "##ctor": 6177, + "##ards": 6178, + "mush": 6179, + "arr": 6180, + "exp": 6181, + "fig": 6182, + "number": 6183, + "featuring": 6184, + "way": 6185, + "##der": 6186, + "mushroom": 6187, + "case": 6188, + "som": 6189, + "zoo": 6190, + "cand": 6191, + "##ied": 6192, + "country": 6193, + "jap": 6194, + "against": 6195, + "graph": 6196, + "japan": 6197, + "short": 6198, + "##ost": 6199, + "veget": 6200, + "##ches": 6201, + "cabin": 6202, + "landscape": 6203, + "using": 6204, + "symb": 6205, + "history": 6206, + "left": 6207, + "stat": 6208, + "flat": 6209, + "use": 6210, + "##oss": 6211, + "chicken": 6212, + "symbol": 6213, + "bas": 6214, + "stuff": 6215, + "king": 6216, + "fant": 6217, + "real": 6218, + "pair": 6219, + "should": 6220, + "##ym": 6221, + "chocolate": 6222, + "##gan": 6223, + "bike": 6224, + "sou": 6225, + "##ters": 6226, + "jacket": 6227, + "austr": 6228, + "market": 6229, + "mer": 6230, + "break": 6231, + "screen": 6232, + "fac": 6233, + "##ced": 6234, + "same": 6235, + "artist": 6236, + "dre": 6237, + "##ives": 6238, + "surface": 6239, + "serv": 6240, + "plastic": 6241, + "place": 6242, + "##alk": 6243, + "jew": 6244, + "##ten": 6245, + "fest": 6246, + "mak": 6247, + "##olog": 6248, + "mask": 6249, + "sol": 6250, + "spring": 6251, + "original": 6252, + "gar": 6253, + "mountains": 6254, + "##ote": 6255, + "recipe": 6256, + "pack": 6257, + "product": 6258, + "west": 6259, + "perfect": 6260, + "riding": 6261, + "bro": 6262, + "tow": 6263, + "##cra": 6264, + "##ror": 6265, + "mother": 6266, + "##illed": 6267, + "##led": 6268, + "paint": 6269, + "lady": 6270, + "##ely": 6271, + "##par": 6272, + "##vers": 6273, + "rea": 6274, + "dining": 6275, + "bride": 6276, + "simple": 6277, + "things": 6278, + "villa": 6279, + "cas": 6280, + "lux": 6281, + "natural": 6282, + "##ok": 6283, + "##met": 6284, + "drag": 6285, + "perform": 6286, + "2020": 6287, + "size": 6288, + "dish": 6289, + "##ience": 6290, + "video": 6291, + "##phant": 6292, + "here": 6293, + "perched": 6294, + "dig": 6295, + "cer": 6296, + "conc": 6297, + "list": 6298, + "cell": 6299, + "##unk": 6300, + "electric": 6301, + "##house": 6302, + "##aid": 6303, + "human": 6304, + "roll": 6305, + "##iny": 6306, + "ge": 6307, + "object": 6308, + "blank": 6309, + "2017": 6310, + "ger": 6311, + "trail": 6312, + "fem": 6313, + "call": 6314, + "tool": 6315, + "hallow": 6316, + "austral": 6317, + "##augh": 6318, + "find": 6319, + "sunset": 6320, + "halloween": 6321, + "costume": 6322, + "sket": 6323, + "items": 6324, + "acc": 6325, + "el": 6326, + "12": 6327, + "##asy": 6328, + "oil": 6329, + "##hou": 6330, + "ital": 6331, + "##ices": 6332, + "only": 6333, + "were": 6334, + "##ories": 6335, + "eye": 6336, + "dream": 6337, + "clu": 6338, + "##ral": 6339, + "above": 6340, + "##ness": 6341, + "resort": 6342, + "##nam": 6343, + "reg": 6344, + "fantasy": 6345, + "games": 6346, + "also": 6347, + "##any": 6348, + "##ured": 6349, + "word": 6350, + "organ": 6351, + "##itary": 6352, + "hal": 6353, + "found": 6354, + "doll": 6355, + "##fore": 6356, + "tower": 6357, + "mir": 6358, + "##vent": 6359, + "##uss": 6360, + "along": 6361, + "fox": 6362, + "##stru": 6363, + "premi": 6364, + "##vas": 6365, + "military": 6366, + "strip": 6367, + "silhou": 6368, + "buildings": 6369, + "typ": 6370, + "sub": 6371, + "##dd": 6372, + "before": 6373, + "premiere": 6374, + "##bers": 6375, + "##iet": 6376, + "turtle": 6377, + "nature": 6378, + "la": 6379, + "golden": 6380, + "now": 6381, + "ham": 6382, + "shows": 6383, + "frame": 6384, + "german": 6385, + "sleeping": 6386, + "drink": 6387, + "##ato": 6388, + "hom": 6389, + "sail": 6390, + "##ana": 6391, + "web": 6392, + "tradition": 6393, + "gray": 6394, + "##be": 6395, + "rom": 6396, + "ligh": 6397, + "bright": 6398, + "working": 6399, + "don": 6400, + "beauty": 6401, + "gen": 6402, + "roof": 6403, + "even": 6404, + "amaz": 6405, + "chairs": 6406, + "town": 6407, + "dirt": 6408, + "vol": 6409, + "power": 6410, + "15": 6411, + "pill": 6412, + "moon": 6413, + "india": 6414, + "female": 6415, + "##gle": 6416, + "station": 6417, + "##craft": 6418, + "stage": 6419, + "##ething": 6420, + "lion": 6421, + "lights": 6422, + "gift": 6423, + "glasses": 6424, + "grand": 6425, + "letter": 6426, + "something": 6427, + "egg": 6428, + "equ": 6429, + "painted": 6430, + "carp": 6431, + "take": 6432, + "canvas": 6433, + "##ries": 6434, + "seen": 6435, + "cot": 6436, + "band": 6437, + "series": 6438, + "pret": 6439, + "fab": 6440, + "england": 6441, + "tit": 6442, + "ann": 6443, + "football": 6444, + "##cil": 6445, + "access": 6446, + "week": 6447, + "covered": 6448, + "##urs": 6449, + "fresh": 6450, + "##ement": 6451, + "motorc": 6452, + "##ler": 6453, + "sticker": 6454, + "growing": 6455, + "colored": 6456, + "dragon": 6457, + "tech": 6458, + "toys": 6459, + "cock": 6460, + "##inese": 6461, + "tong": 6462, + "clothing": 6463, + "lying": 6464, + "##oc": 6465, + "luxury": 6466, + "ring": 6467, + "##ne": 6468, + "french": 6469, + "chinese": 6470, + "plu": 6471, + "##eng": 6472, + "contr": 6473, + "##ole": 6474, + "##ember": 6475, + "clos": 6476, + "ca": 6477, + "san": 6478, + "festival": 6479, + "angel": 6480, + "inst": 6481, + "##eting": 6482, + "places": 6483, + "intern": 6484, + "wheel": 6485, + "sat": 6486, + "tank": 6487, + "well": 6488, + "player": 6489, + "##ween": 6490, + "sports": 6491, + "syst": 6492, + "##ues": 6493, + "brit": 6494, + "11": 6495, + "tips": 6496, + "pass": 6497, + "anc": 6498, + "main": 6499, + "hard": 6500, + "##por": 6501, + "court": 6502, + "between": 6503, + "track": 6504, + "clean": 6505, + "mini": 6506, + "hy": 6507, + "match": 6508, + "retro": 6509, + "if": 6510, + "holiday": 6511, + "wa": 6512, + "movies": 6513, + "##oney": 6514, + "type": 6515, + "##gy": 6516, + "##ific": 6517, + "lear": 6518, + "round": 6519, + "coat": 6520, + "frog": 6521, + "##ments": 6522, + "##0s": 6523, + "chain": 6524, + "kit": 6525, + "cal": 6526, + "##ument": 6527, + "abst": 6528, + "years": 6529, + "than": 6530, + "antique": 6531, + "turn": 6532, + "own": 6533, + "seat": 6534, + "##aving": 6535, + "digital": 6536, + "season": 6537, + "drawings": 6538, + "lib": 6539, + "2016": 6540, + "gall": 6541, + "run": 6542, + "##umb": 6543, + "##ident": 6544, + "##ants": 6545, + "bec": 6546, + "eur": 6547, + "hood": 6548, + "ancient": 6549, + "last": 6550, + "online": 6551, + "comple": 6552, + "news": 6553, + "abstract": 6554, + "##iger": 6555, + "african": 6556, + "emb": 6557, + "add": 6558, + "rac": 6559, + "##uary": 6560, + "square": 6561, + "rust": 6562, + "vehicle": 6563, + "bee": 6564, + "##gl": 6565, + "cheese": 6566, + "craf": 6567, + "brick": 6568, + "id": 6569, + "toile": 6570, + "tongue": 6571, + "club": 6572, + "making": 6573, + "yard": 6574, + "flor": 6575, + "silhouette": 6576, + "##ional": 6577, + "anime": 6578, + "earth": 6579, + "##yard": 6580, + "various": 6581, + "boats": 6582, + "beet": 6583, + "inv": 6584, + "check": 6585, + "gym": 6586, + "##za": 6587, + "##ret": 6588, + "baseball": 6589, + "drawn": 6590, + "filled": 6591, + "hous": 6592, + "univers": 6593, + "writing": 6594, + "bug": 6595, + "traditional": 6596, + "##ism": 6597, + "coast": 6598, + "bottles": 6599, + "scul": 6600, + "soft": 6601, + "miss": 6602, + "company": 6603, + "jewel": 6604, + "##bor": 6605, + "##row": 6606, + "fair": 6607, + "rad": 6608, + "engine": 6609, + "tim": 6610, + "imp": 6611, + "smiling": 6612, + "pig": 6613, + "offic": 6614, + "duck": 6615, + "prot": 6616, + "healthy": 6617, + "hd": 6618, + "castle": 6619, + "cos": 6620, + "mic": 6621, + "nails": 6622, + "##ross": 6623, + "bur": 6624, + "16": 6625, + "mug": 6626, + "sculpt": 6627, + "inspiration": 6628, + "sleeve": 6629, + "prov": 6630, + "shape": 6631, + "aqu": 6632, + "rev": 6633, + "iron": 6634, + "fabric": 6635, + "##ky": 6636, + "champ": 6637, + "shower": 6638, + "pub": 6639, + "##umn": 6640, + "golf": 6641, + "students": 6642, + "watercol": 6643, + "ent": 6644, + "taking": 6645, + "##ology": 6646, + "##uc": 6647, + "name": 6648, + "police": 6649, + "##ching": 6650, + "carpet": 6651, + "shad": 6652, + "dead": 6653, + "bul": 6654, + "pick": 6655, + "studio": 6656, + "crafts": 6657, + "shark": 6658, + "pra": 6659, + "##ute": 6660, + "camp": 6661, + "mirror": 6662, + "stair": 6663, + "stor": 6664, + "techn": 6665, + "##aj": 6666, + "friend": 6667, + "dep": 6668, + "pump": 6669, + "mix": 6670, + "dev": 6671, + "##fl": 6672, + "bad": 6673, + "smo": 6674, + "##struction": 6675, + "uk": 6676, + "##ving": 6677, + "english": 6678, + "gown": 6679, + "had": 6680, + "jo": 6681, + "##gu": 6682, + "##eap": 6683, + "clothes": 6684, + "half": 6685, + "beer": 6686, + "lin": 6687, + "##iles": 6688, + "comic": 6689, + "##glass": 6690, + "care": 6691, + "clouds": 6692, + "##ship": 6693, + "van": 6694, + "pretty": 6695, + "##lywood": 6696, + "cop": 6697, + "shell": 6698, + "capt": 6699, + "statue": 6700, + "bread": 6701, + "bay": 6702, + "morn": 6703, + "instr": 6704, + "brand": 6705, + "pin": 6706, + "law": 6707, + "dess": 6708, + "system": 6709, + "colour": 6710, + "cons": 6711, + "point": 6712, + "saf": 6713, + "counter": 6714, + "fore": 6715, + "tele": 6716, + "tenn": 6717, + "##eds": 6718, + "prom": 6719, + "##me": 6720, + "vac": 6721, + "shot": 6722, + "straw": 6723, + "posters": 6724, + "blanket": 6725, + "##ett": 6726, + "morning": 6727, + "13": 6728, + "going": 6729, + "boots": 6730, + "activ": 6731, + "mill": 6732, + "rug": 6733, + "amazing": 6734, + "graphic": 6735, + "meat": 6736, + "tall": 6737, + "##ham": 6738, + "##ection": 6739, + "gas": 6740, + "##book": 6741, + "##mp": 6742, + "vegetables": 6743, + "##ature": 6744, + "##ung": 6745, + "spir": 6746, + "sau": 6747, + "##ocks": 6748, + "story": 6749, + "state": 6750, + "sold": 6751, + "japanese": 6752, + "##apers": 6753, + "14": 6754, + "jewelry": 6755, + "##uts": 6756, + "adult": 6757, + "gal": 6758, + "iphone": 6759, + "chic": 6760, + "ext": 6761, + "fly": 6762, + "tennis": 6763, + "windows": 6764, + "europe": 6765, + "##wing": 6766, + "mom": 6767, + "vers": 6768, + "fur": 6769, + "los": 6770, + "##read": 6771, + "shap": 6772, + "enj": 6773, + "##ising": 6774, + "homes": 6775, + "wolf": 6776, + "united": 6777, + "pea": 6778, + "sweet": 6779, + "across": 6780, + "helmet": 6781, + "skin": 6782, + "asian": 6783, + "nav": 6784, + "##ga": 6785, + "doing": 6786, + "comics": 6787, + "##made": 6788, + "guide": 6789, + "oct": 6790, + "##ms": 6791, + "2015": 6792, + "inn": 6793, + "race": 6794, + "canad": 6795, + "lif": 6796, + "commun": 6797, + "beetle": 6798, + "stars": 6799, + "want": 6800, + "slim": 6801, + "special": 6802, + "backyard": 6803, + "fact": 6804, + "por": 6805, + "diam": 6806, + "##ized": 6807, + "mos": 6808, + "walls": 6809, + "fut": 6810, + "tom": 6811, + "steel": 6812, + "airpl": 6813, + "says": 6814, + "##fast": 6815, + "watercolor": 6816, + "##abb": 6817, + "prof": 6818, + "scene": 6819, + "china": 6820, + "lang": 6821, + "houses": 6822, + "##terior": 6823, + "pencil": 6824, + "breakfast": 6825, + "##ides": 6826, + "princess": 6827, + "gear": 6828, + "plans": 6829, + "page": 6830, + "knit": 6831, + "##bur": 6832, + "ic": 6833, + "male": 6834, + "gallery": 6835, + "##ution": 6836, + "17": 6837, + "##zz": 6838, + "##inger": 6839, + "##vert": 6840, + "##ision": 6841, + "exterior": 6842, + "trip": 6843, + "ass": 6844, + "##inking": 6845, + "##ov": 6846, + "march": 6847, + "##fish": 6848, + "##enc": 6849, + "##orial": 6850, + "marvel": 6851, + "trucks": 6852, + "force": 6853, + "tools": 6854, + "bottom": 6855, + "him": 6856, + "fishing": 6857, + "##min": 6858, + "help": 6859, + "reading": 6860, + "shoulder": 6861, + "ur": 6862, + "know": 6863, + "british": 6864, + "still": 6865, + "visit": 6866, + "refle": 6867, + "fav": 6868, + "dire": 6869, + "pull": 6870, + "##xic": 6871, + "ven": 6872, + "necklace": 6873, + "unif": 6874, + "##omet": 6875, + "wings": 6876, + "doors": 6877, + "army": 6878, + "sketch": 6879, + "legs": 6880, + "##lam": 6881, + "costumes": 6882, + "industr": 6883, + "floral": 6884, + "##lo": 6885, + "lim": 6886, + "action": 6887, + "second": 6888, + "crab": 6889, + "century": 6890, + "son": 6891, + "seam": 6892, + "corner": 6893, + "desert": 6894, + "##water": 6895, + "##arium": 6896, + "templ": 6897, + "unique": 6898, + "parking": 6899, + "##rap": 6900, + "den": 6901, + "cats": 6902, + "barn": 6903, + "uniform": 6904, + "30": 6905, + "##light": 6906, + "apr": 6907, + "gener": 6908, + "basketball": 6909, + "##iant": 6910, + "bik": 6911, + "ceiling": 6912, + "tail": 6913, + "rabb": 6914, + "diamond": 6915, + "would": 6916, + "int": 6917, + "##berry": 6918, + "##ica": 6919, + "##bow": 6920, + "##ality": 6921, + "boys": 6922, + "def": 6923, + "low": 6924, + "surround": 6925, + "nov": 6926, + "styles": 6927, + "##ze": 6928, + "##ino": 6929, + "lamp": 6930, + "held": 6931, + "cub": 6932, + "tub": 6933, + "equip": 6934, + "title": 6935, + "must": 6936, + "##kin": 6937, + "libr": 6938, + "seamless": 6939, + "paintings": 6940, + "dance": 6941, + "##ield": 6942, + "live": 6943, + "mexic": 6944, + "curtain": 6945, + "master": 6946, + "put": 6947, + "ber": 6948, + "bunch": 6949, + "conf": 6950, + "25": 6951, + "##que": 6952, + "ready": 6953, + "##uly": 6954, + "gor": 6955, + "pand": 6956, + "characters": 6957, + "outfit": 6958, + "##ille": 6959, + "cart": 6960, + "australia": 6961, + "brush": 6962, + "comb": 6963, + "shel": 6964, + "eag": 6965, + "paris": 6966, + "##bon": 6967, + "electron": 6968, + "makeup": 6969, + "eff": 6970, + "construction": 6971, + "eat": 6972, + "shopping": 6973, + "feed": 6974, + "patterns": 6975, + "cooking": 6976, + "fer": 6977, + "##reek": 6978, + "flight": 6979, + "##form": 6980, + "aud": 6981, + "shelf": 6982, + "clip": 6983, + "shown": 6984, + "create": 6985, + "exper": 6986, + "cage": 6987, + "nail": 6988, + "lob": 6989, + "hollywood": 6990, + "drum": 6991, + "##ird": 6992, + "rail": 6993, + "##ston": 6994, + "rooms": 6995, + "##ban": 6996, + "##iter": 6997, + "science": 6998, + "net": 6999, + "projects": 7000, + "spot": 7001, + "village": 7002, + "##rip": 7003, + "plane": 7004, + "sauce": 7005, + "groom": 7006, + "vi": 7007, + "##ars": 7008, + "patio": 7009, + "##glasses": 7010, + "buy": 7011, + "exc": 7012, + "##vy": 7013, + "artwork": 7014, + "##atic": 7015, + "including": 7016, + "service": 7017, + "##lin": 7018, + "##ara": 7019, + "types": 7020, + "library": 7021, + "terri": 7022, + "ir": 7023, + "##und": 7024, + "pad": 7025, + "##sty": 7026, + "april": 7027, + "##hib": 7028, + "tor": 7029, + "sculpture": 7030, + "buck": 7031, + "##walk": 7032, + "aug": 7033, + "autumn": 7034, + "ze": 7035, + "angeles": 7036, + "guest": 7037, + "expl": 7038, + "built": 7039, + "aircraft": 7040, + "aquarium": 7041, + "clim": 7042, + "getting": 7043, + "stuffed": 7044, + "feet": 7045, + "pian": 7046, + "micro": 7047, + "est": 7048, + "atte": 7049, + "storage": 7050, + "mushrooms": 7051, + "wash": 7052, + "umb": 7053, + "deck": 7054, + "##ala": 7055, + "gun": 7056, + "jan": 7057, + "jeans": 7058, + "signs": 7059, + "2014": 7060, + "##res": 7061, + "start": 7062, + "##thern": 7063, + "mens": 7064, + "handle": 7065, + "equipment": 7066, + "##ished": 7067, + "##eakers": 7068, + "ceremon": 7069, + "tal": 7070, + "socc": 7071, + "danc": 7072, + "sunglasses": 7073, + "soccer": 7074, + "hairsty": 7075, + "##ison": 7076, + "international": 7077, + "stain": 7078, + "##ef": 7079, + "lat": 7080, + "wars": 7081, + "##ann": 7082, + "##ipe": 7083, + "umbrel": 7084, + "stop": 7085, + "craw": 7086, + "##val": 7087, + "having": 7088, + "##earch": 7089, + "note": 7090, + "training": 7091, + "mal": 7092, + "accessories": 7093, + "piz": 7094, + "##ded": 7095, + "figure": 7096, + "gradu": 7097, + "rainbow": 7098, + "hea": 7099, + "##etic": 7100, + "magaz": 7101, + "##cl": 7102, + "100": 7103, + "##pack": 7104, + "196": 7105, + "##oodle": 7106, + "circle": 7107, + "diy": 7108, + "pieces": 7109, + "jum": 7110, + "tiny": 7111, + "east": 7112, + "site": 7113, + "sam": 7114, + "both": 7115, + "flo": 7116, + "displayed": 7117, + "driving": 7118, + "products": 7119, + "##ks": 7120, + "##ript": 7121, + "famous": 7122, + "##gest": 7123, + "##pers": 7124, + "today": 7125, + "sweater": 7126, + "dou": 7127, + "##go": 7128, + "clear": 7129, + "render": 7130, + "histor": 7131, + "dressed": 7132, + "##velop": 7133, + "##ility": 7134, + "kid": 7135, + "##pr": 7136, + "##istic": 7137, + "rabbit": 7138, + "spec": 7139, + "bub": 7140, + "july": 7141, + "##des": 7142, + "sund": 7143, + "nice": 7144, + "##ering": 7145, + "path": 7146, + "bell": 7147, + "far": 7148, + "fan": 7149, + "coral": 7150, + "199": 7151, + "##ena": 7152, + "mob": 7153, + "edge": 7154, + "dam": 7155, + "italy": 7156, + "soup": 7157, + "owner": 7158, + "piano": 7159, + "enter": 7160, + "scar": 7161, + "friends": 7162, + "bic": 7163, + "drive": 7164, + "bags": 7165, + "popular": 7166, + "exam": 7167, + "queen": 7168, + "croch": 7169, + "##place": 7170, + "sept": 7171, + "june": 7172, + "features": 7173, + "ford": 7174, + "palm": 7175, + "crow": 7176, + "cru": 7177, + "##get": 7178, + "decorations": 7179, + "hors": 7180, + "mult": 7181, + "caf": 7182, + "hun": 7183, + "cakes": 7184, + "pond": 7185, + "##rance": 7186, + "sof": 7187, + "cabinet": 7188, + "sug": 7189, + "month": 7190, + "valley": 7191, + "detail": 7192, + "##wr": 7193, + "leop": 7194, + "wire": 7195, + "##atures": 7196, + "##ailable": 7197, + "coll": 7198, + "present": 7199, + "temple": 7200, + "##amb": 7201, + "money": 7202, + "russ": 7203, + "ceremony": 7204, + "say": 7205, + "skirt": 7206, + "lead": 7207, + "setting": 7208, + "learn": 7209, + "advent": 7210, + "##case": 7211, + "##idge": 7212, + "phil": 7213, + "mater": 7214, + "rob": 7215, + "##ways": 7216, + "cards": 7217, + "##esth": 7218, + "197": 7219, + "eagle": 7220, + "potter": 7221, + "decoration": 7222, + "ra": 7223, + "mac": 7224, + "##anta": 7225, + "cult": 7226, + "bank": 7227, + "hour": 7228, + "evening": 7229, + "america": 7230, + "jour": 7231, + "airport": 7232, + "cott": 7233, + "span": 7234, + "##oose": 7235, + "hoodie": 7236, + "gifts": 7237, + "tou": 7238, + "pep": 7239, + "inspired": 7240, + "shar": 7241, + "scot": 7242, + "##head": 7243, + "africa": 7244, + "##ogram": 7245, + "mess": 7246, + "skull": 7247, + "lines": 7248, + "##dom": 7249, + "##agement": 7250, + "available": 7251, + "yog": 7252, + "easter": 7253, + "##porary": 7254, + "few": 7255, + "navy": 7256, + "gardens": 7257, + "##ma": 7258, + "21": 7259, + "stad": 7260, + "24": 7261, + "template": 7262, + "designed": 7263, + "container": 7264, + "public": 7265, + "older": 7266, + "hip": 7267, + "smart": 7268, + "save": 7269, + "exhib": 7270, + "wrap": 7271, + "##uments": 7272, + "##rot": 7273, + "lace": 7274, + "greeting": 7275, + "tiger": 7276, + "advert": 7277, + "##ause": 7278, + "lighting": 7279, + "bak": 7280, + "favor": 7281, + "##aff": 7282, + "step": 7283, + "rack": 7284, + "champion": 7285, + "sen": 7286, + "develop": 7287, + "stadium": 7288, + "##ini": 7289, + "198": 7290, + "hairstyles": 7291, + "eleg": 7292, + "##ised": 7293, + "single": 7294, + "magazine": 7295, + "cath": 7296, + "flu": 7297, + "##we": 7298, + "less": 7299, + "##ville": 7300, + "craft": 7301, + "crawling": 7302, + "conn": 7303, + "leopard": 7304, + "bush": 7305, + "stru": 7306, + "entrance": 7307, + "ted": 7308, + "magic": 7309, + "quote": 7310, + "attach": 7311, + "cafe": 7312, + "shoe": 7313, + "trop": 7314, + "##duc": 7315, + "horses": 7316, + "central": 7317, + "##ists": 7318, + "tractor": 7319, + "##ief": 7320, + "racing": 7321, + "rib": 7322, + "quality": 7323, + "##omb": 7324, + "shirts": 7325, + "appear": 7326, + "proper": 7327, + "five": 7328, + "numbers": 7329, + "rot": 7330, + "trend": 7331, + "pu": 7332, + "posing": 7333, + "illustrations": 7334, + "figh": 7335, + "dinner": 7336, + "tent": 7337, + "early": 7338, + "polar": 7339, + "could": 7340, + "glob": 7341, + "away": 7342, + "##omy": 7343, + "control": 7344, + "awards": 7345, + "coming": 7346, + "##itor": 7347, + "##nes": 7348, + "##ges": 7349, + "horn": 7350, + "##orable": 7351, + "beg": 7352, + "owl": 7353, + "file": 7354, + "##iti": 7355, + "du": 7356, + "stove": 7357, + "theat": 7358, + "future": 7359, + "##athers": 7360, + "canada": 7361, + "##antic": 7362, + "activities": 7363, + "lego": 7364, + "drinking": 7365, + "wallpapers": 7366, + "mobile": 7367, + "##ago": 7368, + "cutting": 7369, + "plat": 7370, + "contem": 7371, + "##iety": 7372, + "france": 7373, + "##ients": 7374, + "boxes": 7375, + "##ci": 7376, + "theme": 7377, + "texture": 7378, + "##ari": 7379, + "##ging": 7380, + "press": 7381, + "salad": 7382, + "##ister": 7383, + "lawn": 7384, + "##ental": 7385, + "ep": 7386, + "usa": 7387, + "memes": 7388, + "##ona": 7389, + "diag": 7390, + "cin": 7391, + "##ency": 7392, + "disc": 7393, + "rings": 7394, + "##estern": 7395, + "wel": 7396, + "casual": 7397, + "process": 7398, + "dock": 7399, + "##cover": 7400, + "##esy": 7401, + "players": 7402, + "double": 7403, + "rustic": 7404, + "alb": 7405, + "student": 7406, + "##ength": 7407, + "milk": 7408, + "lots": 7409, + "##ama": 7410, + "lock": 7411, + "palace": 7412, + "underwater": 7413, + "##ypt": 7414, + "dest": 7415, + "panda": 7416, + "mad": 7417, + "##osp": 7418, + "scen": 7419, + "dessert": 7420, + "prints": 7421, + "does": 7422, + "webs": 7423, + "tropical": 7424, + "indoor": 7425, + "wond": 7426, + "teddy": 7427, + "condition": 7428, + "shore": 7429, + "cla": 7430, + "garage": 7431, + "vict": 7432, + "contemporary": 7433, + "50": 7434, + "community": 7435, + "shut": 7436, + "days": 7437, + "italian": 7438, + "giant": 7439, + "meas": 7440, + "2013": 7441, + "##cus": 7442, + "blog": 7443, + "lem": 7444, + "tot": 7445, + "priv": 7446, + "pole": 7447, + "##ses": 7448, + "##elling": 7449, + "hero": 7450, + "##rie": 7451, + "##ca": 7452, + "##ta": 7453, + "thr": 7454, + "oly": 7455, + "##ken": 7456, + "olymp": 7457, + "shaped": 7458, + "squir": 7459, + "educ": 7460, + "##ountain": 7461, + "##erd": 7462, + "##ep": 7463, + "opt": 7464, + "fountain": 7465, + "floating": 7466, + "ing": 7467, + "squirrel": 7468, + "pumpkin": 7469, + "sewing": 7470, + "##ials": 7471, + "bridal": 7472, + "cookies": 7473, + "bud": 7474, + "ski": 7475, + "arts": 7476, + "adorable": 7477, + "burn": 7478, + "sugar": 7479, + "sheet": 7480, + "wheels": 7481, + "##ero": 7482, + "fine": 7483, + "install": 7484, + "##zy": 7485, + "rat": 7486, + "mel": 7487, + "##aster": 7488, + "fireplace": 7489, + "song": 7490, + "##cher": 7491, + "cockt": 7492, + "reef": 7493, + "electronic": 7494, + "centre": 7495, + "level": 7496, + "keep": 7497, + "build": 7498, + "vacation": 7499, + "though": 7500, + "##atoes": 7501, + "weight": 7502, + "stea": 7503, + "##rete": 7504, + "favorite": 7505, + "##arn": 7506, + "chest": 7507, + "##maid": 7508, + "##order": 7509, + "##aging": 7510, + "known": 7511, + "##gue": 7512, + "ess": 7513, + "sidewalk": 7514, + "wide": 7515, + "vehicles": 7516, + "medical": 7517, + "barb": 7518, + "someones": 7519, + "telev": 7520, + "##ored": 7521, + "elegant": 7522, + "##ince": 7523, + "ath": 7524, + "then": 7525, + "hu": 7526, + "beak": 7527, + "never": 7528, + "##itten": 7529, + "times": 7530, + "concrete": 7531, + "sofa": 7532, + "coloring": 7533, + "crochet": 7534, + "dry": 7535, + "##ata": 7536, + "theatre": 7537, + "effect": 7538, + "##ination": 7539, + "western": 7540, + "resting": 7541, + "compet": 7542, + "actor": 7543, + "fol": 7544, + "##nding": 7545, + "photographer": 7546, + "elements": 7547, + "feel": 7548, + "cosplay": 7549, + "carry": 7550, + "roman": 7551, + "##for": 7552, + "pillow": 7553, + "nurs": 7554, + "daugh": 7555, + "friday": 7556, + "quil": 7557, + "enjoy": 7558, + "env": 7559, + "icons": 7560, + "brace": 7561, + "##aves": 7562, + "rice": 7563, + "##over": 7564, + "much": 7565, + "tables": 7566, + "##ometric": 7567, + "2012": 7568, + "##hood": 7569, + "##oons": 7570, + "##town": 7571, + "makes": 7572, + "latest": 7573, + "##iron": 7574, + "gate": 7575, + "cottage": 7576, + "##imal": 7577, + "lemon": 7578, + "local": 7579, + "##bour": 7580, + "oak": 7581, + "hosp": 7582, + "##iam": 7583, + "spac": 7584, + "drinks": 7585, + "classroom": 7586, + "bill": 7587, + "courtesy": 7588, + "arms": 7589, + "review": 7590, + "crown": 7591, + "attached": 7592, + "too": 7593, + "television": 7594, + "jungle": 7595, + "22": 7596, + "fruits": 7597, + "##erence": 7598, + "come": 7599, + "profess": 7600, + "college": 7601, + "##oming": 7602, + "py": 7603, + "ins": 7604, + "aven": 7605, + "speed": 7606, + "##oned": 7607, + "radio": 7608, + "august": 7609, + "university": 7610, + "##quet": 7611, + "##osaur": 7612, + "##itation": 7613, + "##pos": 7614, + "feathers": 7615, + "warm": 7616, + "personal": 7617, + "sele": 7618, + "stairs": 7619, + "tut": 7620, + "australian": 7621, + "landscap": 7622, + "idea": 7623, + "##ged": 7624, + "bou": 7625, + "##dge": 7626, + "without": 7627, + "ul": 7628, + "yoga": 7629, + "delic": 7630, + "195": 7631, + "hus": 7632, + "##gers": 7633, + "ingred": 7634, + "final": 7635, + "##back": 7636, + "shadow": 7637, + "magn": 7638, + "creatures": 7639, + "##ided": 7640, + "sa": 7641, + "satur": 7642, + "ko": 7643, + "rele": 7644, + "##wich": 7645, + "cupc": 7646, + "president": 7647, + "climb": 7648, + "environ": 7649, + "age": 7650, + "sneakers": 7651, + "pract": 7652, + "appar": 7653, + "distr": 7654, + "led": 7655, + "carri": 7656, + "spots": 7657, + "states": 7658, + "##keys": 7659, + "guy": 7660, + "egypt": 7661, + "finger": 7662, + "oper": 7663, + "block": 7664, + "germany": 7665, + "think": 7666, + "got": 7667, + "batman": 7668, + "mat": 7669, + "pit": 7670, + "prepar": 7671, + "actress": 7672, + "apartments": 7673, + "diagram": 7674, + "##life": 7675, + "row": 7676, + "##load": 7677, + "example": 7678, + "stack": 7679, + "##mark": 7680, + "##oph": 7681, + "fre": 7682, + "media": 7683, + "urban": 7684, + "label": 7685, + "bull": 7686, + "##wear": 7687, + "fast": 7688, + "host": 7689, + "research": 7690, + "industrial": 7691, + "div": 7692, + "##erv": 7693, + "discover": 7694, + "pile": 7695, + "branches": 7696, + "salt": 7697, + "##erc": 7698, + "hospital": 7699, + "give": 7700, + "##cted": 7701, + "##irus": 7702, + "##anger": 7703, + "knight": 7704, + "cliff": 7705, + "balls": 7706, + "saturday": 7707, + "test": 7708, + "altern": 7709, + "range": 7710, + "york": 7711, + "plus": 7712, + "emer": 7713, + "deer": 7714, + "called": 7715, + "thank": 7716, + "mor": 7717, + "sunday": 7718, + "concert": 7719, + "version": 7720, + "events": 7721, + "23": 7722, + "png": 7723, + "designer": 7724, + "##ires": 7725, + "lens": 7726, + "40": 7727, + "shr": 7728, + "tracks": 7729, + "embro": 7730, + "leash": 7731, + "deep": 7732, + "pants": 7733, + "ink": 7734, + "did": 7735, + "entry": 7736, + "kiss": 7737, + "support": 7738, + "lege": 7739, + "social": 7740, + "environment": 7741, + "launch": 7742, + "##ians": 7743, + "pear": 7744, + "##ward": 7745, + "ot": 7746, + "dancing": 7747, + "##ka": 7748, + "watches": 7749, + "##amic": 7750, + "date": 7751, + "rul": 7752, + "aesth": 7753, + "stands": 7754, + "eggs": 7755, + "##zer": 7756, + "ll": 7757, + "exerc": 7758, + "architect": 7759, + "candy": 7760, + "auth": 7761, + "because": 7762, + "trib": 7763, + "aer": 7764, + "##ception": 7765, + "banner": 7766, + "reflection": 7767, + "##ners": 7768, + "fashioned": 7769, + "ram": 7770, + "appl": 7771, + "valent": 7772, + "device": 7773, + "##oot": 7774, + "september": 7775, + "based": 7776, + "##hop": 7777, + "##ula": 7778, + "quar": 7779, + "distance": 7780, + "musical": 7781, + "musc": 7782, + "##ize": 7783, + "ornam": 7784, + "fairy": 7785, + "pic": 7786, + "##den": 7787, + "nut": 7788, + "instruments": 7789, + "bang": 7790, + "terra": 7791, + "tile": 7792, + "##zzle": 7793, + "element": 7794, + "link": 7795, + "jumping": 7796, + "teen": 7797, + "planet": 7798, + "buff": 7799, + "cred": 7800, + "ru": 7801, + "farmhouse": 7802, + "minimal": 7803, + "models": 7804, + "common": 7805, + "##rab": 7806, + "contin": 7807, + "galax": 7808, + "guard": 7809, + "todd": 7810, + "chicago": 7811, + "includes": 7812, + "poses": 7813, + "monkeys": 7814, + "program": 7815, + "##mit": 7816, + "##wor": 7817, + "##imate": 7818, + "##thing": 7819, + "sticking": 7820, + "relax": 7821, + "collar": 7822, + "und": 7823, + "daughter": 7824, + "website": 7825, + "##ford": 7826, + "ju": 7827, + "acry": 7828, + "crowd": 7829, + "rh": 7830, + "##lands": 7831, + "dub": 7832, + "ult": 7833, + "##ases": 7834, + "##bage": 7835, + "details": 7836, + "six": 7837, + "stay": 7838, + "##ffic": 7839, + "autom": 7840, + "cold": 7841, + "wildlife": 7842, + "county": 7843, + "paw": 7844, + "pengu": 7845, + "instrument": 7846, + "below": 7847, + "sour": 7848, + "transpar": 7849, + "grad": 7850, + "menu": 7851, + "technology": 7852, + "anat": 7853, + "import": 7854, + "las": 7855, + "edition": 7856, + "scarf": 7857, + "theater": 7858, + "##iers": 7859, + "sheph": 7860, + "hole": 7861, + "balloon": 7862, + "spirit": 7863, + "mean": 7864, + "dinosaur": 7865, + "sad": 7866, + "aesthetic": 7867, + "rendering": 7868, + "##ycles": 7869, + "##fort": 7870, + "thing": 7871, + "##icle": 7872, + "tar": 7873, + "turned": 7874, + "meal": 7875, + "##ha": 7876, + "socks": 7877, + "hab": 7878, + "##bra": 7879, + "##ually": 7880, + "##mon": 7881, + "closed": 7882, + "father": 7883, + "surrounded": 7884, + "##ined": 7885, + "foll": 7886, + "nose": 7887, + "aband": 7888, + "##wh": 7889, + "##icks": 7890, + "premium": 7891, + "adventure": 7892, + "##esting": 7893, + "better": 7894, + "order": 7895, + "medie": 7896, + "express": 7897, + "pock": 7898, + "inform": 7899, + "bit": 7900, + "medieval": 7901, + "##ently": 7902, + "##cel": 7903, + "fitness": 7904, + "##ien": 7905, + "stylish": 7906, + "apparel": 7907, + "octo": 7908, + "why": 7909, + "focus": 7910, + "##oop": 7911, + "##ased": 7912, + "hay": 7913, + "october": 7914, + "cotton": 7915, + "tutorial": 7916, + "##boy": 7917, + "wing": 7918, + "##berries": 7919, + "parts": 7920, + "acrylic": 7921, + "broken": 7922, + "candle": 7923, + "abandoned": 7924, + "lip": 7925, + "##ining": 7926, + "wonder": 7927, + "january": 7928, + "striped": 7929, + "cy": 7930, + "adv": 7931, + "maj": 7932, + "self": 7933, + "eastern": 7934, + "febr": 7935, + "february": 7936, + "ax": 7937, + "liqu": 7938, + "rocky": 7939, + "sunny": 7940, + "suc": 7941, + "private": 7942, + "official": 7943, + "shelves": 7944, + "porch": 7945, + "##self": 7946, + "max": 7947, + "rear": 7948, + "##gr": 7949, + "really": 7950, + "talking": 7951, + "pend": 7952, + "##acks": 7953, + "begin": 7954, + "cups": 7955, + "greek": 7956, + "orch": 7957, + "handmade": 7958, + "rece": 7959, + "annual": 7960, + "bron": 7961, + "members": 7962, + "galaxy": 7963, + "hit": 7964, + "##ony": 7965, + "spa": 7966, + "transparent": 7967, + "beard": 7968, + "##ises": 7969, + "watching": 7970, + "##ray": 7971, + "structure": 7972, + "lettering": 7973, + "ride": 7974, + "grow": 7975, + "border": 7976, + "##erts": 7977, + "mass": 7978, + "##ancy": 7979, + "jar": 7980, + "graduation": 7981, + "lord": 7982, + "bol": 7983, + "earr": 7984, + "##da": 7985, + "##can": 7986, + "finish": 7987, + "sits": 7988, + "scr": 7989, + "##cot": 7990, + "bi": 7991, + "try": 7992, + "such": 7993, + "fort": 7994, + "skate": 7995, + "credit": 7996, + "change": 7997, + "anatomy": 7998, + "senior": 7999, + "always": 8000, + "towel": 8001, + "award": 8002, + "traffic": 8003, + "atl": 8004, + "##clos": 8005, + "downtown": 8006, + "doctor": 8007, + "onto": 8008, + "spl": 8009, + "marine": 8010, + "fu": 8011, + "battle": 8012, + "27": 8013, + "##na": 8014, + "ingredients": 8015, + "lam": 8016, + "##ected": 8017, + "lid": 8018, + "##oe": 8019, + "works": 8020, + "november": 8021, + "##ockey": 8022, + "blood": 8023, + "##ws": 8024, + "##10": 8025, + "dc": 8026, + "rope": 8027, + "tag": 8028, + "santa": 8029, + "sort": 8030, + "letters": 8031, + "breed": 8032, + "trends": 8033, + "those": 8034, + "whole": 8035, + "smile": 8036, + "among": 8037, + "squash": 8038, + "##ware": 8039, + "gymn": 8040, + "cher": 8041, + "secret": 8042, + "shepherd": 8043, + "complete": 8044, + "##ipp": 8045, + "graff": 8046, + "##ae": 8047, + "##ume": 8048, + "aff": 8049, + "success": 8050, + "created": 8051, + "##field": 8052, + "comes": 8053, + "shoot": 8054, + "chang": 8055, + "league": 8056, + "bubble": 8057, + "friendly": 8058, + "yarn": 8059, + "##ization": 8060, + "reception": 8061, + "romantic": 8062, + "ii": 8063, + "##son": 8064, + "storm": 8065, + "quest": 8066, + "cocktail": 8067, + "##otic": 8068, + "celebrity": 8069, + "grill": 8070, + "treat": 8071, + "welc": 8072, + "##mber": 8073, + "whale": 8074, + "28": 8075, + "arab": 8076, + "cruise": 8077, + "##bles": 8078, + "tan": 8079, + "##thouse": 8080, + "nam": 8081, + "flash": 8082, + "climbing": 8083, + "colla": 8084, + "property": 8085, + "monitor": 8086, + "##ergy": 8087, + "26": 8088, + "solar": 8089, + "bears": 8090, + "flooring": 8091, + "decorating": 8092, + "unit": 8093, + "written": 8094, + "enclos": 8095, + "graffiti": 8096, + "##my": 8097, + "speak": 8098, + "entertain": 8099, + "repres": 8100, + "doc": 8101, + "occ": 8102, + "homemade": 8103, + "marble": 8104, + "fluff": 8105, + "trim": 8106, + "##nic": 8107, + "length": 8108, + "medium": 8109, + "quick": 8110, + "alm": 8111, + "kingdom": 8112, + "stripes": 8113, + "hi": 8114, + "##ola": 8115, + "waterf": 8116, + "sketches": 8117, + "decorative": 8118, + "pitch": 8119, + "vegan": 8120, + "cass": 8121, + "waves": 8122, + "copy": 8123, + "button": 8124, + "printer": 8125, + "sound": 8126, + "bikes": 8127, + "once": 8128, + "##iest": 8129, + "chap": 8130, + "contest": 8131, + "orang": 8132, + "##ography": 8133, + "estate": 8134, + "tiles": 8135, + "dom": 8136, + "##iled": 8137, + "##nded": 8138, + "ly": 8139, + "opening": 8140, + "swing": 8141, + "##olution": 8142, + "cloud": 8143, + "bring": 8144, + "parrot": 8145, + "learning": 8146, + "sailing": 8147, + "species": 8148, + "##ription": 8149, + "takes": 8150, + "habit": 8151, + "lost": 8152, + "##etti": 8153, + "##sion": 8154, + "trunk": 8155, + "cathed": 8156, + "allig": 8157, + "refr": 8158, + "screens": 8159, + "late": 8160, + "##cer": 8161, + "gh": 8162, + "##ety": 8163, + "ol": 8164, + "coach": 8165, + "ways": 8166, + "fluffy": 8167, + "##sters": 8168, + "acad": 8169, + "##view": 8170, + "viol": 8171, + "suppl": 8172, + "engagement": 8173, + "2011": 8174, + "stitch": 8175, + "mr": 8176, + "gla": 8177, + "wre": 8178, + "cabinets": 8179, + "coron": 8180, + "hyd": 8181, + "viny": 8182, + "spread": 8183, + "##agon": 8184, + "barrel": 8185, + "transform": 8186, + "bene": 8187, + "mud": 8188, + "cathedral": 8189, + "##vet": 8190, + "decorated": 8191, + "panel": 8192, + "000": 8193, + "packaging": 8194, + "european": 8195, + "rare": 8196, + "##ective": 8197, + "dro": 8198, + "bars": 8199, + "vinyl": 8200, + "gradient": 8201, + "amb": 8202, + "district": 8203, + "##50": 8204, + "gro": 8205, + "transport": 8206, + "mexico": 8207, + "brass": 8208, + "pasta": 8209, + "mont": 8210, + "aerial": 8211, + "##tar": 8212, + "slu": 8213, + "dece": 8214, + "##mar": 8215, + "##atory": 8216, + "mermaid": 8217, + "balcon": 8218, + "advertising": 8219, + "##cut": 8220, + "witch": 8221, + "scale": 8222, + "denim": 8223, + "##ensive": 8224, + "##ift": 8225, + "delicious": 8226, + "syd": 8227, + "gymnast": 8228, + "bunny": 8229, + "largest": 8230, + "academ": 8231, + "reads": 8232, + "hockey": 8233, + "general": 8234, + "font": 8235, + "landscaping": 8236, + "ther": 8237, + "trailer": 8238, + "bible": 8239, + "appro": 8240, + "##istan": 8241, + "culture": 8242, + "mexican": 8243, + "businessman": 8244, + "##ora": 8245, + "mant": 8246, + "sav": 8247, + "graphics": 8248, + "dishes": 8249, + "journ": 8250, + "weather": 8251, + "brain": 8252, + "sydney": 8253, + "suits": 8254, + "collect": 8255, + "comfort": 8256, + "ren": 8257, + "neon": 8258, + "education": 8259, + "belt": 8260, + "flags": 8261, + "course": 8262, + "dy": 8263, + "##ills": 8264, + "##aste": 8265, + "##erve": 8266, + "pics": 8267, + "pendant": 8268, + "honey": 8269, + "ribbon": 8270, + "said": 8271, + "boh": 8272, + "steps": 8273, + "performing": 8274, + "##ired": 8275, + "information": 8276, + "victor": 8277, + "lou": 8278, + "marri": 8279, + "##imp": 8280, + "##icket": 8281, + "formal": 8282, + "views": 8283, + "prince": 8284, + "camping": 8285, + "microphone": 8286, + "energy": 8287, + "nest": 8288, + "fans": 8289, + "services": 8290, + "pir": 8291, + "dol": 8292, + "pine": 8293, + "##ios": 8294, + "rich": 8295, + "lighthouse": 8296, + "foreground": 8297, + "mond": 8298, + "ceramic": 8299, + "december": 8300, + "29": 8301, + "gowns": 8302, + "ladies": 8303, + "warr": 8304, + "ip": 8305, + "##orean": 8306, + "hats": 8307, + "resp": 8308, + "album": 8309, + "lunch": 8310, + "former": 8311, + "symbols": 8312, + "plates": 8313, + "##ilar": 8314, + "cowboy": 8315, + "holds": 8316, + "blo": 8317, + "study": 8318, + "blur": 8319, + "hills": 8320, + "heavy": 8321, + "strong": 8322, + "partic": 8323, + "clown": 8324, + "tee": 8325, + "##aters": 8326, + "pipe": 8327, + "geometric": 8328, + "tray": 8329, + "mand": 8330, + "nursery": 8331, + "rub": 8332, + "##stone": 8333, + "names": 8334, + "covers": 8335, + "foods": 8336, + "trash": 8337, + "##itter": 8338, + "scor": 8339, + "safety": 8340, + "meeting": 8341, + "cann": 8342, + "knee": 8343, + "##ctions": 8344, + "alligator": 8345, + "earrings": 8346, + "##utes": 8347, + "drop": 8348, + "footage": 8349, + "##esome": 8350, + "##writer": 8351, + "cry": 8352, + "goat": 8353, + "exhibition": 8354, + "mind": 8355, + "tooth": 8356, + "ps": 8357, + "resid": 8358, + "thurs": 8359, + "alph": 8360, + "pets": 8361, + "jell": 8362, + "took": 8363, + "ble": 8364, + "thursday": 8365, + "##aught": 8366, + "printed": 8367, + "job": 8368, + "smoke": 8369, + "needs": 8370, + "manga": 8371, + "outline": 8372, + "##gn": 8373, + "videos": 8374, + "2010": 8375, + "roses": 8376, + "##vern": 8377, + "born": 8378, + "strawberry": 8379, + "pac": 8380, + "price": 8381, + "##burg": 8382, + "pier": 8383, + "ha": 8384, + "##cent": 8385, + "gorge": 8386, + "framed": 8387, + "turkey": 8388, + "resc": 8389, + "##elt": 8390, + "fa": 8391, + "##oz": 8392, + "objects": 8393, + "cleaning": 8394, + "throw": 8395, + "irish": 8396, + "sain": 8397, + "journal": 8398, + "dough": 8399, + "##lex": 8400, + "spain": 8401, + "trendy": 8402, + "##yer": 8403, + "monster": 8404, + "ox": 8405, + "##ered": 8406, + "nike": 8407, + "desserts": 8408, + "stainless": 8409, + "##ressed": 8410, + "trump": 8411, + "scoot": 8412, + "variety": 8413, + "wednes": 8414, + "northern": 8415, + "bass": 8416, + "wednesday": 8417, + "hold": 8418, + "tues": 8419, + "balcony": 8420, + "korean": 8421, + "snail": 8422, + "pow": 8423, + "chef": 8424, + "tuesday": 8425, + "##rise": 8426, + "##agh": 8427, + "load": 8428, + "mine": 8429, + "jet": 8430, + "rent": 8431, + "pictured": 8432, + "armor": 8433, + "awesome": 8434, + "##ota": 8435, + "##band": 8436, + "cave": 8437, + "requ": 8438, + "broc": 8439, + "streets": 8440, + "medic": 8441, + "arrow": 8442, + "puzzle": 8443, + "factory": 8444, + "matt": 8445, + "enjo": 8446, + "##ploy": 8447, + "russian": 8448, + "laugh": 8449, + "curtains": 8450, + "civ": 8451, + "happ": 8452, + "performance": 8453, + "bracelet": 8454, + "monday": 8455, + "candles": 8456, + "##tub": 8457, + "ped": 8458, + "southern": 8459, + "bouquet": 8460, + "chall": 8461, + "##esty": 8462, + "kill": 8463, + "pepper": 8464, + "professional": 8465, + "islands": 8466, + "moss": 8467, + "budget": 8468, + "meet": 8469, + "vegetable": 8470, + "bowls": 8471, + "olympic": 8472, + "dem": 8473, + "animation": 8474, + "pa": 8475, + "##ount": 8476, + "##rodu": 8477, + "count": 8478, + "vel": 8479, + "sets": 8480, + "spiral": 8481, + "horiz": 8482, + "poin": 8483, + "hon": 8484, + "since": 8485, + "tap": 8486, + "##ession": 8487, + "silk": 8488, + "enclosure": 8489, + "legend": 8490, + "closet": 8491, + "experience": 8492, + "shorts": 8493, + "carrying": 8494, + "pix": 8495, + "straight": 8496, + "million": 8497, + "blond": 8498, + "commer": 8499, + "bucket": 8500, + "##acy": 8501, + "shade": 8502, + "horror": 8503, + "feeding": 8504, + "platform": 8505, + "unic": 8506, + "competition": 8507, + "##eter": 8508, + "relig": 8509, + "staff": 8510, + "##avirus": 8511, + "##ager": 8512, + "rome": 8513, + "liquid": 8514, + "development": 8515, + "cape": 8516, + "ske": 8517, + "coronavirus": 8518, + "dolls": 8519, + "god": 8520, + "pocket": 8521, + "habitat": 8522, + "slow": 8523, + "##po": 8524, + "goth": 8525, + "##ified": 8526, + "##ctic": 8527, + "sus": 8528, + "##sw": 8529, + "vs": 8530, + "wish": 8531, + "loun": 8532, + "##ivid": 8533, + "fict": 8534, + "sizes": 8535, + "listed": 8536, + "prim": 8537, + "##ald": 8538, + "humor": 8539, + "huge": 8540, + "##chool": 8541, + "article": 8542, + "hed": 8543, + "typewriter": 8544, + "wave": 8545, + "notebook": 8546, + "cig": 8547, + "jok": 8548, + "faces": 8549, + "pineap": 8550, + "everything": 8551, + "die": 8552, + "runway": 8553, + "touch": 8554, + "##onto": 8555, + "sunrise": 8556, + "wife": 8557, + "193": 8558, + "##che": 8559, + "##abet": 8560, + "##asing": 8561, + "closeup": 8562, + "hug": 8563, + "##une": 8564, + "pupp": 8565, + "##chester": 8566, + "##azz": 8567, + "solid": 8568, + "cherry": 8569, + "waiting": 8570, + "ut": 8571, + "##itage": 8572, + "spagh": 8573, + "boston": 8574, + "apples": 8575, + "##ension": 8576, + "section": 8577, + "motiv": 8578, + "attends": 8579, + "welcome": 8580, + "steam": 8581, + "mixed": 8582, + "gorilla": 8583, + "fighting": 8584, + "layout": 8585, + "represent": 8586, + "victorian": 8587, + "sar": 8588, + "native": 8589, + "gather": 8590, + "extra": 8591, + "##oves": 8592, + "##eling": 8593, + "wet": 8594, + "##ua": 8595, + "##phones": 8596, + "nap": 8597, + "return": 8598, + "hind": 8599, + "heritage": 8600, + "toronto": 8601, + "seating": 8602, + "##uk": 8603, + "seeds": 8604, + "via": 8605, + "parties": 8606, + "flav": 8607, + "haw": 8608, + "##ura": 8609, + "bamb": 8610, + "balloons": 8611, + "##ailand": 8612, + "##aming": 8613, + "done": 8614, + "astr": 8615, + "##pie": 8616, + "historic": 8617, + "parade": 8618, + "moving": 8619, + "pose": 8620, + "smooth": 8621, + "thailand": 8622, + "towards": 8623, + "##box": 8624, + "historical": 8625, + "pour": 8626, + "weeke": 8627, + "pearl": 8628, + "spin": 8629, + "##hol": 8630, + "ident": 8631, + "performs": 8632, + "harry": 8633, + "download": 8634, + "##80": 8635, + "proble": 8636, + "myth": 8637, + "lodge": 8638, + "vest": 8639, + "item": 8640, + "rout": 8641, + "##ba": 8642, + "major": 8643, + "##60": 8644, + "soap": 8645, + "rank": 8646, + "warrior": 8647, + "iss": 8648, + "##end": 8649, + "fill": 8650, + "pil": 8651, + "slug": 8652, + "author": 8653, + "broccol": 8654, + "important": 8655, + "kay": 8656, + "rap": 8657, + "hong": 8658, + "drawers": 8659, + "seattle": 8660, + "neigh": 8661, + "unders": 8662, + "poodle": 8663, + "lan": 8664, + "tro": 8665, + "thanks": 8666, + "lobby": 8667, + "spaghetti": 8668, + "annivers": 8669, + "philipp": 8670, + "origin": 8671, + "##itive": 8672, + "##born": 8673, + "artists": 8674, + "creek": 8675, + "baking": 8676, + "backgrounds": 8677, + "##urt": 8678, + "clay": 8679, + "##yon": 8680, + "##bag": 8681, + "captain": 8682, + "bronze": 8683, + "universe": 8684, + "washing": 8685, + "unicorn": 8686, + "##ults": 8687, + "womens": 8688, + "figures": 8689, + "##ential": 8690, + "caught": 8691, + "within": 8692, + "mount": 8693, + "keys": 8694, + "robot": 8695, + "##men": 8696, + "anniversary": 8697, + "tote": 8698, + "tax": 8699, + "spanish": 8700, + "crib": 8701, + "##shire": 8702, + "pulling": 8703, + "addition": 8704, + "worn": 8705, + "nas": 8706, + "soldier": 8707, + "teacher": 8708, + "bamboo": 8709, + "bin": 8710, + "toddler": 8711, + "singap": 8712, + "##ring": 8713, + "leading": 8714, + "nar": 8715, + "illustrated": 8716, + "194": 8717, + "violin": 8718, + "31": 8719, + "stret": 8720, + "##amel": 8721, + "butt": 8722, + "franc": 8723, + "##pend": 8724, + "miniature": 8725, + "##ih": 8726, + "location": 8727, + "thought": 8728, + "exercise": 8729, + "kong": 8730, + "cheap": 8731, + "teeth": 8732, + "giving": 8733, + "slip": 8734, + "whis": 8735, + "curr": 8736, + "##enna": 8737, + "pages": 8738, + "asia": 8739, + "crane": 8740, + "seal": 8741, + "dome": 8742, + "cartoons": 8743, + "third": 8744, + "spaces": 8745, + "desktop": 8746, + "ec": 8747, + "peace": 8748, + "lobster": 8749, + "ears": 8750, + "worksh": 8751, + "ferret": 8752, + "anal": 8753, + "##ening": 8754, + "actors": 8755, + "incred": 8756, + "##unge": 8757, + "dut": 8758, + "##idays": 8759, + "singapore": 8760, + "blonde": 8761, + "##ived": 8762, + "cookie": 8763, + "pottery": 8764, + "championship": 8765, + "##hi": 8766, + "lounge": 8767, + "##igraph": 8768, + "bullet": 8769, + "##hip": 8770, + "death": 8771, + "mart": 8772, + "ships": 8773, + "trian": 8774, + "##estyle": 8775, + "atm": 8776, + "gorgeous": 8777, + "commercial": 8778, + "barbie": 8779, + "pots": 8780, + "##lets": 8781, + "reas": 8782, + "entertainment": 8783, + "bes": 8784, + "calligraph": 8785, + "mural": 8786, + "dad": 8787, + "desc": 8788, + "repair": 8789, + "benef": 8790, + "##iac": 8791, + "cyc": 8792, + "snowy": 8793, + "tul": 8794, + "interesting": 8795, + "screensh": 8796, + "##aland": 8797, + "stones": 8798, + "purse": 8799, + "portraits": 8800, + "##stick": 8801, + "refriger": 8802, + "chop": 8803, + "scotland": 8804, + "##iced": 8805, + "baked": 8806, + "won": 8807, + "jeep": 8808, + "##lus": 8809, + "##rid": 8810, + "preschool": 8811, + "poly": 8812, + "facts": 8813, + "celebration": 8814, + "dip": 8815, + "dail": 8816, + "cigar": 8817, + "budd": 8818, + "##lim": 8819, + "quarter": 8820, + "holidays": 8821, + "eth": 8822, + "adop": 8823, + "soldiers": 8824, + "pav": 8825, + "valentine": 8826, + "unt": 8827, + "served": 8828, + "grassy": 8829, + "daily": 8830, + "session": 8831, + "ma": 8832, + "spee": 8833, + "##iven": 8834, + "enjoying": 8835, + "##ige": 8836, + "feather": 8837, + "cocon": 8838, + "##oles": 8839, + "wool": 8840, + "##anas": 8841, + "offers": 8842, + "carved": 8843, + "mascot": 8844, + "francis": 8845, + "pay": 8846, + "fat": 8847, + "celebrate": 8848, + "buttons": 8849, + "trou": 8850, + "cable": 8851, + "##obile": 8852, + "talk": 8853, + "true": 8854, + "telephone": 8855, + "capital": 8856, + "might": 8857, + "00": 8858, + "fung": 8859, + "##bies": 8860, + "skyline": 8861, + "lower": 8862, + "hearts": 8863, + "ultimate": 8864, + "themed": 8865, + "volley": 8866, + "zealand": 8867, + "##idery": 8868, + "winning": 8869, + "embroidery": 8870, + "esc": 8871, + "inch": 8872, + "globe": 8873, + "getty": 8874, + "motorcycles": 8875, + "pak": 8876, + "named": 8877, + "walks": 8878, + "froz": 8879, + "##arian": 8880, + "sem": 8881, + "empire": 8882, + "beads": 8883, + "spotted": 8884, + "##oise": 8885, + "sword": 8886, + "frozen": 8887, + "compos": 8888, + "weekend": 8889, + "##idden": 8890, + "added": 8891, + "shapes": 8892, + "waterfall": 8893, + "lovel": 8894, + "mosque": 8895, + "buffal": 8896, + "doodle": 8897, + "stickers": 8898, + "cupcakes": 8899, + "distressed": 8900, + "##ado": 8901, + "strawberries": 8902, + "er": 8903, + "individ": 8904, + "coin": 8905, + "stunning": 8906, + "driver": 8907, + "goal": 8908, + "beat": 8909, + "bedrooms": 8910, + "tape": 8911, + "singer": 8912, + "guys": 8913, + "perio": 8914, + "rolls": 8915, + "pean": 8916, + "esp": 8917, + "attract": 8918, + "ducks": 8919, + "crew": 8920, + "##ettes": 8921, + "alternate": 8922, + "alu": 8923, + "##iques": 8924, + "explor": 8925, + "hond": 8926, + "yach": 8927, + "##ply": 8928, + "poss": 8929, + "clipart": 8930, + "surv": 8931, + "cinem": 8932, + "multip": 8933, + "glow": 8934, + "##oms": 8935, + "##ording": 8936, + "ranked": 8937, + "coconut": 8938, + "gir": 8939, + "topped": 8940, + "##urity": 8941, + "##ifle": 8942, + "outdoors": 8943, + "speech": 8944, + "vert": 8945, + "alphabet": 8946, + "fra": 8947, + "wid": 8948, + "resident": 8949, + "batter": 8950, + "document": 8951, + "facade": 8952, + "2009": 8953, + "fight": 8954, + "fed": 8955, + "adj": 8956, + "remo": 8957, + "hell": 8958, + "##inder": 8959, + "invitation": 8960, + "glitter": 8961, + "caul": 8962, + "drums": 8963, + "##pes": 8964, + "impress": 8965, + "##ples": 8966, + "gloves": 8967, + "themes": 8968, + "nuts": 8969, + "russia": 8970, + "hedge": 8971, + "langu": 8972, + "##ucum": 8973, + "calligraphy": 8974, + "lovely": 8975, + "cheet": 8976, + "pacific": 8977, + "da": 8978, + "hop": 8979, + "volleyball": 8980, + "become": 8981, + "upper": 8982, + "sure": 8983, + "source": 8984, + "ket": 8985, + "##ators": 8986, + "toky": 8987, + "##semb": 8988, + "conference": 8989, + "quilt": 8990, + "##ences": 8991, + "deco": 8992, + "##ex": 8993, + "inspirational": 8994, + "fau": 8995, + "height": 8996, + "repub": 8997, + "potatoes": 8998, + "##idered": 8999, + "tokyo": 9000, + "ire": 9001, + "lifestyle": 9002, + "##hog": 9003, + "##ords": 9004, + "safe": 9005, + "kitten": 9006, + "suite": 9007, + "share": 9008, + "seven": 9009, + "prob": 9010, + "foss": 9011, + "##aged": 9012, + "cord": 9013, + "sac": 9014, + "position": 9015, + "ireland": 9016, + "bees": 9017, + "dash": 9018, + "pakistan": 9019, + "phys": 9020, + "reve": 9021, + "govern": 9022, + "mall": 9023, + "material": 9024, + "##oof": 9025, + "organic": 9026, + "##lem": 9027, + "##agu": 9028, + "record": 9029, + "selling": 9030, + "pregn": 9031, + "weddings": 9032, + "avengers": 9033, + "sciss": 9034, + "ben": 9035, + "eld": 9036, + "moment": 9037, + "republic": 9038, + "##ode": 9039, + "hunting": 9040, + "orn": 9041, + "feature": 9042, + "que": 9043, + "demon": 9044, + "fold": 9045, + "##time": 9046, + "direct": 9047, + "yourself": 9048, + "trying": 9049, + "##andel": 9050, + "bout": 9051, + "holder": 9052, + "stem": 9053, + "##uf": 9054, + "dubai": 9055, + "scooter": 9056, + "kee": 9057, + "stream": 9058, + "tas": 9059, + "praying": 9060, + "controll": 9061, + "jackets": 9062, + "ghost": 9063, + "braz": 9064, + "rose": 9065, + "##chan": 9066, + "peac": 9067, + "##elds": 9068, + "flaming": 9069, + "security": 9070, + "beef": 9071, + "knitting": 9072, + "zip": 9073, + "soul": 9074, + "##boat": 9075, + "carn": 9076, + "beast": 9077, + "workshop": 9078, + "##ya": 9079, + "bon": 9080, + "planning": 9081, + "##gles": 9082, + "mead": 9083, + "production": 9084, + "##mes": 9085, + "civil": 9086, + "fighter": 9087, + "brides": 9088, + "60": 9089, + "wallet": 9090, + "monument": 9091, + "period": 9092, + "employ": 9093, + "##leton": 9094, + "interiors": 9095, + "enc": 9096, + "##irty": 9097, + "teaching": 9098, + "carib": 9099, + "renov": 9100, + "feeder": 9101, + "officer": 9102, + "philippines": 9103, + "mounted": 9104, + "barber": 9105, + "fossil": 9106, + "pastel": 9107, + "##enger": 9108, + "saw": 9109, + "maker": 9110, + "report": 9111, + "muscle": 9112, + "mur": 9113, + "unus": 9114, + "glam": 9115, + "crossing": 9116, + "islam": 9117, + "slice": 9118, + "##ront": 9119, + "helic": 9120, + "pillows": 9121, + "powder": 9122, + "##aking": 9123, + "prep": 9124, + "wreath": 9125, + "hook": 9126, + "impro": 9127, + "choose": 9128, + "##zing": 9129, + "math": 9130, + "lau": 9131, + "pineapple": 9132, + "##cing": 9133, + "push": 9134, + "wrist": 9135, + "industry": 9136, + "attend": 9137, + "ok": 9138, + "rescue": 9139, + "grasshop": 9140, + "staircase": 9141, + "brazil": 9142, + "##ships": 9143, + "convers": 9144, + "department": 9145, + "cricket": 9146, + "##cope": 9147, + "rise": 9148, + "francisco": 9149, + "##kee": 9150, + "tomatoes": 9151, + "nearby": 9152, + "viet": 9153, + "jelly": 9154, + "tops": 9155, + "elephants": 9156, + "crazy": 9157, + "fancy": 9158, + "memorial": 9159, + "rod": 9160, + "dragonfly": 9161, + "heads": 9162, + "roast": 9163, + "##ii": 9164, + "helicop": 9165, + "spark": 9166, + "exotic": 9167, + "explore": 9168, + "feb": 9169, + "region": 9170, + "singing": 9171, + "scenes": 9172, + "celebrities": 9173, + "placed": 9174, + "##bean": 9175, + "ver": 9176, + "wrapped": 9177, + "shutters": 9178, + "harbor": 9179, + "##ny": 9180, + "opened": 9181, + "meter": 9182, + "realistic": 9183, + "journey": 9184, + "tube": 9185, + "ranch": 9186, + "network": 9187, + "##ivals": 9188, + "##ees": 9189, + "35": 9190, + "peppers": 9191, + "canadian": 9192, + "##ij": 9193, + "diet": 9194, + "minimalist": 9195, + "##12": 9196, + "##ication": 9197, + "hob": 9198, + "academy": 9199, + "relation": 9200, + "honda": 9201, + "write": 9202, + "wonderful": 9203, + "blouse": 9204, + "mode": 9205, + "bloss": 9206, + "switch": 9207, + "meaning": 9208, + "obs": 9209, + "vietnam": 9210, + "photographic": 9211, + "tell": 9212, + "auto": 9213, + "##erk": 9214, + "burning": 9215, + "cooked": 9216, + "##ersey": 9217, + "engra": 9218, + "gaz": 9219, + "koala": 9220, + "##thers": 9221, + "shells": 9222, + "chips": 9223, + "chains": 9224, + "##ivery": 9225, + "bacon": 9226, + "buffalo": 9227, + "orient": 9228, + "ward": 9229, + "profile": 9230, + "##stic": 9231, + "##ales": 9232, + "seed": 9233, + "rolling": 9234, + "angle": 9235, + "cucum": 9236, + "cities": 9237, + "##ule": 9238, + "bushes": 9239, + "measuring": 9240, + "bm": 9241, + "thin": 9242, + "rugby": 9243, + "canyon": 9244, + "standard": 9245, + "##nut": 9246, + "##ivity": 9247, + "##uba": 9248, + "message": 9249, + "ruler": 9250, + "laund": 9251, + "##cm": 9252, + "##where": 9253, + "bean": 9254, + "##iders": 9255, + "practice": 9256, + "venice": 9257, + "supplies": 9258, + "juice": 9259, + "cush": 9260, + "caribbean": 9261, + "yet": 9262, + "beige": 9263, + "mail": 9264, + "gets": 9265, + "copper": 9266, + "trumpet": 9267, + "##gar": 9268, + "command": 9269, + "velvet": 9270, + "##uten": 9271, + "frost": 9272, + "stained": 9273, + "floors": 9274, + "chalk": 9275, + "shining": 9276, + "cheer": 9277, + "pens": 9278, + "##oodles": 9279, + "ny": 9280, + "dug": 9281, + "tomato": 9282, + "chip": 9283, + "alco": 9284, + "soil": 9285, + "##war": 9286, + "heat": 9287, + "balance": 9288, + "bern": 9289, + "handles": 9290, + "rig": 9291, + "global": 9292, + "##ulation": 9293, + "##ano": 9294, + "cabbage": 9295, + "gree": 9296, + "internet": 9297, + "limited": 9298, + "connect": 9299, + "vertical": 9300, + "##iper": 9301, + "laundry": 9302, + "lemur": 9303, + "const": 9304, + "waters": 9305, + "ten": 9306, + "sticks": 9307, + "hammer": 9308, + "photographed": 9309, + "vectors": 9310, + "potato": 9311, + "##ately": 9312, + "kar": 9313, + "penguin": 9314, + "ta": 9315, + "bananas": 9316, + "##eless": 9317, + "data": 9318, + "##pool": 9319, + "##arch": 9320, + "##fla": 9321, + "chev": 9322, + "##look": 9323, + "superh": 9324, + "airl": 9325, + "offer": 9326, + "conceptual": 9327, + "visual": 9328, + "jersey": 9329, + "bubbles": 9330, + "elder": 9331, + "##forest": 9332, + "tel": 9333, + "arc": 9334, + "##med": 9335, + "arctic": 9336, + "retrie": 9337, + "boutique": 9338, + "newt": 9339, + "greece": 9340, + "hiking": 9341, + "rainforest": 9342, + "railway": 9343, + "basement": 9344, + "cit": 9345, + "joy": 9346, + "compass": 9347, + "machines": 9348, + "scorp": 9349, + "masks": 9350, + "ank": 9351, + "strap": 9352, + "guests": 9353, + "##kins": 9354, + "polish": 9355, + "clar": 9356, + "snap": 9357, + "panels": 9358, + "belie": 9359, + "##rib": 9360, + "athlet": 9361, + "ambul": 9362, + "gymnastics": 9363, + "audio": 9364, + "ken": 9365, + "smartphone": 9366, + "lipstick": 9367, + "grunge": 9368, + "hotels": 9369, + "sunfl": 9370, + "merry": 9371, + "grasshopper": 9372, + "thous": 9373, + "##ites": 9374, + "##oos": 9375, + "holy": 9376, + "##ims": 9377, + "##car": 9378, + "thousand": 9379, + "##phy": 9380, + "cloudy": 9381, + "ple": 9382, + "featured": 9383, + "hope": 9384, + "weap": 9385, + "films": 9386, + "complex": 9387, + "superhero": 9388, + "dolph": 9389, + "bun": 9390, + "marketing": 9391, + "falls": 9392, + "ostr": 9393, + "terrace": 9394, + "pug": 9395, + "##soci": 9396, + "coy": 9397, + "angry": 9398, + "bob": 9399, + "##frie": 9400, + "shed": 9401, + "separ": 9402, + "activity": 9403, + "finished": 9404, + "snakes": 9405, + "deal": 9406, + "husky": 9407, + "rifle": 9408, + "associ": 9409, + "chickens": 9410, + "2008": 9411, + "pant": 9412, + "mist": 9413, + "beans": 9414, + "picnic": 9415, + "hou": 9416, + "putting": 9417, + "maps": 9418, + "jewell": 9419, + "##ester": 9420, + "click": 9421, + "##bourne": 9422, + "screenshot": 9423, + "sell": 9424, + "peacock": 9425, + "##use": 9426, + "##oman": 9427, + "sides": 9428, + "sandals": 9429, + "caps": 9430, + "breath": 9431, + "grade": 9432, + "##ili": 9433, + "jagu": 9434, + "sax": 9435, + "highway": 9436, + "guin": 9437, + "rocket": 9438, + "squad": 9439, + "ruins": 9440, + "typography": 9441, + "announ": 9442, + "choice": 9443, + "larger": 9444, + "manchester": 9445, + "edit": 9446, + "management": 9447, + "delivery": 9448, + "gey": 9449, + "notes": 9450, + "##allow": 9451, + "sens": 9452, + "facing": 9453, + "toyota": 9454, + "patch": 9455, + "fog": 9456, + "otter": 9457, + "knitted": 9458, + "poll": 9459, + "pointing": 9460, + "breeds": 9461, + "incre": 9462, + "bulld": 9463, + "forward": 9464, + "pitcher": 9465, + "mechan": 9466, + "minutes": 9467, + "olive": 9468, + "sequ": 9469, + "loft": 9470, + "##uce": 9471, + "cosmet": 9472, + "mort": 9473, + "manh": 9474, + "gives": 9475, + "##iation": 9476, + "uses": 9477, + "##olic": 9478, + "plays": 9479, + "following": 9480, + "peak": 9481, + "surpr": 9482, + "##bird": 9483, + "jud": 9484, + "areas": 9485, + "vine": 9486, + "saint": 9487, + "language": 9488, + "blind": 9489, + "1920": 9490, + "arena": 9491, + "worker": 9492, + "##cast": 9493, + "##ufact": 9494, + "goes": 9495, + "lantern": 9496, + "released": 9497, + "manufact": 9498, + "chem": 9499, + "medicine": 9500, + "cozy": 9501, + "##ugg": 9502, + "alumin": 9503, + "##eral": 9504, + "camel": 9505, + "##ito": 9506, + "amazon": 9507, + "brook": 9508, + "sleeves": 9509, + "insc": 9510, + "##ali": 9511, + "rv": 9512, + "mah": 9513, + "botan": 9514, + "houston": 9515, + "lessons": 9516, + "##ella": 9517, + "##amples": 9518, + "missing": 9519, + "emerg": 9520, + "curly": 9521, + "eps": 9522, + "kor": 9523, + "move": 9524, + "plush": 9525, + "pirate": 9526, + "sor": 9527, + "cash": 9528, + "alcohol": 9529, + "column": 9530, + "puppies": 9531, + "##ission": 9532, + "ambulance": 9533, + "cale": 9534, + "materials": 9535, + "tick": 9536, + "starfish": 9537, + "individual": 9538, + "##11": 9539, + "##una": 9540, + "director": 9541, + "vib": 9542, + "##burgh": 9543, + "almost": 9544, + "na": 9545, + "cardboard": 9546, + "croco": 9547, + "foundation": 9548, + "remod": 9549, + "mint": 9550, + "dots": 9551, + "youth": 9552, + "ginger": 9553, + "raised": 9554, + "basic": 9555, + "included": 9556, + "workers": 9557, + "others": 9558, + "boar": 9559, + "collage": 9560, + "drill": 9561, + "pel": 9562, + "bake": 9563, + "shield": 9564, + "##reme": 9565, + "upon": 9566, + "vegas": 9567, + "suites": 9568, + "swan": 9569, + "gothic": 9570, + "ho": 9571, + "epis": 9572, + "pip": 9573, + "##lock": 9574, + "turqu": 9575, + "500": 9576, + "dot": 9577, + "turquoise": 9578, + "fix": 9579, + "crocod": 9580, + "helicopter": 9581, + "broad": 9582, + "motion": 9583, + "organization": 9584, + "##cul": 9585, + "worth": 9586, + "calend": 9587, + "rubber": 9588, + "smoking": 9589, + "pulled": 9590, + "similar": 9591, + "boot": 9592, + "cran": 9593, + "##tered": 9594, + "installation": 9595, + "newborn": 9596, + "creating": 9597, + "unusual": 9598, + "scott": 9599, + "egyptian": 9600, + "thick": 9601, + "expos": 9602, + "casino": 9603, + "ornate": 9604, + "tire": 9605, + "shades": 9606, + "illumin": 9607, + "##aign": 9608, + "remain": 9609, + "##inity": 9610, + "##rad": 9611, + "gran": 9612, + "overlook": 9613, + "circa": 9614, + "gluten": 9615, + "delhi": 9616, + "function": 9617, + "jewellery": 9618, + "abs": 9619, + "iconic": 9620, + "##ense": 9621, + "papers": 9622, + "##ogue": 9623, + "novel": 9624, + "came": 9625, + "iv": 9626, + "worlds": 9627, + "parents": 9628, + "igu": 9629, + "corpor": 9630, + "adjust": 9631, + "bone": 9632, + "colours": 9633, + "bulldog": 9634, + "played": 9635, + "exhibit": 9636, + "tourist": 9637, + "##chair": 9638, + "##kes": 9639, + "sloth": 9640, + "artich": 9641, + "emergency": 9642, + "##working": 9643, + "##others": 9644, + "particip": 9645, + "lean": 9646, + "removed": 9647, + "childhood": 9648, + "prop": 9649, + "humming": 9650, + "mission": 9651, + "logos": 9652, + "often": 9653, + "until": 9654, + "rusty": 9655, + "canal": 9656, + "stacked": 9657, + "##flies": 9658, + "husband": 9659, + "bum": 9660, + "handwr": 9661, + "postc": 9662, + "adopt": 9663, + "spect": 9664, + "diving": 9665, + "patient": 9666, + "##jo": 9667, + "##aven": 9668, + "##ati": 9669, + "cond": 9670, + "campaign": 9671, + "gravel": 9672, + "selection": 9673, + "minecraft": 9674, + "pun": 9675, + "fields": 9676, + "lined": 9677, + "snacks": 9678, + "neighbor": 9679, + "ash": 9680, + "sleep": 9681, + "exte": 9682, + "berries": 9683, + "ratt": 9684, + "dust": 9685, + "##ox": 9686, + "alien": 9687, + "imag": 9688, + "mosa": 9689, + "assemb": 9690, + "inde": 9691, + "berlin": 9692, + "merced": 9693, + "hung": 9694, + "introdu": 9695, + "mumb": 9696, + "reserve": 9697, + "electrical": 9698, + "##enge": 9699, + "goo": 9700, + "alone": 9701, + "##ipped": 9702, + "government": 9703, + "playground": 9704, + "##icy": 9705, + "##mm": 9706, + "excell": 9707, + "hours": 9708, + "visible": 9709, + "melbourne": 9710, + "##rant": 9711, + "tac": 9712, + "##inary": 9713, + "##aker": 9714, + "##ctive": 9715, + "yacht": 9716, + "dim": 9717, + "skeleton": 9718, + "mumbai": 9719, + "hidden": 9720, + "ke": 9721, + "chart": 9722, + "calendar": 9723, + "metall": 9724, + "union": 9725, + "liver": 9726, + "reveal": 9727, + "fif": 9728, + "##ances": 9729, + "upd": 9730, + "courty": 9731, + "submar": 9732, + "pom": 9733, + "flute": 9734, + "newsp": 9735, + "futur": 9736, + "horns": 9737, + "##onic": 9738, + "jaguar": 9739, + "evil": 9740, + "nasa": 9741, + "wears": 9742, + "landing": 9743, + "pors": 9744, + "workout": 9745, + "screw": 9746, + "fle": 9747, + "matching": 9748, + "##bu": 9749, + "cris": 9750, + "multiple": 9751, + "portug": 9752, + "##robe": 9753, + "fried": 9754, + "##40": 9755, + "columb": 9756, + "##ological": 9757, + "ray": 9758, + "coyote": 9759, + "##bury": 9760, + "member": 9761, + "dirty": 9762, + "manager": 9763, + "##drop": 9764, + "saying": 9765, + "joh": 9766, + "search": 9767, + "##isting": 9768, + "##adel": 9769, + "overhead": 9770, + "term": 9771, + "##cas": 9772, + "##ned": 9773, + "serving": 9774, + "itself": 9775, + "boxing": 9776, + "soc": 9777, + "scient": 9778, + "kim": 9779, + "due": 9780, + "wires": 9781, + "##ably": 9782, + "circular": 9783, + "bey": 9784, + "bunk": 9785, + "wagon": 9786, + "porsche": 9787, + "include": 9788, + "##friend": 9789, + "rena": 9790, + "##inations": 9791, + "kinds": 9792, + "cupcake": 9793, + "dollar": 9794, + "##rought": 9795, + "##oid": 9796, + "mercedes": 9797, + "splash": 9798, + "cinema": 9799, + "scuba": 9800, + "##ita": 9801, + "months": 9802, + "puff": 9803, + "twin": 9804, + "chewing": 9805, + "penguins": 9806, + "bmw": 9807, + "##rots": 9808, + "thai": 9809, + "crabs": 9810, + "stories": 9811, + "crack": 9812, + "aftern": 9813, + "propos": 9814, + "points": 9815, + "##ker": 9816, + "schem": 9817, + "fallen": 9818, + "garbage": 9819, + "##25": 9820, + "architectural": 9821, + "attractive": 9822, + "spray": 9823, + "emblem": 9824, + "fal": 9825, + "ads": 9826, + "ahead": 9827, + "chief": 9828, + "##erg": 9829, + "microw": 9830, + "miam": 9831, + "adm": 9832, + "babies": 9833, + "everyone": 9834, + "uniforms": 9835, + "paws": 9836, + "catch": 9837, + "slot": 9838, + "miami": 9839, + "meadow": 9840, + "kh": 9841, + "##ummy": 9842, + "speakers": 9843, + "worm": 9844, + "trophy": 9845, + "throughout": 9846, + "purch": 9847, + "mesh": 9848, + "meerk": 9849, + "accord": 9850, + "##ought": 9851, + "nur": 9852, + "pho": 9853, + "challenge": 9854, + "##ened": 9855, + "sister": 9856, + "tournam": 9857, + "##tles": 9858, + "falling": 9859, + "liter": 9860, + "afternoon": 9861, + "provide": 9862, + "##estr": 9863, + "##uction": 9864, + "millions": 9865, + "##unnel": 9866, + "terr": 9867, + "##uth": 9868, + "coastal": 9869, + "surrounding": 9870, + "runs": 9871, + "songs": 9872, + "safari": 9873, + "scorpion": 9874, + "harv": 9875, + "headphones": 9876, + "perfume": 9877, + "dv": 9878, + "benefits": 9879, + "##30": 9880, + "##kn": 9881, + "gardening": 9882, + "memories": 9883, + "inj": 9884, + "traveling": 9885, + "bathtub": 9886, + "meme": 9887, + "boho": 9888, + "##cy": 9889, + "salmon": 9890, + "broom": 9891, + "ful": 9892, + "thinking": 9893, + "later": 9894, + "wig": 9895, + "chevro": 9896, + "sci": 9897, + "guinea": 9898, + "##iser": 9899, + "rooster": 9900, + "wardrobe": 9901, + "reference": 9902, + "shor": 9903, + "chevrolet": 9904, + "detailed": 9905, + "##agen": 9906, + "##ilies": 9907, + "19th": 9908, + "lotus": 9909, + "##real": 9910, + "polit": 9911, + "ballet": 9912, + "containers": 9913, + "invest": 9914, + "maz": 9915, + "bedding": 9916, + "scottish": 9917, + "burger": 9918, + "raw": 9919, + "arrange": 9920, + "inscription": 9921, + "##coming": 9922, + "shipping": 9923, + "pc": 9924, + "##ateg": 9925, + "protect": 9926, + "courtyard": 9927, + "pud": 9928, + "beside": 9929, + "acou": 9930, + "##road": 9931, + "cd": 9932, + "zod": 9933, + "##eau": 9934, + "tin": 9935, + "fitted": 9936, + "residential": 9937, + "pickup": 9938, + "greenhouse": 9939, + "easily": 9940, + "##iner": 9941, + "##pa": 9942, + "enough": 9943, + "celebrates": 9944, + "zodiac": 9945, + "sonic": 9946, + "ornament": 9947, + "beyond": 9948, + "kang": 9949, + "dresser": 9950, + "skills": 9951, + "mansion": 9952, + "temper": 9953, + "religious": 9954, + "parad": 9955, + "christian": 9956, + "holes": 9957, + "kick": 9958, + "cob": 9959, + "##ctors": 9960, + "##ache": 9961, + "calif": 9962, + "##ilt": 9963, + "stripe": 9964, + "trains": 9965, + "teach": 9966, + "appears": 9967, + "boo": 9968, + "relaxing": 9969, + "##works": 9970, + "countries": 9971, + "apron": 9972, + "##used": 9973, + "##ued": 9974, + "ug": 9975, + "divers": 9976, + "##lers": 9977, + "kr": 9978, + "attack": 9979, + "brands": 9980, + "45": 9981, + "independ": 9982, + "futuristic": 9983, + "engineering": 9984, + "backdrop": 9985, + "biggest": 9986, + "pure": 9987, + "costa": 9988, + "islamic": 9989, + "ster": 9990, + "snack": 9991, + "assort": 9992, + "##rd": 9993, + "marking": 9994, + "combination": 9995, + "tournament": 9996, + "jazz": 9997, + "britain": 9998, + "colon": 9999, + "odd": 10000, + "mosaic": 10001, + "pap": 10002, + "thumb": 10003, + "acoustic": 10004, + "wick": 10005, + "korea": 10006, + "gaming": 10007, + "guitars": 10008, + "##hist": 10009, + "package": 10010, + "##ada": 10011, + "monarch": 10012, + "avenue": 10013, + "##ghan": 10014, + "2007": 10015, + "bold": 10016, + "pale": 10017, + "advertis": 10018, + "psy": 10019, + "bikini": 10020, + "rocking": 10021, + "promot": 10022, + "##eland": 10023, + "elev": 10024, + "harm": 10025, + "##tock": 10026, + "##iment": 10027, + "inters": 10028, + "##annel": 10029, + "##down": 10030, + "resour": 10031, + "faith": 10032, + "ridge": 10033, + "volcan": 10034, + "cola": 10035, + "louis": 10036, + "retail": 10037, + "statues": 10038, + "##ils": 10039, + "chapter": 10040, + "coins": 10041, + "##illas": 10042, + "bb": 10043, + "prime": 10044, + "stuck": 10045, + "staring": 10046, + "given": 10047, + "inte": 10048, + "anem": 10049, + "hedgehog": 10050, + "rect": 10051, + "blowing": 10052, + "garlic": 10053, + "toast": 10054, + "conserv": 10055, + "cows": 10056, + "highlight": 10057, + "forces": 10058, + "recyc": 10059, + "pencils": 10060, + "##gian": 10061, + "triangle": 10062, + "alarm": 10063, + "01": 10064, + "poke": 10065, + "era": 10066, + "##issance": 10067, + "tip": 10068, + "weird": 10069, + "saxoph": 10070, + "ones": 10071, + "deb": 10072, + "mam": 10073, + "cater": 10074, + "sla": 10075, + "kissing": 10076, + "##estic": 10077, + "shutterstock": 10078, + "oriental": 10079, + "kaw": 10080, + "ga": 10081, + "defin": 10082, + "##anding": 10083, + "presentation": 10084, + "shuttle": 10085, + "dall": 10086, + "lions": 10087, + "waist": 10088, + "anch": 10089, + "twist": 10090, + "##eline": 10091, + "dav": 10092, + "elderly": 10093, + "70": 10094, + "stamp": 10095, + "woodworking": 10096, + "total": 10097, + "busy": 10098, + "mustang": 10099, + "exe": 10100, + "speaker": 10101, + "catholic": 10102, + "gent": 10103, + "##ulture": 10104, + "harp": 10105, + "hamster": 10106, + "incredible": 10107, + "nurse": 10108, + "##ede": 10109, + "ware": 10110, + "salon": 10111, + "seaf": 10112, + "ipod": 10113, + "meals": 10114, + "terrier": 10115, + "rattle": 10116, + "horizont": 10117, + "loss": 10118, + "entire": 10119, + "va": 10120, + "##oma": 10121, + "society": 10122, + "campus": 10123, + "soda": 10124, + "joker": 10125, + "allow": 10126, + "crocodile": 10127, + "sher": 10128, + "fear": 10129, + "##ingers": 10130, + "newspaper": 10131, + "carnival": 10132, + "grazing": 10133, + "##gra": 10134, + "jam": 10135, + "butterflies": 10136, + "##bre": 10137, + "boards": 10138, + "crash": 10139, + "printing": 10140, + "protest": 10141, + "lamps": 10142, + "mi": 10143, + "sib": 10144, + "isra": 10145, + "footwear": 10146, + "##och": 10147, + "infant": 10148, + "ago": 10149, + "pork": 10150, + "electronics": 10151, + "ornaments": 10152, + "ferr": 10153, + "underst": 10154, + "presents": 10155, + "measure": 10156, + "branding": 10157, + "crop": 10158, + "sled": 10159, + "carrier": 10160, + "lime": 10161, + "figur": 10162, + "fiction": 10163, + "32": 10164, + "string": 10165, + "##asp": 10166, + "bands": 10167, + "turtles": 10168, + "oranges": 10169, + "protection": 10170, + "submarine": 10171, + "labor": 10172, + "ideal": 10173, + "countryside": 10174, + "seats": 10175, + "google": 10176, + "lift": 10177, + "questions": 10178, + "bison": 10179, + "metallic": 10180, + "compan": 10181, + "blocks": 10182, + "antel": 10183, + "jump": 10184, + "liber": 10185, + "snowm": 10186, + "kangar": 10187, + "carriage": 10188, + "plaid": 10189, + "beam": 10190, + "role": 10191, + "tours": 10192, + "israel": 10193, + "geck": 10194, + "remember": 10195, + "##ikes": 10196, + "##imes": 10197, + "heroes": 10198, + "##olis": 10199, + "mc": 10200, + "##weight": 10201, + "meth": 10202, + "marsh": 10203, + "adults": 10204, + "accent": 10205, + "script": 10206, + "lives": 10207, + "malays": 10208, + "##inet": 10209, + "pyth": 10210, + "farmer": 10211, + "dressing": 10212, + "actually": 10213, + "dal": 10214, + "afghan": 10215, + "exclus": 10216, + "episode": 10217, + "##otive": 10218, + "artistic": 10219, + "vamp": 10220, + "booth": 10221, + "avoc": 10222, + "taste": 10223, + "neighborhood": 10224, + "##phia": 10225, + "slide": 10226, + "chandel": 10227, + "##well": 10228, + "punch": 10229, + "vending": 10230, + "preparing": 10231, + "cact": 10232, + "bald": 10233, + "##uum": 10234, + "computers": 10235, + "doesn": 10236, + "##ius": 10237, + "norm": 10238, + "luck": 10239, + "brother": 10240, + "##ention": 10241, + "philadel": 10242, + "occas": 10243, + "growth": 10244, + "80": 10245, + "mantis": 10246, + "textured": 10247, + "insects": 10248, + "philadelphia": 10249, + "convert": 10250, + "badge": 10251, + "ak": 10252, + "rang": 10253, + "##intend": 10254, + "studios": 10255, + "##reat": 10256, + "started": 10257, + "dallas": 10258, + "az": 10259, + "##erness": 10260, + "bathrooms": 10261, + "sterling": 10262, + "saree": 10263, + "wore": 10264, + "audi": 10265, + "swed": 10266, + "armad": 10267, + "lane": 10268, + "##gre": 10269, + "rectang": 10270, + "peach": 10271, + "phoen": 10272, + "batt": 10273, + "es": 10274, + "docked": 10275, + "##rets": 10276, + "pod": 10277, + "muff": 10278, + "bugs": 10279, + "##giving": 10280, + "nutr": 10281, + "cargo": 10282, + "excellent": 10283, + "##ially": 10284, + "walt": 10285, + "tight": 10286, + "pyram": 10287, + "discuss": 10288, + "trust": 10289, + "pee": 10290, + "oval": 10291, + "fund": 10292, + "pressure": 10293, + "shrimp": 10294, + "##ndi": 10295, + "##aries": 10296, + "hardwood": 10297, + "essential": 10298, + "issue": 10299, + "dvd": 10300, + "##print": 10301, + "goals": 10302, + "tumb": 10303, + "cassette": 10304, + "controller": 10305, + "ask": 10306, + "maple": 10307, + "chan": 10308, + "dreams": 10309, + "##mel": 10310, + "vacuum": 10311, + "carbon": 10312, + "tunnel": 10313, + "satin": 10314, + "##itu": 10315, + "sri": 10316, + "wholes": 10317, + "##othing": 10318, + "fen": 10319, + "unkn": 10320, + "aluminum": 10321, + "trek": 10322, + "indones": 10323, + "dar": 10324, + "struct": 10325, + "ik": 10326, + "timber": 10327, + "yorkshire": 10328, + "mop": 10329, + "mans": 10330, + "bor": 10331, + "thread": 10332, + "peanut": 10333, + "frames": 10334, + "embroidered": 10335, + "problem": 10336, + "chevy": 10337, + "accom": 10338, + "dun": 10339, + "tablet": 10340, + "##rup": 10341, + "answ": 10342, + "demonstr": 10343, + "vir": 10344, + "sock": 10345, + "orb": 10346, + "sandy": 10347, + "##ogn": 10348, + "wilderness": 10349, + "toad": 10350, + "plain": 10351, + "unknown": 10352, + "bollywood": 10353, + "braids": 10354, + "eight": 10355, + "##iring": 10356, + "blurred": 10357, + "fuel": 10358, + "##stery": 10359, + "seasons": 10360, + "bib": 10361, + "instant": 10362, + "##eer": 10363, + "thanksgiving": 10364, + "filter": 10365, + "leaning": 10366, + "##fer": 10367, + "arabic": 10368, + "sams": 10369, + "runn": 10370, + "botanical": 10371, + "acorn": 10372, + "bakery": 10373, + "lakes": 10374, + "multi": 10375, + "glowing": 10376, + "marina": 10377, + "cheetah": 10378, + "pome": 10379, + "arg": 10380, + "examples": 10381, + "route": 10382, + "##isc": 10383, + "##cake": 10384, + "screws": 10385, + "province": 10386, + "rural": 10387, + "tribal": 10388, + "venue": 10389, + "wholesale": 10390, + "firef": 10391, + "linear": 10392, + "##va": 10393, + "36": 10394, + "horizontal": 10395, + "1st": 10396, + "barcel": 10397, + "calm": 10398, + "approach": 10399, + "essay": 10400, + "elf": 10401, + "teles": 10402, + "layer": 10403, + "remove": 10404, + "owners": 10405, + "##aii": 10406, + "totem": 10407, + "handsome": 10408, + "renaissance": 10409, + "slices": 10410, + "sheets": 10411, + "##sel": 10412, + "danger": 10413, + "##pot": 10414, + "faux": 10415, + "##verage": 10416, + "means": 10417, + "dodge": 10418, + "transformers": 10419, + "beaver": 10420, + "options": 10421, + "onions": 10422, + "os": 10423, + "##agne": 10424, + "python": 10425, + "relationship": 10426, + "according": 10427, + "cycling": 10428, + "2006": 10429, + "score": 10430, + "##atar": 10431, + "dum": 10432, + "##lor": 10433, + "invit": 10434, + "therm": 10435, + "dashboard": 10436, + "steak": 10437, + "##izer": 10438, + "dais": 10439, + "parks": 10440, + "samsung": 10441, + "##fol": 10442, + "heaven": 10443, + "wheat": 10444, + "##ja": 10445, + "##enty": 10446, + "carous": 10447, + "dutch": 10448, + "release": 10449, + "sunlight": 10450, + "ach": 10451, + "fris": 10452, + "restaurants": 10453, + "godd": 10454, + "instit": 10455, + "joint": 10456, + "##ington": 10457, + "fantastic": 10458, + "##ested": 10459, + "dan": 10460, + "##izing": 10461, + "drops": 10462, + "kne": 10463, + "assist": 10464, + "phoenix": 10465, + "starting": 10466, + "##24": 10467, + "movement": 10468, + "interview": 10469, + "##west": 10470, + "sunflower": 10471, + "sharks": 10472, + "mt": 10473, + "vent": 10474, + "battery": 10475, + "follow": 10476, + "went": 10477, + "corporate": 10478, + "econ": 10479, + "trap": 10480, + "shooting": 10481, + "barcelona": 10482, + "##oven": 10483, + "fisher": 10484, + "lave": 10485, + "wanted": 10486, + "##edy": 10487, + "scary": 10488, + "##rol": 10489, + "loves": 10490, + "porcel": 10491, + "hut": 10492, + "susp": 10493, + "##ls": 10494, + "##izz": 10495, + "##lete": 10496, + "recently": 10497, + "seafood": 10498, + "arrivals": 10499, + "sculptures": 10500, + "porcelain": 10501, + "badger": 10502, + "emot": 10503, + "et": 10504, + "tourists": 10505, + "produce": 10506, + "expensive": 10507, + "unise": 10508, + "pilot": 10509, + "##bridge": 10510, + "lyn": 10511, + "lemons": 10512, + "mood": 10513, + "##ability": 10514, + "piggy": 10515, + "sym": 10516, + "90": 10517, + "##rome": 10518, + "##illy": 10519, + "direction": 10520, + "sample": 10521, + "##ration": 10522, + "sunsh": 10523, + "nintend": 10524, + "nether": 10525, + "cad": 10526, + "##ares": 10527, + "unisex": 10528, + "leave": 10529, + "muslim": 10530, + "prison": 10531, + "tang": 10532, + "stall": 10533, + "spoons": 10534, + "wales": 10535, + "nintendo": 10536, + "gentle": 10537, + "lavender": 10538, + "neut": 10539, + "lovers": 10540, + "photoshop": 10541, + "adap": 10542, + "positive": 10543, + "goddess": 10544, + "baskets": 10545, + "paj": 10546, + "treas": 10547, + "related": 10548, + "##rane": 10549, + "bloom": 10550, + "haired": 10551, + "champagne": 10552, + "saxophone": 10553, + "jes": 10554, + "towels": 10555, + "herbs": 10556, + "beij": 10557, + "herb": 10558, + "paradise": 10559, + "pou": 10560, + "scenery": 10561, + "beijing": 10562, + "##he": 10563, + "##ander": 10564, + "slope": 10565, + "linen": 10566, + "hund": 10567, + "der": 10568, + "stump": 10569, + "marm": 10570, + "code": 10571, + "kings": 10572, + "nyc": 10573, + "sims": 10574, + "heels": 10575, + "##night": 10576, + "volun": 10577, + "roasted": 10578, + "##19": 10579, + "##ively": 10580, + "hundred": 10581, + "shaw": 10582, + "warehouse": 10583, + "pumpkins": 10584, + "pokemon": 10585, + "freed": 10586, + "pum": 10587, + "##cery": 10588, + "##opy": 10589, + "thousands": 10590, + "greatest": 10591, + "ris": 10592, + "cactus": 10593, + "driveway": 10594, + "summit": 10595, + "sent": 10596, + "##lyn": 10597, + "serve": 10598, + "creamy": 10599, + "tun": 10600, + "treatment": 10601, + "##ila": 10602, + "hindu": 10603, + "##ify": 10604, + "##ilton": 10605, + "##istle": 10606, + "drople": 10607, + "vases": 10608, + "##ami": 10609, + "techniques": 10610, + "jets": 10611, + "creature": 10612, + "##cting": 10613, + "carousel": 10614, + "career": 10615, + "oz": 10616, + "value": 10617, + "installed": 10618, + "signed": 10619, + "kayak": 10620, + "recent": 10621, + "current": 10622, + "prayer": 10623, + "comfortable": 10624, + "labels": 10625, + "##gae": 10626, + "waff": 10627, + "##istry": 10628, + "celebs": 10629, + "##vention": 10630, + "championships": 10631, + "postcard": 10632, + "lag": 10633, + "##ranean": 10634, + "angels": 10635, + "vanc": 10636, + "flyer": 10637, + "cinnam": 10638, + "cushion": 10639, + "##ouver": 10640, + "kimon": 10641, + "##15": 10642, + "signature": 10643, + "steamp": 10644, + "veter": 10645, + "accessory": 10646, + "steampunk": 10647, + "harbour": 10648, + "sacred": 10649, + "kitchens": 10650, + "vancouver": 10651, + "saus": 10652, + "cigarette": 10653, + "emerald": 10654, + "atlanta": 10655, + "##ades": 10656, + "springs": 10657, + "hiding": 10658, + "dus": 10659, + "vo": 10660, + "tied": 10661, + "wol": 10662, + "john": 10663, + "folk": 10664, + "##18": 10665, + "cancer": 10666, + "mediter": 10667, + "needed": 10668, + "lighter": 10669, + "narrow": 10670, + "##ically": 10671, + "photographs": 10672, + "meh": 10673, + "labrad": 10674, + "sed": 10675, + "stool": 10676, + "cooker": 10677, + "bod": 10678, + "charm": 10679, + "queens": 10680, + "winner": 10681, + "shallow": 10682, + "##point": 10683, + "teams": 10684, + "ankle": 10685, + "therap": 10686, + "satell": 10687, + "##ilities": 10688, + "execut": 10689, + "##omes": 10690, + "automobile": 10691, + "channel": 10692, + "vampire": 10693, + "chapel": 10694, + "magical": 10695, + "waving": 10696, + "opera": 10697, + "##14": 10698, + "actresses": 10699, + "bulb": 10700, + "eaten": 10701, + "athlete": 10702, + "mun": 10703, + "##onge": 10704, + "cel": 10705, + "##right": 10706, + "taxi": 10707, + "mediterranean": 10708, + "imm": 10709, + "celebrating": 10710, + "netherlands": 10711, + "hallway": 10712, + "premier": 10713, + "volunte": 10714, + "breast": 10715, + "ikea": 10716, + "sunshine": 10717, + "result": 10718, + "alp": 10719, + "honor": 10720, + "cu": 10721, + "deliver": 10722, + "##imed": 10723, + "liverpool": 10724, + "pred": 10725, + "invitations": 10726, + "kangaroo": 10727, + "passenger": 10728, + "mamm": 10729, + "nutrition": 10730, + "kits": 10731, + "gt": 10732, + "sales": 10733, + "suitable": 10734, + "hub": 10735, + "cham": 10736, + "hone": 10737, + "flame": 10738, + "smiles": 10739, + "musician": 10740, + "##aling": 10741, + "##berg": 10742, + "scratch": 10743, + "slight": 10744, + "sliced": 10745, + "chat": 10746, + "diamonds": 10747, + "laughing": 10748, + "blazer": 10749, + "##elda": 10750, + "exclusive": 10751, + "milan": 10752, + "decal": 10753, + "blade": 10754, + "inches": 10755, + "pills": 10756, + "ontar": 10757, + "babo": 10758, + "lu": 10759, + "bones": 10760, + "destination": 10761, + "ontario": 10762, + "buying": 10763, + "lily": 10764, + "caption": 10765, + "applian": 10766, + "coo": 10767, + "cinnamon": 10768, + "jug": 10769, + "zelda": 10770, + "remodel": 10771, + "mugs": 10772, + "clarinet": 10773, + "provided": 10774, + "accordion": 10775, + "dial": 10776, + "brushes": 10777, + "audience": 10778, + "##eness": 10779, + "##ither": 10780, + "bought": 10781, + "makeover": 10782, + "almond": 10783, + "knot": 10784, + "expression": 10785, + "hello": 10786, + "viper": 10787, + "orchid": 10788, + "provides": 10789, + "##opus": 10790, + "sharp": 10791, + "pregnant": 10792, + "tricks": 10793, + "nothing": 10794, + "##70": 10795, + "blossom": 10796, + "##pose": 10797, + "tailed": 10798, + "shang": 10799, + "finding": 10800, + "##anced": 10801, + "##itt": 10802, + "transportation": 10803, + "handbags": 10804, + "plaza": 10805, + "cheesecake": 10806, + "overlooking": 10807, + "accents": 10808, + "##market": 10809, + "canoe": 10810, + "grocery": 10811, + "guns": 10812, + "champions": 10813, + "tourism": 10814, + "##inth": 10815, + "possible": 10816, + "malaysia": 10817, + "tort": 10818, + "repe": 10819, + "mock": 10820, + "markings": 10821, + "ech": 10822, + "waste": 10823, + "aviation": 10824, + "##68": 10825, + "shovel": 10826, + "hugging": 10827, + "##ulous": 10828, + "freedom": 10829, + "##itzer": 10830, + "##list": 10831, + "bohem": 10832, + "##inic": 10833, + "residence": 10834, + "droplets": 10835, + "lank": 10836, + "flour": 10837, + "poo": 10838, + "##burger": 10839, + "dair": 10840, + "canopy": 10841, + "statement": 10842, + "##utions": 10843, + "fingers": 10844, + "fully": 10845, + "zone": 10846, + "33": 10847, + "arrives": 10848, + "smaller": 10849, + "killer": 10850, + "boss": 10851, + "able": 10852, + "alternative": 10853, + "jesus": 10854, + "question": 10855, + "vanity": 10856, + "motel": 10857, + "##coat": 10858, + "sized": 10859, + "vanilla": 10860, + "guesth": 10861, + "newly": 10862, + "customer": 10863, + "attra": 10864, + "ladder": 10865, + "celt": 10866, + "guesthouse": 10867, + "successful": 10868, + "boxer": 10869, + "systems": 10870, + "cree": 10871, + "sharing": 10872, + "destinations": 10873, + "beaches": 10874, + "sph": 10875, + "consider": 10876, + "##isle": 10877, + "dried": 10878, + "ethnic": 10879, + "visitors": 10880, + "innov": 10881, + "scand": 10882, + "##wan": 10883, + "pipes": 10884, + "cau": 10885, + "##las": 10886, + "cannon": 10887, + "##vious": 10888, + "gem": 10889, + "robe": 10890, + "parach": 10891, + "cryst": 10892, + "ages": 10893, + "fries": 10894, + "colourful": 10895, + "hur": 10896, + "flavor": 10897, + "kawaii": 10898, + "##arters": 10899, + "changing": 10900, + "300": 10901, + "##kok": 10902, + "strength": 10903, + "motivational": 10904, + "reasons": 10905, + "dairy": 10906, + "noodles": 10907, + "##eek": 10908, + "loved": 10909, + "foil": 10910, + "anemon": 10911, + "semi": 10912, + "ky": 10913, + "##itz": 10914, + "automatic": 10915, + "marriage": 10916, + "##piece": 10917, + "##anda": 10918, + "##ears": 10919, + "valentines": 10920, + "winds": 10921, + "pretz": 10922, + "grain": 10923, + "circles": 10924, + "panc": 10925, + "composition": 10926, + "gods": 10927, + "brake": 10928, + "gel": 10929, + "##oda": 10930, + "##omic": 10931, + "##ndy": 10932, + "volcano": 10933, + "patri": 10934, + "believe": 10935, + "bust": 10936, + "sust": 10937, + "##acle": 10938, + "rising": 10939, + "si": 10940, + "prote": 10941, + "2nd": 10942, + "##antine": 10943, + "active": 10944, + "protective": 10945, + "coats": 10946, + "lac": 10947, + "##dam": 10948, + "geyse": 10949, + "loaf": 10950, + "lum": 10951, + "compact": 10952, + "madrid": 10953, + "astrona": 10954, + "##ometer": 10955, + "destro": 10956, + "mower": 10957, + "sliding": 10958, + "assortment": 10959, + "cone": 10960, + "ker": 10961, + "olympics": 10962, + "mehndi": 10963, + "##ification": 10964, + "perg": 10965, + "diego": 10966, + "planes": 10967, + "convertible": 10968, + "##igs": 10969, + "##igan": 10970, + "cattle": 10971, + "forever": 10972, + "brooklyn": 10973, + "diver": 10974, + "carving": 10975, + "##uted": 10976, + "illuminated": 10977, + "columbia": 10978, + "inner": 10979, + "revolution": 10980, + "publish": 10981, + "instruction": 10982, + "asp": 10983, + "dise": 10984, + "sew": 10985, + "chanel": 10986, + "##awa": 10987, + "sustain": 10988, + "fourth": 10989, + "pony": 10990, + "helm": 10991, + "##28": 10992, + "goose": 10993, + "##aga": 10994, + "overlay": 10995, + "##gi": 10996, + "prepared": 10997, + "lyric": 10998, + "mash": 10999, + "sight": 11000, + "squadron": 11001, + "viking": 11002, + "treats": 11003, + "##itude": 11004, + "widow": 11005, + "##gon": 11006, + "##iler": 11007, + "dusk": 11008, + "90s": 11009, + "##outs": 11010, + "weapons": 11011, + "universal": 11012, + "mosqu": 11013, + "##stan": 11014, + "##ador": 11015, + "colt": 11016, + "trade": 11017, + "##osph": 11018, + "fridge": 11019, + "council": 11020, + "throwing": 11021, + "afp": 11022, + "heel": 11023, + "bali": 11024, + "##bread": 11025, + "folding": 11026, + "gang": 11027, + "##mont": 11028, + "buses": 11029, + "complet": 11030, + "20th": 11031, + "roads": 11032, + "needle": 11033, + "##iffon": 11034, + "octopus": 11035, + "hacks": 11036, + "dies": 11037, + "surg": 11038, + "##pton": 11039, + "leaders": 11040, + "##east": 11041, + "imper": 11042, + "caucas": 11043, + "liberty": 11044, + "division": 11045, + "beginning": 11046, + "anchor": 11047, + "##17": 11048, + "uss": 11049, + "felt": 11050, + "##amas": 11051, + "telescope": 11052, + "75": 11053, + "panor": 11054, + "silhouettes": 11055, + "##90": 11056, + "smok": 11057, + "satellite": 11058, + "fungus": 11059, + "somet": 11060, + "##ihu": 11061, + "vend": 11062, + "connected": 11063, + "wait": 11064, + "algae": 11065, + "cans": 11066, + "hourglass": 11067, + "##ida": 11068, + "##ji": 11069, + "swiss": 11070, + "plaque": 11071, + "isometric": 11072, + "chrome": 11073, + "##brush": 11074, + "##itan": 11075, + "basil": 11076, + "skiing": 11077, + "soon": 11078, + "gond": 11079, + "tab": 11080, + "format": 11081, + "medal": 11082, + "cout": 11083, + "typical": 11084, + "prog": 11085, + "camer": 11086, + "rough": 11087, + "charl": 11088, + "originals": 11089, + "bohemian": 11090, + "phones": 11091, + "missile": 11092, + "setup": 11093, + "agric": 11094, + "##oto": 11095, + "mast": 11096, + "##16": 11097, + "##uters": 11098, + "orchest": 11099, + "gest": 11100, + "prec": 11101, + "vans": 11102, + "roots": 11103, + "recre": 11104, + "rand": 11105, + "pergola": 11106, + "cycle": 11107, + "sponge": 11108, + "circus": 11109, + "##leans": 11110, + "##more": 11111, + "displays": 11112, + "musicians": 11113, + "indonesia": 11114, + "##ela": 11115, + "pins": 11116, + "serious": 11117, + "skies": 11118, + "##bell": 11119, + "##ocular": 11120, + "cosmetics": 11121, + "persp": 11122, + "orleans": 11123, + "personalized": 11124, + "engraving": 11125, + "reason": 11126, + "solution": 11127, + "altar": 11128, + "shops": 11129, + "onion": 11130, + "tale": 11131, + "chandelier": 11132, + "##gin": 11133, + "kitty": 11134, + "layers": 11135, + "##idas": 11136, + "##acles": 11137, + "tib": 11138, + "##hai": 11139, + "curled": 11140, + "winn": 11141, + "armour": 11142, + "join": 11143, + "filling": 11144, + "##elsea": 11145, + "paral": 11146, + "drama": 11147, + "laser": 11148, + "split": 11149, + "elementary": 11150, + "##bug": 11151, + "bangkok": 11152, + "pouring": 11153, + "overs": 11154, + "shelter": 11155, + "templates": 11156, + "superman": 11157, + "defe": 11158, + "cliffs": 11159, + "couture": 11160, + "2000": 11161, + "describ": 11162, + "achie": 11163, + "cosm": 11164, + "animated": 11165, + "fenced": 11166, + "zuc": 11167, + "maxi": 11168, + "binocular": 11169, + "chelsea": 11170, + "cases": 11171, + "##ements": 11172, + "adidas": 11173, + "ms": 11174, + "orchestra": 11175, + "farmers": 11176, + "couples": 11177, + "neg": 11178, + "barrels": 11179, + "pairs": 11180, + "motivation": 11181, + "ferrari": 11182, + "##ifying": 11183, + "infinity": 11184, + "34": 11185, + "corps": 11186, + "ton": 11187, + "sting": 11188, + "sailboat": 11189, + "##icting": 11190, + "tutorials": 11191, + "lynx": 11192, + "gala": 11193, + "##lr": 11194, + "tara": 11195, + "bless": 11196, + "fake": 11197, + "covering": 11198, + "financ": 11199, + "prair": 11200, + "leader": 11201, + "virus": 11202, + "##aki": 11203, + "##ava": 11204, + "adding": 11205, + "belle": 11206, + "families": 11207, + "victory": 11208, + "##ensions": 11209, + "rules": 11210, + "camou": 11211, + "chiffon": 11212, + "##ops": 11213, + "190": 11214, + "drawer": 11215, + "nfl": 11216, + "##chi": 11217, + "camoufla": 11218, + "meerkat": 11219, + "trick": 11220, + "teapot": 11221, + "captured": 11222, + "teenage": 11223, + "macro": 11224, + "##owers": 11225, + "##ania": 11226, + "facebook": 11227, + "nation": 11228, + "virt": 11229, + "problems": 11230, + "towers": 11231, + "##roit": 11232, + "fictional": 11233, + "tone": 11234, + "showc": 11235, + "##matic": 11236, + "atlantic": 11237, + "sir": 11238, + "kun": 11239, + "regular": 11240, + "feeling": 11241, + "2005": 11242, + "rattler": 11243, + "bridesmaid": 11244, + "impact": 11245, + "caucasian": 11246, + "contains": 11247, + "##aya": 11248, + "##rus": 11249, + "rio": 11250, + "progress": 11251, + "detroit": 11252, + "od": 11253, + "mythology": 11254, + "primary": 11255, + "roller": 11256, + "##13": 11257, + "matter": 11258, + "mosc": 11259, + "ugly": 11260, + "sony": 11261, + "jars": 11262, + "portable": 11263, + "##sole": 11264, + "##care": 11265, + "parent": 11266, + "##bul": 11267, + "matches": 11268, + "mortar": 11269, + "hound": 11270, + "naut": 11271, + "eb": 11272, + "##jector": 11273, + "##xture": 11274, + "##eing": 11275, + "powerful": 11276, + "rides": 11277, + "bathing": 11278, + "crit": 11279, + "##ographic": 11280, + "dung": 11281, + "envel": 11282, + "envelope": 11283, + "supply": 11284, + "justice": 11285, + "##bec": 11286, + "helping": 11287, + "##aka": 11288, + "lugg": 11289, + "yellowstone": 11290, + "rooft": 11291, + "rental": 11292, + "speaks": 11293, + "married": 11294, + "vision": 11295, + "prey": 11296, + "dental": 11297, + "palette": 11298, + "##iage": 11299, + "subway": 11300, + "spiritual": 11301, + "##hound": 11302, + "least": 11303, + "chihu": 11304, + "chihuah": 11305, + "nba": 11306, + "##oli": 11307, + "titled": 11308, + "sometimes": 11309, + "attire": 11310, + "ivory": 11311, + "carb": 11312, + "##bar": 11313, + "beatles": 11314, + "whether": 11315, + "darkness": 11316, + "##oa": 11317, + "hunt": 11318, + "moscow": 11319, + "either": 11320, + "software": 11321, + "binoculars": 11322, + "crock": 11323, + "ladybug": 11324, + "simply": 11325, + "youtub": 11326, + "tx": 11327, + "legends": 11328, + "lanka": 11329, + "opport": 11330, + "brow": 11331, + "kitt": 11332, + "afford": 11333, + "celtic": 11334, + "facility": 11335, + "stores": 11336, + "situ": 11337, + "youtube": 11338, + "somewhere": 11339, + "strange": 11340, + "wrest": 11341, + "opportun": 11342, + "festive": 11343, + "tam": 11344, + "jer": 11345, + "lightning": 11346, + "handwriting": 11347, + "##inav": 11348, + "##oke": 11349, + "##arding": 11350, + "##tenance": 11351, + "devil": 11352, + "mes": 11353, + "plated": 11354, + "leads": 11355, + "aisle": 11356, + "scandinav": 11357, + "prior": 11358, + "shiny": 11359, + "appliances": 11360, + "dh": 11361, + "maintenance": 11362, + "creation": 11363, + "resist": 11364, + "cribt": 11365, + "priest": 11366, + "shadows": 11367, + "arrang": 11368, + "tiss": 11369, + "columns": 11370, + "vegetarian": 11371, + "protein": 11372, + "furry": 11373, + "hermit": 11374, + "grid": 11375, + "circuit": 11376, + "##gen": 11377, + "minister": 11378, + "ripe": 11379, + "plank": 11380, + "texts": 11381, + "junior": 11382, + "##ificial": 11383, + "caramel": 11384, + "cannes": 11385, + "depicting": 11386, + "combat": 11387, + "helps": 11388, + "##li": 11389, + "perspective": 11390, + "projector": 11391, + "camper": 11392, + "neutral": 11393, + "adventures": 11394, + "advertisement": 11395, + "exch": 11396, + "instead": 11397, + "wea": 11398, + "textile": 11399, + "switzer": 11400, + "intersection": 11401, + "pigs": 11402, + "depict": 11403, + "decorate": 11404, + "remov": 11405, + "sanct": 11406, + "companies": 11407, + "luggage": 11408, + "pound": 11409, + "amph": 11410, + "didn": 11411, + "catching": 11412, + "smith": 11413, + "##proof": 11414, + "switzerland": 11415, + "dolphin": 11416, + "prairie": 11417, + "##ogy": 11418, + "renovation": 11419, + "mandala": 11420, + "##bed": 11421, + "hostel": 11422, + "##ns": 11423, + "adjustable": 11424, + "helmets": 11425, + "zero": 11426, + "##ocal": 11427, + "cocktails": 11428, + "##arten": 11429, + "hilar": 11430, + "revol": 11431, + "dryer": 11432, + "tripod": 11433, + "mango": 11434, + "18th": 11435, + "classical": 11436, + "moth": 11437, + "chess": 11438, + "extended": 11439, + "##sa": 11440, + "educational": 11441, + "##lad": 11442, + "labrador": 11443, + "38": 11444, + "lesson": 11445, + "ipad": 11446, + "constru": 11447, + "chihuahua": 11448, + "hem": 11449, + "quite": 11450, + "recogn": 11451, + "bark": 11452, + "chemical": 11453, + "corals": 11454, + "console": 11455, + "teal": 11456, + "description": 11457, + "##hire": 11458, + "grilled": 11459, + "polka": 11460, + "##nity": 11461, + "trav": 11462, + "jord": 11463, + "generation": 11464, + "artificial": 11465, + "political": 11466, + "hilarious": 11467, + "rid": 11468, + "42": 11469, + "passing": 11470, + "dramatic": 11471, + "wheelchair": 11472, + "grove": 11473, + "condu": 11474, + "appeared": 11475, + "headed": 11476, + "bung": 11477, + "##ready": 11478, + "pride": 11479, + "volks": 11480, + "memory": 11481, + "content": 11482, + "toward": 11483, + "##ubs": 11484, + "foliage": 11485, + "##neath": 11486, + "already": 11487, + "underneath": 11488, + "rider": 11489, + "housing": 11490, + "expla": 11491, + "outer": 11492, + "underground": 11493, + "lon": 11494, + "##iki": 11495, + "rainy": 11496, + "mich": 11497, + "cha": 11498, + "##wagen": 11499, + "pajamas": 11500, + "37": 11501, + "cubs": 11502, + "probably": 11503, + "crust": 11504, + "##eon": 11505, + "difference": 11506, + "chimp": 11507, + "plow": 11508, + "matte": 11509, + "##ibles": 11510, + "epic": 11511, + "##estone": 11512, + "##lon": 11513, + "foam": 11514, + "supermarket": 11515, + "tay": 11516, + "monoch": 11517, + "walkway": 11518, + "blush": 11519, + "volkswagen": 11520, + "therapy": 11521, + "##laces": 11522, + "compar": 11523, + "##cially": 11524, + "##cci": 11525, + "resources": 11526, + "flip": 11527, + "stroll": 11528, + "cement": 11529, + "zomb": 11530, + "trench": 11531, + "institute": 11532, + "stress": 11533, + "admir": 11534, + "polo": 11535, + "iran": 11536, + "target": 11537, + "nine": 11538, + "goods": 11539, + "bomb": 11540, + "princ": 11541, + "cause": 11542, + "brew": 11543, + "abb": 11544, + "indic": 11545, + "schools": 11546, + "isn": 11547, + "beaded": 11548, + "shanghai": 11549, + "rivers": 11550, + "grab": 11551, + "jokes": 11552, + "giraff": 11553, + "eiff": 11554, + "cello": 11555, + "escape": 11556, + "invol": 11557, + "##lah": 11558, + "cyber": 11559, + "resorts": 11560, + "##istol": 11561, + "horizon": 11562, + "manhole": 11563, + "occasion": 11564, + "contract": 11565, + "booksh": 11566, + "rim": 11567, + "lincol": 11568, + "blinds": 11569, + "##rem": 11570, + "tires": 11571, + "marks": 11572, + "infl": 11573, + "##estrian": 11574, + "identity": 11575, + "planets": 11576, + "til": 11577, + "beagle": 11578, + "##akers": 11579, + "customers": 11580, + "executive": 11581, + "whipped": 11582, + "rally": 11583, + "grapes": 11584, + "lincoln": 11585, + "pest": 11586, + "avocado": 11587, + "sap": 11588, + "brief": 11589, + "tric": 11590, + "##urance": 11591, + "metro": 11592, + "reuters": 11593, + "48": 11594, + "airlines": 11595, + "kittens": 11596, + "banjo": 11597, + "auction": 11598, + "visiting": 11599, + "harvest": 11600, + "pops": 11601, + "gib": 11602, + "##sen": 11603, + "quiet": 11604, + "tags": 11605, + "revers": 11606, + "pudding": 11607, + "init": 11608, + "kimono": 11609, + "##ever": 11610, + "brothers": 11611, + "portugal": 11612, + "sausage": 11613, + "##ternal": 11614, + "savann": 11615, + "##mond": 11616, + "bom": 11617, + "styling": 11618, + "camouflage": 11619, + "dublin": 11620, + "address": 11621, + "99": 11622, + "sneak": 11623, + "bund": 11624, + "replace": 11625, + "##cts": 11626, + "##icing": 11627, + "weeks": 11628, + "fonts": 11629, + "##roud": 11630, + "interact": 11631, + "tromb": 11632, + "scoop": 11633, + "##iel": 11634, + "##vey": 11635, + "financial": 11636, + "trous": 11637, + "lil": 11638, + "completed": 11639, + "repl": 11640, + "weas": 11641, + "slipper": 11642, + "gone": 11643, + "##aph": 11644, + "units": 11645, + "loaded": 11646, + "##ighter": 11647, + "miles": 11648, + "wy": 11649, + "##entic": 11650, + "pouch": 11651, + "scandinavian": 11652, + "lips": 11653, + "prepare": 11654, + "snowman": 11655, + "modest": 11656, + "tibet": 11657, + "chow": 11658, + "raj": 11659, + "rasp": 11660, + "deter": 11661, + "wishes": 11662, + "cav": 11663, + "presented": 11664, + "muscles": 11665, + "exercises": 11666, + "eyed": 11667, + "##lum": 11668, + "stretch": 11669, + "hipster": 11670, + "2004": 11671, + "contact": 11672, + "##ott": 11673, + "##rones": 11674, + "80s": 11675, + "assembly": 11676, + "approx": 11677, + "folded": 11678, + "spice": 11679, + "dump": 11680, + "separate": 11681, + "glove": 11682, + "maze": 11683, + "umbrellas": 11684, + "02": 11685, + "##cin": 11686, + "##rier": 11687, + "petals": 11688, + "zucch": 11689, + "skinny": 11690, + "belly": 11691, + "ast": 11692, + "ants": 11693, + "villas": 11694, + "##fully": 11695, + "lightweight": 11696, + "minute": 11697, + "geor": 11698, + "skunk": 11699, + "subst": 11700, + "dur": 11701, + "messages": 11702, + "manne": 11703, + "rum": 11704, + "smoothie": 11705, + "##gie": 11706, + "kenya": 11707, + "flies": 11708, + "ding": 11709, + "slightly": 11710, + "sanctuary": 11711, + "racer": 11712, + "gau": 11713, + "bungal": 11714, + "colonial": 11715, + "##gings": 11716, + "zipper": 11717, + "laboratory": 11718, + "hive": 11719, + "burgu": 11720, + "tart": 11721, + "purpose": 11722, + "warriors": 11723, + "loose": 11724, + "greet": 11725, + "bicycles": 11726, + "harley": 11727, + "jordan": 11728, + "runner": 11729, + "passion": 11730, + "4th": 11731, + "cobra": 11732, + "tents": 11733, + "hardware": 11734, + "hydra": 11735, + "structures": 11736, + "bisc": 11737, + "buffet": 11738, + "burgundy": 11739, + "catal": 11740, + "oy": 11741, + "planting": 11742, + "effort": 11743, + "tasty": 11744, + "##front": 11745, + "bound": 11746, + "tig": 11747, + "##eed": 11748, + "imperial": 11749, + "pools": 11750, + "truth": 11751, + "suff": 11752, + "spor": 11753, + "##iolet": 11754, + "impression": 11755, + "cm": 11756, + "kinderg": 11757, + "diesel": 11758, + "physical": 11759, + "straps": 11760, + "favour": 11761, + "tier": 11762, + "hamburger": 11763, + "##oul": 11764, + "landmark": 11765, + "gog": 11766, + "##zzles": 11767, + "stir": 11768, + "hatch": 11769, + "observ": 11770, + "styl": 11771, + "gy": 11772, + "breaking": 11773, + "became": 11774, + "rooftop": 11775, + "400": 11776, + "kitchenette": 11777, + "diaper": 11778, + "ord": 11779, + "resolution": 11780, + "atmosph": 11781, + "##corn": 11782, + "dorm": 11783, + "awar": 11784, + "mystery": 11785, + "nom": 11786, + "55": 11787, + "paddle": 11788, + "leggings": 11789, + "vess": 11790, + "oppos": 11791, + "merch": 11792, + "##engers": 11793, + "climate": 11794, + "corre": 11795, + "watermel": 11796, + "environmental": 11797, + "frag": 11798, + "monsters": 11799, + "##anes": 11800, + "##ucks": 11801, + "supreme": 11802, + "crossed": 11803, + "awareness": 11804, + "##rage": 11805, + "architects": 11806, + "exposed": 11807, + "##body": 11808, + "pixar": 11809, + "eve": 11810, + "##pal": 11811, + "tumblr": 11812, + "paralle": 11813, + "influ": 11814, + "##rors": 11815, + "greater": 11816, + "cyl": 11817, + "##idence": 11818, + "whales": 11819, + "manor": 11820, + "reach": 11821, + "axe": 11822, + "astronaut": 11823, + "##nuts": 11824, + "tapest": 11825, + "haz": 11826, + "random": 11827, + "everyday": 11828, + "specific": 11829, + "benz": 11830, + "lic": 11831, + "blues": 11832, + "niss": 11833, + "marath": 11834, + "operations": 11835, + "hitting": 11836, + "broch": 11837, + "amster": 11838, + "amsterdam": 11839, + "##mers": 11840, + "scrap": 11841, + "murals": 11842, + "amount": 11843, + "alps": 11844, + "kindergarten": 11845, + "##fire": 11846, + "rept": 11847, + "wreck": 11848, + "espe": 11849, + "suspension": 11850, + "gucci": 11851, + "cleaner": 11852, + "banks": 11853, + "brought": 11854, + "stere": 11855, + "logs": 11856, + "pul": 11857, + "relief": 11858, + "evolution": 11859, + "iceland": 11860, + "##ridge": 11861, + "bricks": 11862, + "thrones": 11863, + "##otes": 11864, + "lamb": 11865, + "mile": 11866, + "arranged": 11867, + "##isters": 11868, + "clut": 11869, + "carrots": 11870, + "maternity": 11871, + "mol": 11872, + "furnished": 11873, + "arrangement": 11874, + "gad": 11875, + "siber": 11876, + "especially": 11877, + "officers": 11878, + "forms": 11879, + "longer": 11880, + "blow": 11881, + "commission": 11882, + "pyramid": 11883, + "warning": 11884, + "appearance": 11885, + "upside": 11886, + "apply": 11887, + "vogue": 11888, + "hike": 11889, + "##oration": 11890, + "crystals": 11891, + "cork": 11892, + "proud": 11893, + "##ros": 11894, + "eiffel": 11895, + "wants": 11896, + "boyfriend": 11897, + "method": 11898, + "fc": 11899, + "##edic": 11900, + "spiders": 11901, + "grounds": 11902, + "rights": 11903, + "admiral": 11904, + "parl": 11905, + "upcoming": 11906, + "whiskey": 11907, + "dancer": 11908, + "sick": 11909, + "walnut": 11910, + "leaving": 11911, + "##dale": 11912, + "blooming": 11913, + "disp": 11914, + "edible": 11915, + "avatar": 11916, + "tus": 11917, + "authentic": 11918, + "formula": 11919, + "grown": 11920, + "nissan": 11921, + "##under": 11922, + "##rey": 11923, + "beetles": 11924, + "mann": 11925, + "glamour": 11926, + "ib": 11927, + "usually": 11928, + "retriever": 11929, + "holster": 11930, + "cuts": 11931, + "luc": 11932, + "##eries": 11933, + "moments": 11934, + "agency": 11935, + "benches": 11936, + "mustard": 11937, + "syn": 11938, + "previous": 11939, + "##low": 11940, + "tanks": 11941, + "lies": 11942, + "cert": 11943, + "##gets": 11944, + "##oosing": 11945, + "knives": 11946, + "volume": 11947, + "brings": 11948, + "subur": 11949, + "crying": 11950, + "edin": 11951, + "nearly": 11952, + "oat": 11953, + "spicy": 11954, + "39": 11955, + "prices": 11956, + "railroad": 11957, + "conver": 11958, + "shag": 11959, + "torch": 11960, + "sten": 11961, + "norway": 11962, + "voice": 11963, + "association": 11964, + "##ysis": 11965, + "##worth": 11966, + "advice": 11967, + "buddha": 11968, + "fiber": 11969, + "carts": 11970, + "spices": 11971, + "absol": 11972, + "mul": 11973, + "changes": 11974, + "casser": 11975, + "##jab": 11976, + "lt": 11977, + "contour": 11978, + "biscuit": 11979, + "gul": 11980, + "kart": 11981, + "pooh": 11982, + "petting": 11983, + "creepy": 11984, + "denver": 11985, + "bearded": 11986, + "ced": 11987, + "wicker": 11988, + "edinburgh": 11989, + "directly": 11990, + "headqu": 11991, + "coloured": 11992, + "lifting": 11993, + "survival": 11994, + "choosing": 11995, + "##uous": 11996, + "drug": 11997, + "##sey": 11998, + "pregnancy": 11999, + "##olia": 12000, + "##mod": 12001, + "bracelets": 12002, + "daisy": 12003, + "gong": 12004, + "##word": 12005, + "option": 12006, + "##esco": 12007, + "parallel": 12008, + "massive": 12009, + "##yr": 12010, + "skill": 12011, + "reprodu": 12012, + "peb": 12013, + "tai": 12014, + "tales": 12015, + "affordable": 12016, + "earl": 12017, + "damage": 12018, + "kil": 12019, + "seeing": 12020, + "mosquito": 12021, + "pirates": 12022, + "manic": 12023, + "tex": 12024, + "graduate": 12025, + "arte": 12026, + "sayings": 12027, + "flex": 12028, + "keeping": 12029, + "bite": 12030, + "currently": 12031, + "swims": 12032, + "strings": 12033, + "disease": 12034, + "condo": 12035, + "##x4": 12036, + "cultural": 12037, + "##oops": 12038, + "crate": 12039, + "rode": 12040, + "anything": 12041, + "herd": 12042, + "simps": 12043, + "refer": 12044, + "looked": 12045, + "goats": 12046, + "abbey": 12047, + "##ican": 12048, + "underside": 12049, + "passengers": 12050, + "buddhist": 12051, + "anemone": 12052, + "monochrome": 12053, + "##icult": 12054, + "meditation": 12055, + "fireworks": 12056, + "chili": 12057, + "reaching": 12058, + "ens": 12059, + "opens": 12060, + "flames": 12061, + "luxur": 12062, + "##off": 12063, + "##aca": 12064, + "adoptable": 12065, + "cuis": 12066, + "casserole": 12067, + "turns": 12068, + "punk": 12069, + "unc": 12070, + "peru": 12071, + "turk": 12072, + "lett": 12073, + "metrop": 12074, + "##ui": 12075, + "compon": 12076, + "happiness": 12077, + "soph": 12078, + "wax": 12079, + "49": 12080, + "categ": 12081, + "stew": 12082, + "effects": 12083, + "woodland": 12084, + "tomb": 12085, + "overall": 12086, + "pavil": 12087, + "##ulated": 12088, + "highest": 12089, + "haun": 12090, + "##ll": 12091, + "leban": 12092, + "goldfish": 12093, + "wad": 12094, + "certain": 12095, + "tide": 12096, + "stencil": 12097, + "sites": 12098, + "rover": 12099, + "duty": 12100, + "tissue": 12101, + "spinning": 12102, + "answer": 12103, + "##cement": 12104, + "mixing": 12105, + "squares": 12106, + "kunst": 12107, + "popcorn": 12108, + "##whe": 12109, + "##gress": 12110, + "hawk": 12111, + "jobs": 12112, + "##edo": 12113, + "facial": 12114, + "##hammer": 12115, + "delux": 12116, + "locom": 12117, + "3rd": 12118, + "gloss": 12119, + "sect": 12120, + "ties": 12121, + "bent": 12122, + "temperature": 12123, + "##vo": 12124, + "##quer": 12125, + "tux": 12126, + "retreat": 12127, + "alongside": 12128, + "skys": 12129, + "##27": 12130, + "##wall": 12131, + "taylor": 12132, + "clinic": 12133, + "vit": 12134, + "residents": 12135, + "skyscra": 12136, + "motif": 12137, + "sprink": 12138, + "thor": 12139, + "beng": 12140, + "balt": 12141, + "continue": 12142, + "##ledge": 12143, + "scenic": 12144, + "improve": 12145, + "remains": 12146, + "laid": 12147, + "##ruit": 12148, + "buckle": 12149, + "##alt": 12150, + "dangerous": 12151, + "tril": 12152, + "forum": 12153, + "##ani": 12154, + "landscapes": 12155, + "woven": 12156, + "deluxe": 12157, + "##fin": 12158, + "189": 12159, + "frank": 12160, + "##icester": 12161, + "analog": 12162, + "powered": 12163, + "outst": 12164, + "curry": 12165, + "radi": 12166, + "mars": 12167, + "non": 12168, + "blend": 12169, + "##oty": 12170, + "packed": 12171, + "weights": 12172, + "duvet": 12173, + "vibrant": 12174, + "wolves": 12175, + "claus": 12176, + "independence": 12177, + "crush": 12178, + "results": 12179, + "classy": 12180, + "operation": 12181, + "##usal": 12182, + "luxurious": 12183, + "jac": 12184, + "meets": 12185, + "psych": 12186, + "produced": 12187, + "religion": 12188, + "parachute": 12189, + "chance": 12190, + "rush": 12191, + "jackson": 12192, + "shock": 12193, + "cameras": 12194, + "lik": 12195, + "tiled": 12196, + "thunder": 12197, + "##ki": 12198, + "granite": 12199, + "highland": 12200, + "waterproof": 12201, + "headquarters": 12202, + "mansions": 12203, + "##don": 12204, + "dragons": 12205, + "wis": 12206, + "employee": 12207, + "##29": 12208, + "offering": 12209, + "gathered": 12210, + "##olk": 12211, + "browse": 12212, + "blogger": 12213, + "arrows": 12214, + "lover": 12215, + "harness": 12216, + "flick": 12217, + "reel": 12218, + "ott": 12219, + "sweat": 12220, + "sense": 12221, + "gain": 12222, + "istan": 12223, + "istanbul": 12224, + "solutions": 12225, + "beginners": 12226, + "wil": 12227, + "##lar": 12228, + "tapestry": 12229, + "inspire": 12230, + "exchange": 12231, + "cupboard": 12232, + "gathering": 12233, + "aust": 12234, + "stocking": 12235, + "##attan": 12236, + "2003": 12237, + "recycled": 12238, + "clocks": 12239, + "pag": 12240, + "consum": 12241, + "reflected": 12242, + "jama": 12243, + "romance": 12244, + "##asting": 12245, + "eco": 12246, + "usb": 12247, + "secrets": 12248, + "contrast": 12249, + "attention": 12250, + "shared": 12251, + "cardigan": 12252, + "##istics": 12253, + "lyrics": 12254, + "allows": 12255, + "dj": 12256, + "conservation": 12257, + "44": 12258, + "rick": 12259, + "fabulous": 12260, + "43": 12261, + "flood": 12262, + "##mill": 12263, + "barrier": 12264, + "reality": 12265, + "spea": 12266, + "partners": 12267, + "##erved": 12268, + "ros": 12269, + "##gate": 12270, + "trousers": 12271, + "##pan": 12272, + "##opes": 12273, + "pockets": 12274, + "additional": 12275, + "virtual": 12276, + "##amin": 12277, + "yummy": 12278, + "else": 12279, + "scra": 12280, + "plug": 12281, + "boeing": 12282, + "recording": 12283, + "princip": 12284, + "##wash": 12285, + "collabor": 12286, + "fail": 12287, + "completely": 12288, + "knights": 12289, + "trombone": 12290, + "chim": 12291, + "charming": 12292, + "steth": 12293, + "warhammer": 12294, + "jumps": 12295, + "required": 12296, + "harves": 12297, + "glu": 12298, + "application": 12299, + "lizards": 12300, + "turning": 12301, + "loving": 12302, + "decals": 12303, + "gauge": 12304, + "armored": 12305, + "hunter": 12306, + "coca": 12307, + "cells": 12308, + "##loaded": 12309, + "aur": 12310, + "core": 12311, + "shots": 12312, + "distrib": 12313, + "1950s": 12314, + "pollution": 12315, + "defense": 12316, + "##itors": 12317, + "myster": 12318, + "comedy": 12319, + "##selves": 12320, + "puffer": 12321, + "tack": 12322, + "alex": 12323, + "chin": 12324, + "##uge": 12325, + "ebay": 12326, + "mits": 12327, + "##cha": 12328, + "##escent": 12329, + "greens": 12330, + "stro": 12331, + "kneeling": 12332, + "hul": 12333, + "embra": 12334, + "gour": 12335, + "##ria": 12336, + "charco": 12337, + "cheerful": 12338, + "bord": 12339, + "stylized": 12340, + "pavilion": 12341, + "leicester": 12342, + "extension": 12343, + "03": 12344, + "nc": 12345, + "##oscope": 12346, + "sant": 12347, + "chill": 12348, + "waterfront": 12349, + "faced": 12350, + "poured": 12351, + "brunch": 12352, + "peel": 12353, + "anyone": 12354, + "kettle": 12355, + "fry": 12356, + "aired": 12357, + "##atable": 12358, + "fifth": 12359, + "exploring": 12360, + "peanuts": 12361, + "venom": 12362, + "##ingham": 12363, + "dolphins": 12364, + "nepal": 12365, + "alley": 12366, + "##aha": 12367, + "engineer": 12368, + "uph": 12369, + "##23": 12370, + "patrol": 12371, + "parliam": 12372, + "talks": 12373, + "instructions": 12374, + "perfectly": 12375, + "except": 12376, + "curved": 12377, + "pitts": 12378, + "henna": 12379, + "charcoal": 12380, + "flowing": 12381, + "isle": 12382, + "domestic": 12383, + "espress": 12384, + "argent": 12385, + "marathon": 12386, + "smash": 12387, + "snowmobile": 12388, + "napkin": 12389, + "friendship": 12390, + "athletic": 12391, + "parliament": 12392, + "shaker": 12393, + "winnie": 12394, + "emir": 12395, + "cedar": 12396, + "task": 12397, + "blankets": 12398, + "##watch": 12399, + "foxes": 12400, + "root": 12401, + "##sticks": 12402, + "plenty": 12403, + "poet": 12404, + "merc": 12405, + "mattress": 12406, + "keeps": 12407, + "poor": 12408, + "sov": 12409, + "bbq": 12410, + "##iction": 12411, + "communication": 12412, + "illustrator": 12413, + "ore": 12414, + "figs": 12415, + "suede": 12416, + "saints": 12417, + "pd": 12418, + "##usalem": 12419, + "jerusalem": 12420, + "##haw": 12421, + "##ream": 12422, + "siberian": 12423, + "chainsaw": 12424, + "shaggy": 12425, + "188": 12426, + "increase": 12427, + "forget": 12428, + "cuisine": 12429, + "groups": 12430, + "nin": 12431, + "##urai": 12432, + "virgin": 12433, + "depth": 12434, + "flamingo": 12435, + "##station": 12436, + "analysis": 12437, + "treasure": 12438, + "grape": 12439, + "changed": 12440, + "portfol": 12441, + "puzzles": 12442, + "flee": 12443, + "##bit": 12444, + "drone": 12445, + "##etime": 12446, + "haircut": 12447, + "expect": 12448, + "arcade": 12449, + "penins": 12450, + "cane": 12451, + "fishes": 12452, + "##sive": 12453, + "burr": 12454, + "rows": 12455, + "##flix": 12456, + "lush": 12457, + "birdhouse": 12458, + "netflix": 12459, + "user": 12460, + "insurance": 12461, + "toe": 12462, + "han": 12463, + "##hap": 12464, + "lonely": 12465, + "worked": 12466, + "bond": 12467, + "convention": 12468, + "tractors": 12469, + "damaged": 12470, + "tact": 12471, + "starry": 12472, + "hydr": 12473, + "finally": 12474, + "brig": 12475, + "##sex": 12476, + "atmosphere": 12477, + "osc": 12478, + "peas": 12479, + "frosting": 12480, + "upgra": 12481, + "mason": 12482, + "require": 12483, + "appet": 12484, + "ukra": 12485, + "blueberry": 12486, + "##21": 12487, + "##asis": 12488, + "planter": 12489, + "spra": 12490, + "rugs": 12491, + "impressive": 12492, + "inspiring": 12493, + "mole": 12494, + "indig": 12495, + "probos": 12496, + "150": 12497, + "launched": 12498, + "disneyland": 12499, + "haven": 12500, + "overcoat": 12501, + "ww": 12502, + "mous": 12503, + "anth": 12504, + "savannah": 12505, + "pantry": 12506, + "wizard": 12507, + "1960s": 12508, + "update": 12509, + "mig": 12510, + "indoors": 12511, + "##weed": 12512, + "raising": 12513, + "##77": 12514, + "pleas": 12515, + "watermelon": 12516, + "violet": 12517, + "grin": 12518, + "oldest": 12519, + "syrup": 12520, + "terminal": 12521, + "##hold": 12522, + "surreal": 12523, + "##minster": 12524, + "jellyfish": 12525, + "shawl": 12526, + "##iance": 12527, + "pixel": 12528, + "opener": 12529, + "rays": 12530, + "challeng": 12531, + "documents": 12532, + "hulk": 12533, + "stam": 12534, + "engraved": 12535, + "chasing": 12536, + "midi": 12537, + "bris": 12538, + "william": 12539, + "transl": 12540, + "eagles": 12541, + "maxim": 12542, + "sundial": 12543, + "bells": 12544, + "controls": 12545, + "dent": 12546, + "##ista": 12547, + "ledge": 12548, + "anton": 12549, + "disk": 12550, + "adapt": 12551, + "containing": 12552, + "bh": 12553, + "cannot": 12554, + "belong": 12555, + "figurine": 12556, + "leis": 12557, + "dwar": 12558, + "purchase": 12559, + "knowledge": 12560, + "creates": 12561, + "schn": 12562, + "stereo": 12563, + "soviet": 12564, + "titan": 12565, + "integr": 12566, + "dancers": 12567, + "cere": 12568, + "muffins": 12569, + "baboa": 12570, + "pans": 12571, + "wiring": 12572, + "ruff": 12573, + "surprise": 12574, + "##wa": 12575, + "##bons": 12576, + "stork": 12577, + "corv": 12578, + "mirrors": 12579, + "quilts": 12580, + "skating": 12581, + "1970s": 12582, + "brochure": 12583, + "levels": 12584, + "microsc": 12585, + "phr": 12586, + "prevent": 12587, + "riv": 12588, + "##olitan": 12589, + "plum": 12590, + "##mat": 12591, + "poles": 12592, + "maintain": 12593, + "##empt": 12594, + "limb": 12595, + "weigh": 12596, + "##ibility": 12597, + "dating": 12598, + "##ris": 12599, + "designers": 12600, + "rav": 12601, + "##obe": 12602, + "sustainable": 12603, + "vendor": 12604, + "surfing": 12605, + "trails": 12606, + "quilting": 12607, + "beaks": 12608, + "##cier": 12609, + "bodybu": 12610, + "gender": 12611, + "worst": 12612, + "guardians": 12613, + "sin": 12614, + "##ibly": 12615, + "manufactur": 12616, + "lambor": 12617, + "glacier": 12618, + "pakistani": 12619, + "particular": 12620, + "##exp": 12621, + "om": 12622, + "pittsburgh": 12623, + "lamborgh": 12624, + "cuff": 12625, + "ferry": 12626, + "oxford": 12627, + "manhattan": 12628, + "shrine": 12629, + "##attered": 12630, + "##udi": 12631, + "zombie": 12632, + "geek": 12633, + "hundreds": 12634, + "boom": 12635, + "retrieve": 12636, + "##ste": 12637, + "however": 12638, + "scroll": 12639, + "classes": 12640, + "patterned": 12641, + "##press": 12642, + "##ties": 12643, + "meters": 12644, + "picks": 12645, + "peninsula": 12646, + "##ells": 12647, + "saving": 12648, + "##ician": 12649, + "##airs": 12650, + "portland": 12651, + "hyen": 12652, + "ench": 12653, + "officials": 12654, + "##ancing": 12655, + "duke": 12656, + "hen": 12657, + "snowfla": 12658, + "##boats": 12659, + "exact": 12660, + "reind": 12661, + "rusted": 12662, + "shares": 12663, + "locomotive": 12664, + "lamborghini": 12665, + "tigers": 12666, + "died": 12667, + "gave": 12668, + "lose": 12669, + "lucky": 12670, + "wii": 12671, + "##erson": 12672, + "listening": 12673, + "koi": 12674, + "##roid": 12675, + "marines": 12676, + "nick": 12677, + "wireless": 12678, + "twil": 12679, + "emirates": 12680, + "leisure": 12681, + "monogram": 12682, + "organism": 12683, + "hav": 12684, + "twilight": 12685, + "oh": 12686, + "killed": 12687, + "chalkboard": 12688, + "understand": 12689, + "speaking": 12690, + "rou": 12691, + "combine": 12692, + "broadway": 12693, + "crossword": 12694, + "spacious": 12695, + "sails": 12696, + "enh": 12697, + "spinach": 12698, + "partner": 12699, + "repla": 12700, + "##ko": 12701, + "austria": 12702, + "47": 12703, + "##abad": 12704, + "bros": 12705, + "##65": 12706, + "femin": 12707, + "representing": 12708, + "venues": 12709, + "reh": 12710, + "motors": 12711, + "lie": 12712, + "capit": 12713, + "skirts": 12714, + "goggles": 12715, + "glen": 12716, + "sweets": 12717, + "turkish": 12718, + "muzzle": 12719, + "pallet": 12720, + "formation": 12721, + "favourite": 12722, + "crest": 12723, + "maid": 12724, + "sweden": 12725, + "nash": 12726, + "wrong": 12727, + "extreme": 12728, + "surgery": 12729, + "connection": 12730, + "dinosaurs": 12731, + "##encing": 12732, + "shoulders": 12733, + "latt": 12734, + "supplied": 12735, + "cant": 12736, + "westminster": 12737, + "##ellig": 12738, + "manicure": 12739, + "##bel": 12740, + "considered": 12741, + "cottages": 12742, + "homest": 12743, + "dye": 12744, + "dumb": 12745, + "keto": 12746, + "pumps": 12747, + "steering": 12748, + "exped": 12749, + "difficult": 12750, + "antenna": 12751, + "drain": 12752, + "samurai": 12753, + "cair": 12754, + "yurt": 12755, + "##bod": 12756, + "panther": 12757, + "##tra": 12758, + "austin": 12759, + "nuc": 12760, + "tilted": 12761, + "recon": 12762, + "ramp": 12763, + "hose": 12764, + "##emic": 12765, + "banners": 12766, + "viewed": 12767, + "dams": 12768, + "tired": 12769, + "tulle": 12770, + "lebanon": 12771, + "admin": 12772, + "antenn": 12773, + "##veland": 12774, + "meg": 12775, + "heater": 12776, + "privacy": 12777, + "##eller": 12778, + "devices": 12779, + "nursing": 12780, + "##inery": 12781, + "tram": 12782, + "solo": 12783, + "##inct": 12784, + "##iang": 12785, + "biker": 12786, + "rabbits": 12787, + "dwarf": 12788, + "bc": 12789, + "highlights": 12790, + "interest": 12791, + "scheme": 12792, + "ets": 12793, + "##wide": 12794, + "2001": 12795, + "respons": 12796, + "recover": 12797, + "cube": 12798, + "##bras": 12799, + "womb": 12800, + "##rys": 12801, + "shab": 12802, + "martin": 12803, + "rises": 12804, + "cleveland": 12805, + "subject": 12806, + "quarantine": 12807, + "chunk": 12808, + "intellig": 12809, + "reindeer": 12810, + "organized": 12811, + "garn": 12812, + "published": 12813, + "refresh": 12814, + "##pse": 12815, + "##fa": 12816, + "##mingham": 12817, + "highly": 12818, + "charity": 12819, + "dynam": 12820, + "camels": 12821, + "qual": 12822, + "fashionable": 12823, + "bour": 12824, + "flock": 12825, + "padd": 12826, + "throne": 12827, + "ultra": 12828, + "65": 12829, + "##ho": 12830, + "weasel": 12831, + "turbo": 12832, + "normal": 12833, + "endanger": 12834, + "nucle": 12835, + "regional": 12836, + "sailor": 12837, + "mantisce": 12838, + "drives": 12839, + "developed": 12840, + "hawai": 12841, + "thigh": 12842, + "##imb": 12843, + "sched": 12844, + "torn": 12845, + "cambridge": 12846, + "himself": 12847, + "percent": 12848, + "introduction": 12849, + "etsy": 12850, + "##vis": 12851, + "tender": 12852, + "received": 12853, + "touching": 12854, + "64": 12855, + "records": 12856, + "expedition": 12857, + "ft": 12858, + "siding": 12859, + "##meal": 12860, + "astron": 12861, + "##ico": 12862, + "coupe": 12863, + "fleet": 12864, + "apocal": 12865, + "jay": 12866, + "promo": 12867, + "encou": 12868, + "aged": 12869, + "desp": 12870, + "simpsons": 12871, + "##iences": 12872, + "endangered": 12873, + "04": 12874, + "coal": 12875, + "bungalow": 12876, + "creat": 12877, + "birmingham": 12878, + "inflatable": 12879, + "nj": 12880, + "trainer": 12881, + "signal": 12882, + "textures": 12883, + "showcasing": 12884, + "##mine": 12885, + "##ects": 12886, + "begins": 12887, + "studies": 12888, + "initial": 12889, + "cranes": 12890, + "destroy": 12891, + "whit": 12892, + "extrem": 12893, + "pushing": 12894, + "poland": 12895, + "gesture": 12896, + "##eston": 12897, + "org": 12898, + "padlock": 12899, + "flowering": 12900, + "hits": 12901, + "canon": 12902, + "clo": 12903, + "blackboard": 12904, + "##acular": 12905, + "sneaker": 12906, + "send": 12907, + "pads": 12908, + "##anted": 12909, + "antonio": 12910, + "##ori": 12911, + "phant": 12912, + "manual": 12913, + "cove": 12914, + "gentleman": 12915, + "instru": 12916, + "appe": 12917, + "##cloth": 12918, + "##iking": 12919, + "##brid": 12920, + "hog": 12921, + "collector": 12922, + "alt": 12923, + "vacations": 12924, + "09": 12925, + "gum": 12926, + "nights": 12927, + "portfolio": 12928, + "nord": 12929, + "restoration": 12930, + "replica": 12931, + "vari": 12932, + "shabby": 12933, + "##away": 12934, + "##35": 12935, + "pharm": 12936, + "##ras": 12937, + "##zed": 12938, + "##37": 12939, + "potted": 12940, + "lobsters": 12941, + "yogurt": 12942, + "gyps": 12943, + "aid": 12944, + "warth": 12945, + "register": 12946, + "railing": 12947, + "##26": 12948, + "expert": 12949, + "##aro": 12950, + "hawaiian": 12951, + "attic": 12952, + "illusion": 12953, + "population": 12954, + "candid": 12955, + "berry": 12956, + "pancakes": 12957, + "gram": 12958, + "zebras": 12959, + "brill": 12960, + "gam": 12961, + "suv": 12962, + "loop": 12963, + "hack": 12964, + "owned": 12965, + "##through": 12966, + "seag": 12967, + "rhino": 12968, + "heron": 12969, + "##itely": 12970, + "ill": 12971, + "xbox": 12972, + "grave": 12973, + "ela": 12974, + "excited": 12975, + "trips": 12976, + "fern": 12977, + "moved": 12978, + "coated": 12979, + "cooper": 12980, + "glas": 12981, + "chamel": 12982, + "08": 12983, + "mis": 12984, + "numer": 12985, + "origami": 12986, + "calf": 12987, + "##rum": 12988, + "thoughts": 12989, + "nautical": 12990, + "weapon": 12991, + "##horn": 12992, + "tibetan": 12993, + "##uses": 12994, + "taiwan": 12995, + "showroom": 12996, + "crafted": 12997, + "##face": 12998, + "sab": 12999, + "daytime": 13000, + "alpine": 13001, + "conditions": 13002, + "sleeved": 13003, + "thatched": 13004, + "dimensions": 13005, + "swirl": 13006, + "triple": 13007, + "risk": 13008, + "vod": 13009, + "fleece": 13010, + "seated": 13011, + "pavement": 13012, + "delight": 13013, + "##leg": 13014, + "contrib": 13015, + "pastry": 13016, + "injured": 13017, + "junk": 13018, + "auditor": 13019, + "rectangular": 13020, + "deg": 13021, + "wrestling": 13022, + "gene": 13023, + "constell": 13024, + "##illet": 13025, + "hoop": 13026, + "##oom": 13027, + "##ications": 13028, + "tentacles": 13029, + "paleo": 13030, + "negative": 13031, + "picking": 13032, + "##udd": 13033, + "healing": 13034, + "writer": 13035, + "falcon": 13036, + "baham": 13037, + "trol": 13038, + "zen": 13039, + "playstation": 13040, + "##lings": 13041, + "##asts": 13042, + "dober": 13043, + "##gow": 13044, + "explos": 13045, + "##osh": 13046, + "revolver": 13047, + "apart": 13048, + "gingerbread": 13049, + "laz": 13050, + "coco": 13051, + "repeat": 13052, + "proposal": 13053, + "stom": 13054, + "swift": 13055, + "lifted": 13056, + "accommod": 13057, + "farming": 13058, + "dedic": 13059, + "discount": 13060, + "hare": 13061, + "param": 13062, + "07": 13063, + "uphol": 13064, + "trout": 13065, + "oyster": 13066, + "heights": 13067, + "employees": 13068, + "peters": 13069, + "llam": 13070, + "citiz": 13071, + "conversation": 13072, + "domin": 13073, + "hobbit": 13074, + "##oir": 13075, + "watercolour": 13076, + "##wick": 13077, + "##thus": 13078, + "delicate": 13079, + "gadgets": 13080, + "rate": 13081, + "taco": 13082, + "teachers": 13083, + "moves": 13084, + "rica": 13085, + "avoid": 13086, + "##edia": 13087, + "##ima": 13088, + "racket": 13089, + "orca": 13090, + "topping": 13091, + "invent": 13092, + "##woman": 13093, + "bons": 13094, + "##aring": 13095, + "ape": 13096, + "stag": 13097, + "naval": 13098, + "cabins": 13099, + "##cal": 13100, + "sear": 13101, + "##yst": 13102, + "external": 13103, + "explorer": 13104, + "aurora": 13105, + "hex": 13106, + "iii": 13107, + "spend": 13108, + "oatmeal": 13109, + "array": 13110, + "proof": 13111, + "layered": 13112, + "##izes": 13113, + "bumper": 13114, + "52": 13115, + "##ru": 13116, + "transition": 13117, + "dread": 13118, + "brigade": 13119, + "cem": 13120, + "hers": 13121, + "discovered": 13122, + "transfer": 13123, + "effective": 13124, + "signific": 13125, + "yam": 13126, + "##dog": 13127, + "##mary": 13128, + "2002": 13129, + "gaze": 13130, + "tu": 13131, + "05": 13132, + "carried": 13133, + "pinwhe": 13134, + "firm": 13135, + "haunted": 13136, + "##oard": 13137, + "swinging": 13138, + "##elled": 13139, + "cylinder": 13140, + "hire": 13141, + "accident": 13142, + "curb": 13143, + "calcul": 13144, + "issues": 13145, + "nik": 13146, + "##yl": 13147, + "mothers": 13148, + "strike": 13149, + "castles": 13150, + "outlet": 13151, + "mario": 13152, + "lagoon": 13153, + "jail": 13154, + "grooming": 13155, + "personalised": 13156, + "pointer": 13157, + "opportunity": 13158, + "ts": 13159, + "wins": 13160, + "rather": 13161, + "ses": 13162, + "raspberry": 13163, + "archive": 13164, + "pray": 13165, + "ivy": 13166, + "##ml": 13167, + "70s": 13168, + "poison": 13169, + "hilton": 13170, + "moda": 13171, + "icing": 13172, + "antiques": 13173, + "girlfriend": 13174, + "bomber": 13175, + "phantom": 13176, + "omb": 13177, + "cc": 13178, + "began": 13179, + "scales": 13180, + "##asth": 13181, + "necklaces": 13182, + "internal": 13183, + "locations": 13184, + "raf": 13185, + "races": 13186, + "davids": 13187, + "##47": 13188, + "scan": 13189, + "mrs": 13190, + "extremely": 13191, + "gates": 13192, + "oklah": 13193, + "wik": 13194, + "alth": 13195, + "parrots": 13196, + "chestnut": 13197, + "f1": 13198, + "maybe": 13199, + "interactive": 13200, + "##wards": 13201, + "delta": 13202, + "assign": 13203, + "##reci": 13204, + "organs": 13205, + "athletes": 13206, + "cuba": 13207, + "glasgow": 13208, + "rip": 13209, + "kenn": 13210, + "ni": 13211, + "testing": 13212, + "ribbons": 13213, + "auditorium": 13214, + "heading": 13215, + "worship": 13216, + "ping": 13217, + "mit": 13218, + "##icles": 13219, + "starts": 13220, + "allowed": 13221, + "rajasth": 13222, + "crime": 13223, + "ends": 13224, + "##arees": 13225, + "announced": 13226, + "pearls": 13227, + "correct": 13228, + "##bro": 13229, + "massage": 13230, + "sisters": 13231, + "unve": 13232, + "seaside": 13233, + "editorial": 13234, + "chron": 13235, + "signage": 13236, + "carav": 13237, + "##edies": 13238, + "collections": 13239, + "personality": 13240, + "tulip": 13241, + "styled": 13242, + "motorb": 13243, + "ombre": 13244, + "fring": 13245, + "modified": 13246, + "behav": 13247, + "receiver": 13248, + "shrubs": 13249, + "mong": 13250, + "##anna": 13251, + "advanced": 13252, + "mold": 13253, + "effic": 13254, + "veil": 13255, + "sniff": 13256, + "platter": 13257, + "xmas": 13258, + "chrys": 13259, + "46": 13260, + "doberman": 13261, + "##punk": 13262, + "harmon": 13263, + "bowling": 13264, + "##girl": 13265, + "gap": 13266, + "lenses": 13267, + "##keeper": 13268, + "sud": 13269, + "bonsai": 13270, + "##orous": 13271, + "##eran": 13272, + "although": 13273, + "client": 13274, + "##zech": 13275, + "pv": 13276, + "dishwash": 13277, + "ran": 13278, + "##bane": 13279, + "doorway": 13280, + "vessel": 13281, + "cyberpunk": 13282, + "jur": 13283, + "basin": 13284, + "##ablish": 13285, + "eel": 13286, + "guac": 13287, + "niger": 13288, + "reflecting": 13289, + "ka": 13290, + "concepts": 13291, + "reverse": 13292, + "les": 13293, + "info": 13294, + "##uel": 13295, + "seoul": 13296, + "hairstyle": 13297, + "##maids": 13298, + "armchair": 13299, + "stan": 13300, + "beanie": 13301, + "41": 13302, + "painter": 13303, + "cooling": 13304, + "airplanes": 13305, + "1970": 13306, + "cooler": 13307, + "capture": 13308, + "ninja": 13309, + "##igation": 13310, + "beverage": 13311, + "##osa": 13312, + "hybrid": 13313, + "establish": 13314, + "5th": 13315, + "orth": 13316, + "kra": 13317, + "simpl": 13318, + "rs": 13319, + "calling": 13320, + "assistant": 13321, + "rag": 13322, + "select": 13323, + "tribe": 13324, + "fits": 13325, + "sarees": 13326, + "##char": 13327, + "fisherman": 13328, + "llama": 13329, + "360": 13330, + "proposed": 13331, + "corvette": 13332, + "fungi": 13333, + "beams": 13334, + "sush": 13335, + "arrive": 13336, + "sandwiches": 13337, + "##asty": 13338, + "##ishing": 13339, + "sep": 13340, + "cemet": 13341, + "##ollo": 13342, + "higher": 13343, + "definition": 13344, + "davidson": 13345, + "float": 13346, + "##rens": 13347, + "technique": 13348, + "##unt": 13349, + "sailboats": 13350, + "suggest": 13351, + "themselves": 13352, + "restored": 13353, + "category": 13354, + "wrapping": 13355, + "nashville": 13356, + "removal": 13357, + "##di": 13358, + "seem": 13359, + "washed": 13360, + "scout": 13361, + "accept": 13362, + "gulf": 13363, + "armed": 13364, + "dup": 13365, + "further": 13366, + "amphib": 13367, + "shine": 13368, + "oxy": 13369, + "contents": 13370, + "oversized": 13371, + "checking": 13372, + "average": 13373, + "strateg": 13374, + "bengal": 13375, + "persons": 13376, + "measures": 13377, + "cluster": 13378, + "federal": 13379, + "administ": 13380, + "necess": 13381, + "chal": 13382, + "fabrics": 13383, + "incor": 13384, + "podium": 13385, + "lanterns": 13386, + "mp": 13387, + "robes": 13388, + "vegetation": 13389, + "wisdom": 13390, + "zucchini": 13391, + "##oque": 13392, + "##icol": 13393, + "sage": 13394, + "##alg": 13395, + "600": 13396, + "##fi": 13397, + "labyr": 13398, + "beneath": 13399, + "trolley": 13400, + "cemetery": 13401, + "striking": 13402, + "greetings": 13403, + "labyrinth": 13404, + "quickly": 13405, + "alask": 13406, + "wav": 13407, + "frogs": 13408, + "tulips": 13409, + "cleaned": 13410, + "kept": 13411, + "charge": 13412, + "##mission": 13413, + "##ented": 13414, + "marching": 13415, + "dior": 13416, + "meatball": 13417, + "##pit": 13418, + "##lington": 13419, + "windsor": 13420, + "dedicated": 13421, + "##22": 13422, + "farms": 13423, + "prague": 13424, + "czech": 13425, + "##36": 13426, + "##elle": 13427, + "lifetime": 13428, + "mechanical": 13429, + "##esla": 13430, + "android": 13431, + "##vement": 13432, + "attraction": 13433, + "carries": 13434, + "hooded": 13435, + "##34": 13436, + "##x10": 13437, + "converse": 13438, + "queensland": 13439, + "##ometry": 13440, + "spectacular": 13441, + "brilliant": 13442, + "waffle": 13443, + "##ifier": 13444, + "marmot": 13445, + "##atti": 13446, + "ome": 13447, + "##soft": 13448, + "tough": 13449, + "dishwasher": 13450, + "##anny": 13451, + "##lyph": 13452, + "pigeon": 13453, + "vodka": 13454, + "congr": 13455, + "croat": 13456, + "raise": 13457, + "swedish": 13458, + "ranger": 13459, + "##ighters": 13460, + "##mo": 13461, + "handwritten": 13462, + "photographers": 13463, + "cul": 13464, + "##icip": 13465, + "bernard": 13466, + "trium": 13467, + "sedan": 13468, + "sole": 13469, + "watering": 13470, + "williams": 13471, + "##uese": 13472, + "##edi": 13473, + "bristol": 13474, + "portuguese": 13475, + "appreci": 13476, + "cushions": 13477, + "bulls": 13478, + "depot": 13479, + "combined": 13480, + "tribute": 13481, + "clad": 13482, + "##eas": 13483, + "firefighter": 13484, + "apollo": 13485, + "adds": 13486, + "quart": 13487, + "##enh": 13488, + "tones": 13489, + "ticket": 13490, + "bead": 13491, + "tarot": 13492, + "orland": 13493, + "wadi": 13494, + "oxygen": 13495, + "##win": 13496, + "##umbs": 13497, + "definitely": 13498, + "ri": 13499, + "martial": 13500, + "nh": 13501, + "panoramic": 13502, + "pdf": 13503, + "magnific": 13504, + "ct": 13505, + "##omme": 13506, + "posts": 13507, + "##75": 13508, + "##cat": 13509, + "remedies": 13510, + "cray": 13511, + "equal": 13512, + "ostric": 13513, + "##iza": 13514, + "grav": 13515, + "knock": 13516, + "aqua": 13517, + "leadership": 13518, + "headband": 13519, + "southeast": 13520, + "donkey": 13521, + "dive": 13522, + "peeking": 13523, + "feminine": 13524, + "##inis": 13525, + "originally": 13526, + "lilac": 13527, + "wasp": 13528, + "plains": 13529, + "represents": 13530, + "tees": 13531, + "hosted": 13532, + "recorder": 13533, + "##onia": 13534, + "flavors": 13535, + "firefighters": 13536, + "intric": 13537, + "cambod": 13538, + "essent": 13539, + "doodles": 13540, + "alber": 13541, + "respon": 13542, + "##occo": 13543, + "seems": 13544, + "theory": 13545, + "##ira": 13546, + "crim": 13547, + "setter": 13548, + "athens": 13549, + "stomach": 13550, + "##quis": 13551, + "degree": 13552, + "mick": 13553, + "##tail": 13554, + "useful": 13555, + "venet": 13556, + "ingredient": 13557, + "routine": 13558, + "wonders": 13559, + "recomme": 13560, + "childrens": 13561, + "graduates": 13562, + "magazines": 13563, + "spacesh": 13564, + "properties": 13565, + "walkthrough": 13566, + "##quets": 13567, + "blurry": 13568, + "inh": 13569, + "airship": 13570, + "thistle": 13571, + "##inum": 13572, + "latin": 13573, + "igua": 13574, + "arches": 13575, + "orlando": 13576, + "belg": 13577, + "rhin": 13578, + "demand": 13579, + "brisbane": 13580, + "##mouth": 13581, + "crochee": 13582, + "rods": 13583, + "ry": 13584, + "decided": 13585, + "earlier": 13586, + "textiles": 13587, + "loading": 13588, + "helped": 13589, + "components": 13590, + "winners": 13591, + "passes": 13592, + "montreal": 13593, + "##assic": 13594, + "cosmetic": 13595, + "thumbn": 13596, + "humans": 13597, + "aboard": 13598, + "swans": 13599, + "wat": 13600, + "messy": 13601, + "rule": 13602, + "kent": 13603, + "agriculture": 13604, + "atop": 13605, + "terrain": 13606, + "##odes": 13607, + "dual": 13608, + "pon": 13609, + "##pings": 13610, + "##mings": 13611, + "newest": 13612, + "bookcase": 13613, + "camps": 13614, + "gi": 13615, + "wand": 13616, + "strate": 13617, + "uploaded": 13618, + "machinery": 13619, + "professor": 13620, + "displaying": 13621, + "##alsa": 13622, + "presidential": 13623, + "receive": 13624, + "constructed": 13625, + "broo": 13626, + "surge": 13627, + "wake": 13628, + "selective": 13629, + "##bus": 13630, + "oklahoman": 13631, + "listen": 13632, + "finals": 13633, + "bridges": 13634, + "hamm": 13635, + "shelving": 13636, + "discovery": 13637, + "diary": 13638, + "06": 13639, + "lone": 13640, + "sushi": 13641, + "strategy": 13642, + "##ipur": 13643, + "reefs": 13644, + "creativity": 13645, + "##oopy": 13646, + "moose": 13647, + "salv": 13648, + "bigh": 13649, + "halo": 13650, + "charger": 13651, + "##laimed": 13652, + "timeless": 13653, + "##ault": 13654, + "relaxed": 13655, + "ner": 13656, + "##usion": 13657, + "##ifting": 13658, + "##ression": 13659, + "capitol": 13660, + "nations": 13661, + "oasis": 13662, + "verse": 13663, + "lun": 13664, + "optical": 13665, + "stup": 13666, + "amuse": 13667, + "dyn": 13668, + "bighorn": 13669, + "101": 13670, + "dandel": 13671, + "closer": 13672, + "ti": 13673, + "perth": 13674, + "spotlight": 13675, + "airline": 13676, + "drivers": 13677, + "footprint": 13678, + "sapph": 13679, + "barbec": 13680, + "highlands": 13681, + "airman": 13682, + "arsen": 13683, + "thumbnail": 13684, + "ark": 13685, + "mane": 13686, + "##rical": 13687, + "bahamas": 13688, + "microscope": 13689, + "##imately": 13690, + "sesame": 13691, + "1000": 13692, + "lived": 13693, + "survive": 13694, + "dumbs": 13695, + "recreation": 13696, + "dove": 13697, + "mesa": 13698, + "phase": 13699, + "truly": 13700, + "judge": 13701, + "lava": 13702, + "leh": 13703, + "occup": 13704, + "elabor": 13705, + "billi": 13706, + "californ": 13707, + "reptiles": 13708, + "tah": 13709, + "##oric": 13710, + "planner": 13711, + "jamaica": 13712, + "jigs": 13713, + "auck": 13714, + "totally": 13715, + "auckland": 13716, + "nova": 13717, + "##bill": 13718, + "whose": 13719, + "##upt": 13720, + "##99": 13721, + "mayor": 13722, + "greyhound": 13723, + "practical": 13724, + "dates": 13725, + "lee": 13726, + "paints": 13727, + "##borough": 13728, + "gypsy": 13729, + "jewish": 13730, + "neb": 13731, + "dipl": 13732, + "##com": 13733, + "camo": 13734, + "##rons": 13735, + "euro": 13736, + "aren": 13737, + "17th": 13738, + "nerd": 13739, + "groundhog": 13740, + "##nel": 13741, + "reduce": 13742, + "mickey": 13743, + "stopped": 13744, + "plot": 13745, + "triumph": 13746, + "belgium": 13747, + "aquatic": 13748, + "commander": 13749, + "pandemic": 13750, + "blossoms": 13751, + "metropolitan": 13752, + "offices": 13753, + "mut": 13754, + "westin": 13755, + "lex": 13756, + "sniffing": 13757, + "abdom": 13758, + "posted": 13759, + "easier": 13760, + "arrival": 13761, + "spent": 13762, + "siam": 13763, + "ballroom": 13764, + "prize": 13765, + "66": 13766, + "attempt": 13767, + "volunteer": 13768, + "##father": 13769, + "eyel": 13770, + "rectangle": 13771, + "##bir": 13772, + "##aces": 13773, + "ja": 13774, + "##eria": 13775, + "nost": 13776, + "morocco": 13777, + "capac": 13778, + "associated": 13779, + "absolut": 13780, + "bio": 13781, + "fet": 13782, + "goss": 13783, + "studying": 13784, + "gecko": 13785, + "crescent": 13786, + "denmark": 13787, + "poem": 13788, + "archae": 13789, + "emper": 13790, + "md": 13791, + "saddle": 13792, + "carl": 13793, + "##45": 13794, + "##eeps": 13795, + "##ouses": 13796, + "northwest": 13797, + "whistle": 13798, + "schemes": 13799, + "opposite": 13800, + "fitting": 13801, + "notice": 13802, + "##scape": 13803, + "wildfl": 13804, + "sphere": 13805, + "raven": 13806, + "##pad": 13807, + "##put": 13808, + "swords": 13809, + "oakland": 13810, + "##aco": 13811, + "whisky": 13812, + "stalk": 13813, + "inclusive": 13814, + "prehist": 13815, + "bows": 13816, + "saudi": 13817, + "experiment": 13818, + "reserv": 13819, + "legacy": 13820, + "##net": 13821, + "reven": 13822, + "coiled": 13823, + "chopped": 13824, + "cucums": 13825, + "inject": 13826, + "platinum": 13827, + "foreign": 13828, + "dame": 13829, + "pedestrian": 13830, + "seaweed": 13831, + "countertop": 13832, + "##uki": 13833, + "##utan": 13834, + "ss": 13835, + "importance": 13836, + "##48": 13837, + "moist": 13838, + "monitors": 13839, + "salsa": 13840, + "guess": 13841, + "phrase": 13842, + "carol": 13843, + "persian": 13844, + "formed": 13845, + "##stein": 13846, + "newf": 13847, + "teens": 13848, + "archives": 13849, + "expected": 13850, + "becoming": 13851, + "gin": 13852, + "celebrations": 13853, + "mythical": 13854, + "locks": 13855, + "spy": 13856, + "flats": 13857, + "recycling": 13858, + "donuts": 13859, + "95": 13860, + "portion": 13861, + "skillet": 13862, + "casa": 13863, + "bringing": 13864, + "independent": 13865, + "needles": 13866, + "dunes": 13867, + "##porting": 13868, + "neat": 13869, + "existing": 13870, + "planters": 13871, + "steep": 13872, + "backpacks": 13873, + "tob": 13874, + "paras": 13875, + "##inate": 13876, + "battles": 13877, + "geometry": 13878, + "returns": 13879, + "vulture": 13880, + "grouse": 13881, + "mixer": 13882, + "potential": 13883, + "sounds": 13884, + "told": 13885, + "##itory": 13886, + "turb": 13887, + "petersburg": 13888, + "puck": 13889, + "showcase": 13890, + "snoopy": 13891, + "incorpor": 13892, + "pont": 13893, + "lemurs": 13894, + "cinder": 13895, + "beautifully": 13896, + "bats": 13897, + "ornamental": 13898, + "hollow": 13899, + "automotive": 13900, + "twig": 13901, + "hippie": 13902, + "amusement": 13903, + "arsenal": 13904, + "echid": 13905, + "nostalg": 13906, + "lis": 13907, + "fame": 13908, + "##inous": 13909, + "newfound": 13910, + "legendary": 13911, + "howl": 13912, + "lionfish": 13913, + "reviews": 13914, + "creations": 13915, + "advant": 13916, + "borders": 13917, + "##ilding": 13918, + "##ennial": 13919, + "copyright": 13920, + "##bucks": 13921, + "becomes": 13922, + "antennae": 13923, + "sprou": 13924, + "swamp": 13925, + "skulls": 13926, + "souven": 13927, + "concern": 13928, + "resin": 13929, + "baroque": 13930, + "##leep": 13931, + "jurassic": 13932, + "brave": 13933, + "chocolates": 13934, + "mechanic": 13935, + "##shirt": 13936, + "elegance": 13937, + "creator": 13938, + "slog": 13939, + "magnolia": 13940, + "tesla": 13941, + "bugatti": 13942, + "cum": 13943, + "bolt": 13944, + "modular": 13945, + "##zu": 13946, + "##atives": 13947, + "bare": 13948, + "nu": 13949, + "visited": 13950, + "operating": 13951, + "mixture": 13952, + "bree": 13953, + "hillside": 13954, + "stockings": 13955, + "##iro": 13956, + "ode": 13957, + "gn": 13958, + "greenery": 13959, + "genu": 13960, + "ki": 13961, + "asleep": 13962, + "surprised": 13963, + "retire": 13964, + "swimsuit": 13965, + "tactical": 13966, + "angles": 13967, + "alive": 13968, + "bagel": 13969, + "souvenir": 13970, + "##mint": 13971, + "eyebr": 13972, + "wra": 13973, + "##ogen": 13974, + "antar": 13975, + "bigger": 13976, + "desks": 13977, + "1969": 13978, + "coord": 13979, + "##tale": 13980, + "continuous": 13981, + "vene": 13982, + "printable": 13983, + "##world": 13984, + "bodies": 13985, + "fas": 13986, + "##lines": 13987, + "sweaters": 13988, + "corrid": 13989, + "##mere": 13990, + "##ulate": 13991, + "trailers": 13992, + "significant": 13993, + "##akh": 13994, + "noodle": 13995, + "hide": 13996, + "sweatshirt": 13997, + "poetry": 13998, + "sands": 13999, + "motherboard": 14000, + "jellys": 14001, + "twins": 14002, + "expans": 14003, + "rowing": 14004, + "##oh": 14005, + "barbecue": 14006, + "magnetic": 14007, + "homeless": 14008, + "pion": 14009, + "sha": 14010, + "clips": 14011, + "buns": 14012, + "##etra": 14013, + "##esis": 14014, + "##ulations": 14015, + "profession": 14016, + "spiderman": 14017, + "##alay": 14018, + "##ran": 14019, + "willow": 14020, + "craftsman": 14021, + "cereal": 14022, + "doz": 14023, + "magnificent": 14024, + "ecol": 14025, + "jaw": 14026, + "ukraine": 14027, + "##wise": 14028, + "duch": 14029, + "claws": 14030, + "incredibles": 14031, + "4x4": 14032, + "marriott": 14033, + "nic": 14034, + "prix": 14035, + "nebula": 14036, + "mental": 14037, + "fender": 14038, + "directed": 14039, + "croatia": 14040, + "prepares": 14041, + "##hill": 14042, + "##imore": 14043, + "fountains": 14044, + "scattered": 14045, + "limous": 14046, + "funn": 14047, + "poppy": 14048, + "welding": 14049, + "cornwall": 14050, + "response": 14051, + "bruss": 14052, + "starbucks": 14053, + "catering": 14054, + "mald": 14055, + "laptops": 14056, + "##gun": 14057, + "##wy": 14058, + "cones": 14059, + "gears": 14060, + "1960": 14061, + "dynasty": 14062, + "polished": 14063, + "voy": 14064, + "actual": 14065, + "disco": 14066, + "seg": 14067, + "stranger": 14068, + "kerala": 14069, + "packing": 14070, + "utility": 14071, + "mineral": 14072, + "##cup": 14073, + "derby": 14074, + "54": 14075, + "##dr": 14076, + "##bt": 14077, + "festivals": 14078, + "ltd": 14079, + "##iop": 14080, + "##sk": 14081, + "glory": 14082, + "##rom": 14083, + "respect": 14084, + "wander": 14085, + "providing": 14086, + "brushing": 14087, + "washer": 14088, + "width": 14089, + "baltimore": 14090, + "##aru": 14091, + "limo": 14092, + "tuxedo": 14093, + "exist": 14094, + "gossip": 14095, + "guards": 14096, + "midnight": 14097, + "personnel": 14098, + "renovated": 14099, + "1920s": 14100, + "newfoundland": 14101, + "##otype": 14102, + "##aug": 14103, + "absolutely": 14104, + "designing": 14105, + "filip": 14106, + "wit": 14107, + "marker": 14108, + "##agus": 14109, + "awak": 14110, + "edges": 14111, + "thames": 14112, + "honeym": 14113, + "ups": 14114, + "milky": 14115, + "optim": 14116, + "softball": 14117, + "glue": 14118, + "##aser": 14119, + "archery": 14120, + "screening": 14121, + "detach": 14122, + "##occ": 14123, + "##usc": 14124, + "banglad": 14125, + "bangladesh": 14126, + "coffin": 14127, + "orchard": 14128, + "prefer": 14129, + "wellness": 14130, + "1900": 14131, + "microsoft": 14132, + "forb": 14133, + "##emy": 14134, + "pedest": 14135, + "##95": 14136, + "mustache": 14137, + "seals": 14138, + "mostly": 14139, + "technical": 14140, + "sle": 14141, + "tamp": 14142, + "1980s": 14143, + "chimney": 14144, + "mysterious": 14145, + "taj": 14146, + "drift": 14147, + "nou": 14148, + "receiving": 14149, + "fell": 14150, + "plantation": 14151, + "##xtures": 14152, + "exposure": 14153, + "toss": 14154, + "76": 14155, + "lanc": 14156, + "emperor": 14157, + "forear": 14158, + "coke": 14159, + "##alis": 14160, + "instructor": 14161, + "hp": 14162, + "rein": 14163, + "toppings": 14164, + "bolts": 14165, + "alberta": 14166, + "jas": 14167, + "links": 14168, + "paired": 14169, + "plaster": 14170, + "canc": 14171, + "functional": 14172, + "genuine": 14173, + "scientist": 14174, + "curve": 14175, + "sapphire": 14176, + "bould": 14177, + "strips": 14178, + "cranberry": 14179, + "au": 14180, + "##ams": 14181, + "despite": 14182, + "essentials": 14183, + "bore": 14184, + "calls": 14185, + "##ods": 14186, + "colomb": 14187, + "galleries": 14188, + "billboard": 14189, + "boost": 14190, + "ku": 14191, + "natur": 14192, + "neighbour": 14193, + "##38": 14194, + "panama": 14195, + "gourmet": 14196, + "13th": 14197, + "constant": 14198, + "rene": 14199, + "##apolis": 14200, + "bills": 14201, + "intricate": 14202, + "scientific": 14203, + "doctors": 14204, + "cica": 14205, + "enthus": 14206, + "herbal": 14207, + "##lap": 14208, + "enthusi": 14209, + "egret": 14210, + "##ocr": 14211, + "##cules": 14212, + "quad": 14213, + "keych": 14214, + "duchess": 14215, + "jedi": 14216, + "transit": 14217, + "clever": 14218, + "fundra": 14219, + "tri": 14220, + "ves": 14221, + "files": 14222, + "rajasthan": 14223, + "aim": 14224, + "##shore": 14225, + "attractions": 14226, + "california": 14227, + "billion": 14228, + "nuclear": 14229, + "arriving": 14230, + "tried": 14231, + "miner": 14232, + "honeymoon": 14233, + "manila": 14234, + "birch": 14235, + "enamel": 14236, + "literature": 14237, + "offered": 14238, + "wheelbar": 14239, + "spaceship": 14240, + "6th": 14241, + "vines": 14242, + "audiobook": 14243, + "announcement": 14244, + "gm": 14245, + "##oni": 14246, + "sparkling": 14247, + "pomeran": 14248, + "continues": 14249, + "maldives": 14250, + "pike": 14251, + "slate": 14252, + "argentina": 14253, + "benefit": 14254, + "hips": 14255, + "monaster": 14256, + "confetti": 14257, + "upright": 14258, + "pope": 14259, + "holland": 14260, + "tasting": 14261, + "biscuits": 14262, + "guar": 14263, + "banda": 14264, + "1999": 14265, + "seash": 14266, + "purchased": 14267, + "1990": 14268, + "rangoli": 14269, + "geographic": 14270, + "sophistic": 14271, + "##bee": 14272, + "bulletin": 14273, + "caravan": 14274, + "##vin": 14275, + "sunflowers": 14276, + "##plant": 14277, + "forearm": 14278, + "dach": 14279, + "snowmob": 14280, + "##dson": 14281, + "selected": 14282, + "cay": 14283, + "expand": 14284, + "citrus": 14285, + "replaced": 14286, + "##rac": 14287, + "elaborate": 14288, + "##ores": 14289, + "drip": 14290, + "unesco": 14291, + "disap": 14292, + "majestic": 14293, + "liv": 14294, + "mailbox": 14295, + "prehistoric": 14296, + "academia": 14297, + "ethiop": 14298, + "crosses": 14299, + "##anmar": 14300, + "bart": 14301, + "multicol": 14302, + "wilds": 14303, + "volunteers": 14304, + "##inated": 14305, + "kale": 14306, + "##bank": 14307, + "spons": 14308, + "converted": 14309, + "anole": 14310, + "cables": 14311, + "bellied": 14312, + "civic": 14313, + "##break": 14314, + "conven": 14315, + "limousine": 14316, + "##gro": 14317, + "##fy": 14318, + "##ared": 14319, + "cors": 14320, + "ability": 14321, + "innovation": 14322, + "pomeranian": 14323, + "scramb": 14324, + "bact": 14325, + "56": 14326, + "fixtures": 14327, + "madison": 14328, + "crisp": 14329, + "topper": 14330, + "curren": 14331, + "myanmar": 14332, + "cambodia": 14333, + "shake": 14334, + "skip": 14335, + "planned": 14336, + "refuge": 14337, + "practicing": 14338, + "teenager": 14339, + "replacement": 14340, + "frequ": 14341, + "glaze": 14342, + "pvc": 14343, + "lets": 14344, + "punching": 14345, + "##stairs": 14346, + "butcher": 14347, + "sul": 14348, + "##vings": 14349, + "amid": 14350, + "##onna": 14351, + "holders": 14352, + "cure": 14353, + "facilities": 14354, + "hives": 14355, + "southwest": 14356, + "chemistry": 14357, + "apocaly": 14358, + "unfin": 14359, + "unfinished": 14360, + "penny": 14361, + "verses": 14362, + "##using": 14363, + "bella": 14364, + "monte": 14365, + "banquet": 14366, + "guarant": 14367, + "serves": 14368, + "regency": 14369, + "cudd": 14370, + "ali": 14371, + "##adan": 14372, + "morocc": 14373, + "eats": 14374, + "patches": 14375, + "##boys": 14376, + "messenger": 14377, + "moroccan": 14378, + "jigsaw": 14379, + "chi": 14380, + "household": 14381, + "temporary": 14382, + "hooks": 14383, + "##hes": 14384, + "elite": 14385, + "1980": 14386, + "honeyc": 14387, + "lunar": 14388, + "guardian": 14389, + "##bl": 14390, + "chile": 14391, + "bend": 14392, + "nascar": 14393, + "economy": 14394, + "apocalypse": 14395, + "1968": 14396, + "ceil": 14397, + "grizz": 14398, + "rotary": 14399, + "fixed": 14400, + "stap": 14401, + "worldwide": 14402, + "geese": 14403, + "whims": 14404, + "##abil": 14405, + "##green": 14406, + "##ifl": 14407, + "recruit": 14408, + "puts": 14409, + "malt": 14410, + "applied": 14411, + "pedestal": 14412, + "brighton": 14413, + "threat": 14414, + "preview": 14415, + "forg": 14416, + "suz": 14417, + "venetian": 14418, + "trunks": 14419, + "1996": 14420, + "53": 14421, + "brussels": 14422, + "knob": 14423, + "emotional": 14424, + "performed": 14425, + "demol": 14426, + "800": 14427, + "stations": 14428, + "forests": 14429, + "obe": 14430, + "yoda": 14431, + "thumbs": 14432, + "adorn": 14433, + "shall": 14434, + "masters": 14435, + "rolex": 14436, + "behavior": 14437, + "aston": 14438, + "harvesting": 14439, + "walked": 14440, + "limit": 14441, + "lettuce": 14442, + "amp": 14443, + "missiles": 14444, + "polygon": 14445, + "foggy": 14446, + "observatory": 14447, + "cigarettes": 14448, + "##atus": 14449, + "flap": 14450, + "##cker": 14451, + "tickets": 14452, + "1998": 14453, + "barbell": 14454, + "casc": 14455, + "mau": 14456, + "corridor": 14457, + "stems": 14458, + "parka": 14459, + "##uto": 14460, + "shakes": 14461, + "sevent": 14462, + "baker": 14463, + "slippers": 14464, + "##ew": 14465, + "weekly": 14466, + "shorth": 14467, + "symmet": 14468, + "crowds": 14469, + "1950": 14470, + "##eman": 14471, + "markets": 14472, + "stu": 14473, + "fellow": 14474, + "rolled": 14475, + "rescued": 14476, + "marked": 14477, + "settings": 14478, + "mighty": 14479, + "a3": 14480, + "charts": 14481, + "perch": 14482, + "partridge": 14483, + "ign": 14484, + "orthod": 14485, + "coop": 14486, + "##66": 14487, + "preparation": 14488, + "charms": 14489, + "retain": 14490, + "ceilings": 14491, + "51": 14492, + "hik": 14493, + "jul": 14494, + "nook": 14495, + "##pass": 14496, + "brazilian": 14497, + "revealed": 14498, + "mum": 14499, + "##isions": 14500, + "pasture": 14501, + "glazed": 14502, + "giraf": 14503, + "##weiler": 14504, + "gats": 14505, + "harmonica": 14506, + "kom": 14507, + "ceo": 14508, + "##mother": 14509, + "##udge": 14510, + "cityscape": 14511, + "weaving": 14512, + "##medi": 14513, + "hydro": 14514, + "tomor": 14515, + "herald": 14516, + "survey": 14517, + "lump": 14518, + "appeal": 14519, + "reflections": 14520, + "hobby": 14521, + "guitarist": 14522, + "flamingos": 14523, + "starring": 14524, + "temples": 14525, + "boardwalk": 14526, + "saucer": 14527, + "collie": 14528, + "remodeling": 14529, + "mashed": 14530, + "adobe": 14531, + "alcoholic": 14532, + "scientists": 14533, + "involved": 14534, + "account": 14535, + "##uges": 14536, + "quarterback": 14537, + "hungry": 14538, + "##ament": 14539, + "##apest": 14540, + "budapest": 14541, + "cn": 14542, + "buckingham": 14543, + "jumpsuit": 14544, + "gatsby": 14545, + "duff": 14546, + "curious": 14547, + "drummer": 14548, + "espresso": 14549, + "##78": 14550, + "dollars": 14551, + "##gular": 14552, + "ensure": 14553, + "investment": 14554, + "##reen": 14555, + "braided": 14556, + "asked": 14557, + "grum": 14558, + "celebrated": 14559, + "crispy": 14560, + "##agas": 14561, + "snor": 14562, + "##elic": 14563, + "liquor": 14564, + "commem": 14565, + "hijab": 14566, + "cloak": 14567, + "aba": 14568, + "joined": 14569, + "arabian": 14570, + "agent": 14571, + "promotional": 14572, + "sleek": 14573, + "nsw": 14574, + "spell": 14575, + "bach": 14576, + "congrat": 14577, + "##poke": 14578, + "lol": 14579, + "obst": 14580, + "participants": 14581, + "clutch": 14582, + "raid": 14583, + "##ultural": 14584, + "lotion": 14585, + "lor": 14586, + "assass": 14587, + "wavy": 14588, + "swings": 14589, + "bespoke": 14590, + "substance": 14591, + "pref": 14592, + "purses": 14593, + "lining": 14594, + "wellington": 14595, + "checkered": 14596, + "finds": 14597, + "250": 14598, + "dart": 14599, + "workplace": 14600, + "daughters": 14601, + "supers": 14602, + "asph": 14603, + "remind": 14604, + "1984": 14605, + "likely": 14606, + "##lend": 14607, + "dispens": 14608, + "surfer": 14609, + "sparkle": 14610, + "crisis": 14611, + "rc": 14612, + "tomorrow": 14613, + "##eder": 14614, + "planted": 14615, + "ramadan": 14616, + "danish": 14617, + "##rep": 14618, + "reclaimed": 14619, + "approximately": 14620, + "artif": 14621, + "couches": 14622, + "tribune": 14623, + "vineyard": 14624, + "palms": 14625, + "attended": 14626, + "arrived": 14627, + "##oes": 14628, + "sid": 14629, + "##emble": 14630, + "patients": 14631, + "sheer": 14632, + "boun": 14633, + "deals": 14634, + "sequin": 14635, + "mercury": 14636, + "chase": 14637, + "pt": 14638, + "intro": 14639, + "mammals": 14640, + "cairo": 14641, + "##usa": 14642, + "##ku": 14643, + "adopted": 14644, + "##pon": 14645, + "##ounder": 14646, + "paul": 14647, + "constellation": 14648, + "stages": 14649, + "##pole": 14650, + "16th": 14651, + "artifact": 14652, + "pent": 14653, + "cb": 14654, + "latte": 14655, + "cowboys": 14656, + "robots": 14657, + "warthog": 14658, + "##adu": 14659, + "blooms": 14660, + "kolk": 14661, + "filipino": 14662, + "##uries": 14663, + "experts": 14664, + "chestnuts": 14665, + "##ighting": 14666, + "worms": 14667, + "##assy": 14668, + "abu": 14669, + "##boarding": 14670, + "debut": 14671, + "chronic": 14672, + "fifa": 14673, + "atlas": 14674, + "##ibu": 14675, + "abc": 14676, + "digging": 14677, + "ah": 14678, + "expo": 14679, + "1967": 14680, + "1965": 14681, + "kolkata": 14682, + "pineapples": 14683, + "relationships": 14684, + "kual": 14685, + "##bing": 14686, + "marshm": 14687, + "seasonal": 14688, + "programs": 14689, + "jol": 14690, + "forming": 14691, + "1994": 14692, + "monastery": 14693, + "hangs": 14694, + "tear": 14695, + "hudson": 14696, + "bites": 14697, + "quir": 14698, + "bundle": 14699, + "blueprint": 14700, + "breaks": 14701, + "##uals": 14702, + "otters": 14703, + "spook": 14704, + "ply": 14705, + "experim": 14706, + "jog": 14707, + "springer": 14708, + "shoreline": 14709, + "explosion": 14710, + "blueberries": 14711, + "gan": 14712, + "homecoming": 14713, + "centers": 14714, + "entering": 14715, + "kal": 14716, + "interface": 14717, + "understanding": 14718, + "chamber": 14719, + "builder": 14720, + "fixture": 14721, + "##aukee": 14722, + "updated": 14723, + "kuala": 14724, + "shih": 14725, + "freshly": 14726, + "chamela": 14727, + "tol": 14728, + "guides": 14729, + "burberry": 14730, + "meatballs": 14731, + "asphalt": 14732, + "sr": 14733, + "ceram": 14734, + "florida": 14735, + "tubes": 14736, + "rex": 14737, + "crates": 14738, + "alba": 14739, + "pagoda": 14740, + "biting": 14741, + "fencing": 14742, + "jim": 14743, + "emotions": 14744, + "decision": 14745, + "gig": 14746, + "ecle": 14747, + "##hen": 14748, + "##osoph": 14749, + "haird": 14750, + "preserve": 14751, + "certific": 14752, + "shorthair": 14753, + "pretzel": 14754, + "agricultural": 14755, + "happen": 14756, + "sporting": 14757, + "picked": 14758, + "##haust": 14759, + "triangles": 14760, + "congress": 14761, + "cutter": 14762, + "vienna": 14763, + "##mare": 14764, + "stationery": 14765, + "exactly": 14766, + "economic": 14767, + "dice": 14768, + "##obs": 14769, + "tampa": 14770, + "currency": 14771, + "85": 14772, + "oc": 14773, + "blender": 14774, + "infin": 14775, + "hanger": 14776, + "hydrant": 14777, + "gazebo": 14778, + "initi": 14779, + "atomic": 14780, + "biking": 14781, + "4k": 14782, + "tn": 14783, + "corona": 14784, + "pars": 14785, + "myself": 14786, + "versat": 14787, + "fins": 14788, + "arrangements": 14789, + "bales": 14790, + "rams": 14791, + "exhaust": 14792, + "baller": 14793, + "capped": 14794, + "forkl": 14795, + "120": 14796, + "glyph": 14797, + "##pill": 14798, + "blades": 14799, + "belgian": 14800, + "tasman": 14801, + "bagels": 14802, + "riders": 14803, + "bitter": 14804, + "philosoph": 14805, + "rink": 14806, + "##mart": 14807, + "##vi": 14808, + "twitter": 14809, + "costs": 14810, + "munich": 14811, + "caterpill": 14812, + "partially": 14813, + "resource": 14814, + "reproduction": 14815, + "fist": 14816, + "imagine": 14817, + "pist": 14818, + "guild": 14819, + "##yme": 14820, + "critical": 14821, + "##bone": 14822, + "loom": 14823, + "stools": 14824, + "lect": 14825, + "sons": 14826, + "innovative": 14827, + "apa": 14828, + "lazy": 14829, + "quant": 14830, + "##tor": 14831, + "bos": 14832, + "silent": 14833, + "crunch": 14834, + "cockpit": 14835, + "windmill": 14836, + "favors": 14837, + "earring": 14838, + "outstanding": 14839, + "xl": 14840, + "suspe": 14841, + "chimpan": 14842, + "distin": 14843, + "panorama": 14844, + "cycl": 14845, + "predator": 14846, + "##wali": 14847, + "caulifl": 14848, + "retirement": 14849, + "devon": 14850, + "driven": 14851, + "brack": 14852, + "hammock": 14853, + "listing": 14854, + "##ading": 14855, + "tallest": 14856, + "spraying": 14857, + "offshore": 14858, + "1975": 14859, + "secretary": 14860, + "identify": 14861, + "storey": 14862, + "##iev": 14863, + "hate": 14864, + "silic": 14865, + "1995": 14866, + "soy": 14867, + "##aria": 14868, + "props": 14869, + "##iva": 14870, + "quartz": 14871, + "##rit": 14872, + "miniatures": 14873, + "##quin": 14874, + "diwali": 14875, + "volvo": 14876, + "crut": 14877, + "guided": 14878, + "##rene": 14879, + "yachts": 14880, + "patriotic": 14881, + "pelican": 14882, + "false": 14883, + "basics": 14884, + "wraps": 14885, + "caves": 14886, + "iced": 14887, + "bibl": 14888, + "intelligence": 14889, + "hamp": 14890, + "##pear": 14891, + "fairytale": 14892, + "trained": 14893, + "jewels": 14894, + "rendered": 14895, + "cauliflower": 14896, + "inver": 14897, + "##stock": 14898, + "evergreen": 14899, + "odeon": 14900, + "##pies": 14901, + "1966": 14902, + "finest": 14903, + "##han": 14904, + "##waukee": 14905, + "roy": 14906, + "1989": 14907, + "nightmare": 14908, + "injury": 14909, + "mm": 14910, + "72": 14911, + "exit": 14912, + "tricycle": 14913, + "searching": 14914, + "yang": 14915, + "##pg": 14916, + "detective": 14917, + "##fu": 14918, + "twenty": 14919, + "68": 14920, + "astronomy": 14921, + "legal": 14922, + "cellphone": 14923, + "krish": 14924, + "omega": 14925, + "acorns": 14926, + "wrought": 14927, + "##test": 14928, + "horned": 14929, + "sie": 14930, + "slopes": 14931, + "lith": 14932, + "arbor": 14933, + "engines": 14934, + "chrysler": 14935, + "parag": 14936, + "sheath": 14937, + "collecting": 14938, + "rounded": 14939, + "mockup": 14940, + "##phis": 14941, + "shred": 14942, + "flare": 14943, + "clubs": 14944, + "##arta": 14945, + "destroyed": 14946, + "artis": 14947, + "pros": 14948, + "##illes": 14949, + "milwaukee": 14950, + "##posed": 14951, + "eclectic": 14952, + "disab": 14953, + "vin": 14954, + "traveler": 14955, + "hamilton": 14956, + "cinderella": 14957, + "sophisticated": 14958, + "bever": 14959, + "fiat": 14960, + "##rix": 14961, + "beginner": 14962, + "boarding": 14963, + "promotion": 14964, + "welsh": 14965, + "stol": 14966, + "playful": 14967, + "racco": 14968, + "fract": 14969, + "buds": 14970, + "clipping": 14971, + "ceramics": 14972, + "distribution": 14973, + "casting": 14974, + "cubes": 14975, + "demonstration": 14976, + "collected": 14977, + "businesswoman": 14978, + "deadly": 14979, + "sensory": 14980, + "draft": 14981, + "moto": 14982, + "sag": 14983, + "richmond": 14984, + "##illac": 14985, + "documentary": 14986, + "tacos": 14987, + "sah": 14988, + "##mitted": 14989, + "drilling": 14990, + "buttern": 14991, + "bulbs": 14992, + "witcher": 14993, + "1997": 14994, + "screenshots": 14995, + "precious": 14996, + "acid": 14997, + "ace": 14998, + "elk": 14999, + "merchand": 15000, + "naturally": 15001, + "spooky": 15002, + "muffin": 15003, + "pixie": 15004, + "collectible": 15005, + "poker": 15006, + "notre": 15007, + "toddlers": 15008, + "69": 15009, + "##76": 15010, + "##cuss": 15011, + "caused": 15012, + "vatic": 15013, + "kat": 15014, + "pyramids": 15015, + "afro": 15016, + "healthcare": 15017, + "##bound": 15018, + "dew": 15019, + "menswear": 15020, + "cadillac": 15021, + "##gary": 15022, + "##ados": 15023, + "mahal": 15024, + "rivi": 15025, + "warming": 15026, + "krishna": 15027, + "##rable": 15028, + "knows": 15029, + "zoom": 15030, + "secure": 15031, + "afghanistan": 15032, + "ecl": 15033, + "minn": 15034, + "ropes": 15035, + "##39": 15036, + "consult": 15037, + "groomed": 15038, + "introduced": 15039, + "challenges": 15040, + "quirky": 15041, + "bapt": 15042, + "memphis": 15043, + "tanz": 15044, + "sectional": 15045, + "capacity": 15046, + "sel": 15047, + "##enga": 15048, + "##ipse": 15049, + "retired": 15050, + "diagrams": 15051, + "##stand": 15052, + "passport": 15053, + "arabia": 15054, + "commemor": 15055, + "dane": 15056, + "continental": 15057, + "lumpur": 15058, + "touring": 15059, + "consists": 15060, + "pomegr": 15061, + "suns": 15062, + "brooch": 15063, + "divine": 15064, + "minor": 15065, + "captivity": 15066, + "peaks": 15067, + "rodeo": 15068, + "##gs": 15069, + "##eable": 15070, + "##plement": 15071, + "diverse": 15072, + "vatican": 15073, + "dreamworks": 15074, + "winery": 15075, + "##yth": 15076, + "brewery": 15077, + "hoo": 15078, + "debr": 15079, + "##000": 15080, + "cookbook": 15081, + "##uru": 15082, + "articles": 15083, + "aspar": 15084, + "apps": 15085, + "roofing": 15086, + "1930s": 15087, + "fond": 15088, + "##illing": 15089, + "showed": 15090, + "infr": 15091, + "nickel": 15092, + "norwe": 15093, + "pandora": 15094, + "anemons": 15095, + "##vens": 15096, + "comfor": 15097, + "madagas": 15098, + "dandelion": 15099, + "wise": 15100, + "customized": 15101, + "commons": 15102, + "diner": 15103, + "scoring": 15104, + "desce": 15105, + "wealth": 15106, + "ampl": 15107, + "burst": 15108, + "dynamic": 15109, + "paid": 15110, + "hobbies": 15111, + "closing": 15112, + "travels": 15113, + "magnifying": 15114, + "pretzels": 15115, + "achieve": 15116, + "melted": 15117, + "orthodox": 15118, + "assault": 15119, + "funniest": 15120, + "norwegian": 15121, + "##folk": 15122, + "##ec": 15123, + "##osis": 15124, + "shim": 15125, + "##fo": 15126, + "riviera": 15127, + "##lined": 15128, + "ecu": 15129, + "fails": 15130, + "##edd": 15131, + "diaries": 15132, + "depe": 15133, + "anti": 15134, + "keychain": 15135, + "saved": 15136, + "##vre": 15137, + "##enhagen": 15138, + "parenting": 15139, + "grumpy": 15140, + "caval": 15141, + "ira": 15142, + "sketchbook": 15143, + "10th": 15144, + "50th": 15145, + "tast": 15146, + "ritz": 15147, + "resistant": 15148, + "debris": 15149, + "whip": 15150, + "glamorous": 15151, + "##ni": 15152, + "##⁄4": 15153, + "##erted": 15154, + "awa": 15155, + "biblical": 15156, + "##iters": 15157, + "foyer": 15158, + "iguana": 15159, + "flickr": 15160, + "finland": 15161, + "copenhagen": 15162, + "diameter": 15163, + "generated": 15164, + "advertisements": 15165, + "roofs": 15166, + "editing": 15167, + "dear": 15168, + "elm": 15169, + "directions": 15170, + "gems": 15171, + "pas": 15172, + "oils": 15173, + "kicking": 15174, + "pebbles": 15175, + "twice": 15176, + "loud": 15177, + "martini": 15178, + "##oi": 15179, + "succ": 15180, + "terms": 15181, + "garland": 15182, + "courses": 15183, + "mazda": 15184, + "markers": 15185, + "bish": 15186, + "##ropri": 15187, + "washington": 15188, + "skyscraper": 15189, + "resol": 15190, + "charging": 15191, + "psd": 15192, + "##shake": 15193, + "ministry": 15194, + "wooded": 15195, + "vietnamese": 15196, + "spat": 15197, + "wrang": 15198, + "wonderland": 15199, + "havana": 15200, + "forks": 15201, + "regions": 15202, + "911": 15203, + "attending": 15204, + "##iana": 15205, + "agree": 15206, + "fool": 15207, + "superior": 15208, + "blown": 15209, + "57": 15210, + "vet": 15211, + "##pu": 15212, + "modeling": 15213, + "lef": 15214, + "hearing": 15215, + "moonlight": 15216, + "labeled": 15217, + "finishing": 15218, + "rims": 15219, + "minerals": 15220, + "handler": 15221, + "warner": 15222, + "sinks": 15223, + "traditions": 15224, + "alpha": 15225, + "##uan": 15226, + "airbus": 15227, + "flamings": 15228, + "carvings": 15229, + "##yn": 15230, + "shakers": 15231, + "resume": 15232, + "magnet": 15233, + "marin": 15234, + "dentist": 15235, + "##alu": 15236, + "feels": 15237, + "collective": 15238, + "##umes": 15239, + "##iche": 15240, + "##here": 15241, + "haul": 15242, + "forbidden": 15243, + "election": 15244, + "fastest": 15245, + "peek": 15246, + "##ser": 15247, + "founder": 15248, + "refreshing": 15249, + "necessary": 15250, + "proced": 15251, + "kicks": 15252, + "businesses": 15253, + "calgary": 15254, + "sussex": 15255, + "depicts": 15256, + "nomad": 15257, + "bikinis": 15258, + "manufacturer": 15259, + "compound": 15260, + "seconds": 15261, + "wan": 15262, + "lands": 15263, + "chunky": 15264, + "yester": 15265, + "##dles": 15266, + "##vard": 15267, + "complement": 15268, + "consist": 15269, + "mammal": 15270, + "longest": 15271, + "indianapolis": 15272, + "adoption": 15273, + "67": 15274, + "##nati": 15275, + "##cet": 15276, + "##elin": 15277, + "whist": 15278, + "towed": 15279, + "connector": 15280, + "snails": 15281, + "olives": 15282, + "albums": 15283, + "analy": 15284, + "hottest": 15285, + "quiz": 15286, + "shift": 15287, + "iris": 15288, + "badges": 15289, + "psyched": 15290, + "promote": 15291, + "tortoise": 15292, + "georgian": 15293, + "glossy": 15294, + "cincin": 15295, + "caric": 15296, + "21st": 15297, + "cucumber": 15298, + "geneva": 15299, + "buckets": 15300, + "democr": 15301, + "romania": 15302, + "chateau": 15303, + "yamaha": 15304, + "stroke": 15305, + "lids": 15306, + "guid": 15307, + "pillars": 15308, + "pioneer": 15309, + "cincinnati": 15310, + "intimate": 15311, + "lack": 15312, + "##van": 15313, + "waterfalls": 15314, + "ante": 15315, + "entertaining": 15316, + "##bert": 15317, + "supporting": 15318, + "scooters": 15319, + "poems": 15320, + "regiment": 15321, + "hampton": 15322, + "legged": 15323, + "fringe": 15324, + "abdomen": 15325, + "caterpillar": 15326, + "stolen": 15327, + "lumber": 15328, + "deploy": 15329, + "nordic": 15330, + "madagascar": 15331, + "topics": 15332, + "values": 15333, + "breeze": 15334, + "gamb": 15335, + "corgi": 15336, + "compartment": 15337, + "##abi": 15338, + "percuss": 15339, + "bunn": 15340, + "topic": 15341, + "elevation": 15342, + "smallest": 15343, + "78": 15344, + "younger": 15345, + "180": 15346, + "reveals": 15347, + "kir": 15348, + "arched": 15349, + "enchanted": 15350, + "stretched": 15351, + "salma": 15352, + "fossils": 15353, + "municip": 15354, + "##burn": 15355, + "psychedelic": 15356, + "##essed": 15357, + "filming": 15358, + "##mates": 15359, + "##uda": 15360, + "brightly": 15361, + "##hu": 15362, + "##elines": 15363, + "newport": 15364, + "determine": 15365, + "overview": 15366, + "paths": 15367, + "receives": 15368, + "norfolk": 15369, + "connecting": 15370, + "##zers": 15371, + "increasing": 15372, + "interp": 15373, + "steve": 15374, + "butternut": 15375, + "pointed": 15376, + "equipped": 15377, + "beverages": 15378, + "7th": 15379, + "##yan": 15380, + "funky": 15381, + "confident": 15382, + "conversion": 15383, + "calc": 15384, + "1992": 15385, + "veggie": 15386, + "dod": 15387, + "choices": 15388, + "##ody": 15389, + "provin": 15390, + "burnt": 15391, + "##ais": 15392, + "charleston": 15393, + "ml": 15394, + "braid": 15395, + "dealers": 15396, + "##stage": 15397, + "ecuador": 15398, + "vot": 15399, + "satis": 15400, + "travelling": 15401, + "##hole": 15402, + "bodybuilding": 15403, + "gob": 15404, + "dor": 15405, + "vista": 15406, + "spar": 15407, + "peaceful": 15408, + "##stones": 15409, + "endless": 15410, + "campfire": 15411, + "##castle": 15412, + "condom": 15413, + "kan": 15414, + "bah": 15415, + "3⁄4": 15416, + "bachel": 15417, + "alaska": 15418, + "63": 15419, + "alle": 15420, + "viewing": 15421, + "hexagon": 15422, + "lisbon": 15423, + "detailing": 15424, + "removing": 15425, + "leeds": 15426, + "index": 15427, + "trading": 15428, + "##mith": 15429, + "pilgr": 15430, + "clash": 15431, + "capsule": 15432, + "ariz": 15433, + "collaboration": 15434, + "brakes": 15435, + "owls": 15436, + "situated": 15437, + "tuck": 15438, + "kin": 15439, + "mack": 15440, + "combo": 15441, + "tint": 15442, + "throws": 15443, + "dont": 15444, + "##oline": 15445, + "rapid": 15446, + "spectrum": 15447, + "carr": 15448, + "puppet": 15449, + "scripture": 15450, + "58": 15451, + "##44": 15452, + "postcards": 15453, + "packages": 15454, + "layouts": 15455, + "##isha": 15456, + "trinity": 15457, + "tries": 15458, + "59": 15459, + "centerpiece": 15460, + "muddy": 15461, + "namib": 15462, + "claw": 15463, + "1964": 15464, + "##fall": 15465, + "unity": 15466, + "karate": 15467, + "twisted": 15468, + "corners": 15469, + "digit": 15470, + "lehenga": 15471, + "nostalgia": 15472, + "ordinary": 15473, + "uns": 15474, + "suspended": 15475, + "sticky": 15476, + "electricity": 15477, + "maur": 15478, + "processing": 15479, + "quebec": 15480, + "delivered": 15481, + "manufacturing": 15482, + "vice": 15483, + "enterpr": 15484, + "robin": 15485, + "stethoscope": 15486, + "flask": 15487, + "finance": 15488, + "coastline": 15489, + "battal": 15490, + "plywood": 15491, + "counting": 15492, + "##esta": 15493, + "closure": 15494, + "elevator": 15495, + "vul": 15496, + "maroon": 15497, + "dyed": 15498, + "8th": 15499, + "tuna": 15500, + "##pting": 15501, + "ruffle": 15502, + "carolina": 15503, + "boars": 15504, + "##izations": 15505, + "smell": 15506, + "yesterday": 15507, + "translation": 15508, + "tested": 15509, + "muscular": 15510, + "amen": 15511, + "underwear": 15512, + "curated": 15513, + "kath": 15514, + "snowflakes": 15515, + "ya": 15516, + "bale": 15517, + "samples": 15518, + "showers": 15519, + "carlton": 15520, + "learned": 15521, + "prohib": 15522, + "dachsh": 15523, + "foto": 15524, + "fired": 15525, + "towns": 15526, + "1986": 15527, + "herself": 15528, + "outbreak": 15529, + "dayton": 15530, + "spo": 15531, + "squid": 15532, + "elastic": 15533, + "sofia": 15534, + "130": 15535, + "leo": 15536, + "biological": 15537, + "dated": 15538, + "##birds": 15539, + "yours": 15540, + "eclipse": 15541, + "biz": 15542, + "##rest": 15543, + "perman": 15544, + "beth": 15545, + "##lifting": 15546, + "playroom": 15547, + "clam": 15548, + "##inkle": 15549, + "nouv": 15550, + "nowhere": 15551, + "exciting": 15552, + "veteran": 15553, + "ottoman": 15554, + "bouquets": 15555, + "battalion": 15556, + "##ologist": 15557, + "utens": 15558, + "scal": 15559, + "everywhere": 15560, + "himalay": 15561, + "residences": 15562, + "enhance": 15563, + "##tm": 15564, + "grows": 15565, + "1985": 15566, + "loch": 15567, + "gund": 15568, + "kyoto": 15569, + "whimsical": 15570, + "saga": 15571, + "concent": 15572, + "beeh": 15573, + "manusc": 15574, + "paste": 15575, + "brownie": 15576, + "catalog": 15577, + "bling": 15578, + "##rative": 15579, + "eu": 15580, + "cocoa": 15581, + "##zel": 15582, + "stitches": 15583, + "raz": 15584, + "##oring": 15585, + "freshwater": 15586, + "durable": 15587, + "iraq": 15588, + "winged": 15589, + "aa": 15590, + "insert": 15591, + "succul": 15592, + "protected": 15593, + "burgers": 15594, + "lent": 15595, + "##abul": 15596, + "unl": 15597, + "skyscrapers": 15598, + "competing": 15599, + "heating": 15600, + "slogan": 15601, + "pharmacy": 15602, + "##chief": 15603, + "##ologies": 15604, + "technician": 15605, + "##reh": 15606, + "hib": 15607, + "relaxation": 15608, + "affair": 15609, + "typing": 15610, + "##eration": 15611, + "versatile": 15612, + "discussion": 15613, + "bangs": 15614, + "ay": 15615, + "functions": 15616, + "##enous": 15617, + "chiang": 15618, + "conce": 15619, + "basilica": 15620, + "##mans": 15621, + "##essions": 15622, + "revival": 15623, + "websites": 15624, + "catcher": 15625, + "laure": 15626, + "tanzania": 15627, + "presenting": 15628, + "##32": 15629, + "##olds": 15630, + "185": 15631, + "badass": 15632, + "troops": 15633, + "cellar": 15634, + "thrown": 15635, + "efficient": 15636, + "pebble": 15637, + "##ele": 15638, + "wedge": 15639, + "cruiser": 15640, + "luminous": 15641, + "plym": 15642, + "installing": 15643, + "carefully": 15644, + "sewer": 15645, + "democratic": 15646, + "sham": 15647, + "fairies": 15648, + "##erie": 15649, + "autism": 15650, + "measurements": 15651, + "ballerina": 15652, + "dune": 15653, + "indigenous": 15654, + "picket": 15655, + "builders": 15656, + "furious": 15657, + "buzz": 15658, + "playoff": 15659, + "charlie": 15660, + "tug": 15661, + "reservoir": 15662, + "spok": 15663, + "acre": 15664, + "ripped": 15665, + "flights": 15666, + "##oria": 15667, + "bachelor": 15668, + "##ilo": 15669, + "autograph": 15670, + "trimming": 15671, + "digest": 15672, + "slab": 15673, + "ankara": 15674, + "88": 15675, + "ribs": 15676, + "brewing": 15677, + "geography": 15678, + "adela": 15679, + "curls": 15680, + "wi": 15681, + "oun": 15682, + "lick": 15683, + "##ael": 15684, + "valve": 15685, + "positions": 15686, + "euc": 15687, + "pem": 15688, + "reed": 15689, + "backs": 15690, + "operated": 15691, + "##cos": 15692, + "shoots": 15693, + "tights": 15694, + "##ube": 15695, + "renew": 15696, + "##aba": 15697, + "harmony": 15698, + "plymouth": 15699, + "oscar": 15700, + "adelaide": 15701, + "asking": 15702, + "decades": 15703, + "passed": 15704, + "aluminium": 15705, + "tackle": 15706, + "officially": 15707, + "kayaking": 15708, + "voc": 15709, + "upholstery": 15710, + "enorm": 15711, + "veterans": 15712, + "##hand": 15713, + "##fare": 15714, + "diversity": 15715, + "asparagus": 15716, + "bridesmaids": 15717, + "1979": 15718, + "exceptional": 15719, + "lg": 15720, + "silly": 15721, + "reader": 15722, + "mice": 15723, + "clients": 15724, + "litter": 15725, + "##ivia": 15726, + "##lights": 15727, + "caulion": 15728, + "vitamin": 15729, + "delivers": 15730, + "stretching": 15731, + "rapper": 15732, + "enormous": 15733, + "##inoa": 15734, + "feelings": 15735, + "filing": 15736, + "hint": 15737, + "nile": 15738, + "scrub": 15739, + "mystic": 15740, + "shaking": 15741, + "participate": 15742, + "imagination": 15743, + "reun": 15744, + "##undai": 15745, + "##agara": 15746, + "grille": 15747, + "hyundai": 15748, + "chester": 15749, + "mathem": 15750, + "licens": 15751, + "##rium": 15752, + "transformed": 15753, + "situation": 15754, + "kam": 15755, + "goa": 15756, + "tobac": 15757, + "speedometer": 15758, + "reported": 15759, + "louvre": 15760, + "handcraft": 15761, + "mara": 15762, + "lilies": 15763, + "peeling": 15764, + "oceans": 15765, + "artifacts": 15766, + "##illery": 15767, + "pomegran": 15768, + "700": 15769, + "causes": 15770, + "selfie": 15771, + "suzuki": 15772, + "inaug": 15773, + "nouveau": 15774, + "jane": 15775, + "sided": 15776, + "##x12": 15777, + "smoked": 15778, + "##cases": 15779, + "ell": 15780, + "patt": 15781, + "sak": 15782, + "##fts": 15783, + "neighbourhood": 15784, + "##49": 15785, + "tears": 15786, + "commonly": 15787, + "cracked": 15788, + "gauges": 15789, + "tabby": 15790, + "##isation": 15791, + "walla": 15792, + "appropri": 15793, + "distinctive": 15794, + "wrangler": 15795, + "tat": 15796, + "pulls": 15797, + "40k": 15798, + "cosmos": 15799, + "##taking": 15800, + "sources": 15801, + "fudge": 15802, + "described": 15803, + "##ogany": 15804, + "tread": 15805, + "saturn": 15806, + "couldn": 15807, + "david": 15808, + "cosmic": 15809, + "alloy": 15810, + "thro": 15811, + "equestrian": 15812, + "compete": 15813, + "mahogany": 15814, + "sofas": 15815, + "barbed": 15816, + "termin": 15817, + "grandmother": 15818, + "timer": 15819, + "fusion": 15820, + "organisms": 15821, + "##ardi": 15822, + "degre": 15823, + "chronicle": 15824, + "##eo": 15825, + "camaro": 15826, + "serp": 15827, + "sporty": 15828, + "borealis": 15829, + "1993": 15830, + "composite": 15831, + "cheerle": 15832, + "cashmere": 15833, + "prad": 15834, + "waffles": 15835, + "prest": 15836, + "enclosed": 15837, + "penn": 15838, + "locked": 15839, + "meanings": 15840, + "##orse": 15841, + "ferrets": 15842, + "bourbon": 15843, + "classics": 15844, + "artworks": 15845, + "##retched": 15846, + "##ctica": 15847, + "occur": 15848, + "1930": 15849, + "rockets": 15850, + "elop": 15851, + "ala": 15852, + "meant": 15853, + "certificate": 15854, + "exploration": 15855, + "upgrade": 15856, + "assigned": 15857, + "riverside": 15858, + "bankn": 15859, + "krug": 15860, + "belize": 15861, + "clearing": 15862, + "degrees": 15863, + "whisper": 15864, + "odys": 15865, + "##store": 15866, + "encoun": 15867, + "venture": 15868, + "scall": 15869, + "musk": 15870, + "tobacco": 15871, + "drying": 15872, + "decade": 15873, + "1991": 15874, + "dunge": 15875, + "##holm": 15876, + "believed": 15877, + "vault": 15878, + "recovery": 15879, + "schedule": 15880, + "bunnies": 15881, + "vacc": 15882, + "lemonade": 15883, + "ottawa": 15884, + "decks": 15885, + "focused": 15886, + "whate": 15887, + "inexp": 15888, + "subtle": 15889, + "developing": 15890, + "stable": 15891, + "##tingu": 15892, + "peter": 15893, + "juicy": 15894, + "eden": 15895, + "whatever": 15896, + "approved": 15897, + "tavern": 15898, + "manufactured": 15899, + "pint": 15900, + "perhap": 15901, + "celest": 15902, + "niagara": 15903, + "perhaps": 15904, + "bins": 15905, + "organizer": 15906, + "meatlo": 15907, + "buddh": 15908, + "runners": 15909, + "spare": 15910, + "##46": 15911, + "encourage": 15912, + "revenge": 15913, + "trike": 15914, + "##oyd": 15915, + "requires": 15916, + "vespa": 15917, + "##pp": 15918, + "compreh": 15919, + "prove": 15920, + "enjoyed": 15921, + "acres": 15922, + "brun": 15923, + "committ": 15924, + "grande": 15925, + "beers": 15926, + "comfy": 15927, + "proph": 15928, + "##aurus": 15929, + "##oved": 15930, + "comparison": 15931, + "colombia": 15932, + "##bc": 15933, + "depos": 15934, + "rehears": 15935, + "raccoon": 15936, + "tails": 15937, + "spart": 15938, + "##eaway": 15939, + "committee": 15940, + "fert": 15941, + "tass": 15942, + "attitude": 15943, + "joke": 15944, + "##inian": 15945, + "melting": 15946, + "fiesta": 15947, + "remark": 15948, + "choir": 15949, + "lockdown": 15950, + "puddle": 15951, + "numerous": 15952, + "roadster": 15953, + "minim": 15954, + "racks": 15955, + "fortune": 15956, + "sympt": 15957, + "bav": 15958, + "antarctica": 15959, + "##33": 15960, + "daisies": 15961, + "lighted": 15962, + "##hesive": 15963, + "yards": 15964, + "kur": 15965, + "patagon": 15966, + "iceberg": 15967, + "thermometer": 15968, + "tonight": 15969, + "##79": 15970, + "##fc": 15971, + "##ju": 15972, + "treated": 15973, + "visits": 15974, + "toucan": 15975, + "handcrafted": 15976, + "baza": 15977, + "luxe": 15978, + "elopement": 15979, + "odyssey": 15980, + "sector": 15981, + "##houses": 15982, + "producer": 15983, + "touchdown": 15984, + "dag": 15985, + "fot": 15986, + "vast": 15987, + "##mail": 15988, + "acting": 15989, + "methods": 15990, + "scoreboard": 15991, + "gundam": 15992, + "supern": 15993, + "versace": 15994, + "alexand": 15995, + "lefto": 15996, + "mandal": 15997, + "injection": 15998, + "giv": 15999, + "##gal": 16000, + "1988": 16001, + "ambass": 16002, + "sixth": 16003, + "spending": 16004, + "boule": 16005, + "expansion": 16006, + "bid": 16007, + "status": 16008, + "malta": 16009, + "hunger": 16010, + "##ulty": 16011, + "oaks": 16012, + "catam": 16013, + "subaru": 16014, + "novelty": 16015, + "##hawks": 16016, + "laur": 16017, + "myst": 16018, + "equation": 16019, + "clog": 16020, + "refere": 16021, + "respond": 16022, + "1972": 16023, + "holi": 16024, + "presc": 16025, + "visitor": 16026, + "1977": 16027, + "pines": 16028, + "spacecraft": 16029, + "accompan": 16030, + "bleach": 16031, + "conservatory": 16032, + "kruger": 16033, + "checks": 16034, + "faucet": 16035, + "meadows": 16036, + "##lop": 16037, + "excav": 16038, + "locker": 16039, + "stitching": 16040, + "mega": 16041, + "monaco": 16042, + "engineers": 16043, + "defender": 16044, + "signing": 16045, + "somers": 16046, + "campground": 16047, + "girly": 16048, + "villages": 16049, + "##conn": 16050, + "gateway": 16051, + "fascin": 16052, + "feast": 16053, + "pi": 16054, + "##ounts": 16055, + "lind": 16056, + "sari": 16057, + "bagu": 16058, + "inexpensive": 16059, + "tran": 16060, + "##inals": 16061, + "##urb": 16062, + "wu": 16063, + "shav": 16064, + "thermal": 16065, + "catfish": 16066, + "stamps": 16067, + "##osaurus": 16068, + "ment": 16069, + "wigs": 16070, + "sic": 16071, + "mining": 16072, + "dora": 16073, + "thom": 16074, + "##andy": 16075, + "calories": 16076, + "polymer": 16077, + "nz": 16078, + "grinch": 16079, + "forgot": 16080, + "clover": 16081, + "ended": 16082, + "practices": 16083, + "possibly": 16084, + "punjab": 16085, + "kiw": 16086, + "##anth": 16087, + "deadpool": 16088, + "liner": 16089, + "hermes": 16090, + "anx": 16091, + "thorn": 16092, + "cruc": 16093, + "##ishi": 16094, + "daddy": 16095, + "##arf": 16096, + "sheff": 16097, + "furnish": 16098, + "determin": 16099, + "pleasant": 16100, + "##eet": 16101, + "commit": 16102, + "request": 16103, + "tells": 16104, + "overnight": 16105, + "forklift": 16106, + "imposs": 16107, + "lung": 16108, + "##elve": 16109, + "phen": 16110, + "##ulting": 16111, + "engaged": 16112, + "##uma": 16113, + "cod": 16114, + "margar": 16115, + "##atta": 16116, + "email": 16117, + "1987": 16118, + "resale": 16119, + "authority": 16120, + "accommodation": 16121, + "sheffield": 16122, + "##cuts": 16123, + "##mate": 16124, + "tud": 16125, + "weeds": 16126, + "sombr": 16127, + "strategies": 16128, + "walker": 16129, + "##efa": 16130, + "sections": 16131, + "appetizer": 16132, + "##ias": 16133, + "aggress": 16134, + "ale": 16135, + "pies": 16136, + "baths": 16137, + "surrey": 16138, + "outstretched": 16139, + "citizens": 16140, + "evidence": 16141, + "flop": 16142, + "unveiled": 16143, + "divided": 16144, + "fixing": 16145, + "stole": 16146, + "##onym": 16147, + "conclu": 16148, + "volt": 16149, + "bbc": 16150, + "molec": 16151, + "regal": 16152, + "freezer": 16153, + "##ption": 16154, + "uefa": 16155, + "##ifi": 16156, + "##ecting": 16157, + "##eart": 16158, + "crowded": 16159, + "##agers": 16160, + "staying": 16161, + "bacteria": 16162, + "manuscript": 16163, + "characteristics": 16164, + "transformation": 16165, + "happens": 16166, + "mama": 16167, + "license": 16168, + "colony": 16169, + "vote": 16170, + "compost": 16171, + "##erto": 16172, + "##igate": 16173, + "chame": 16174, + "##ifer": 16175, + "elbow": 16176, + "dens": 16177, + "anywhere": 16178, + "competes": 16179, + "janeiro": 16180, + "finch": 16181, + "reflective": 16182, + "domes": 16183, + "bing": 16184, + "cradle": 16185, + "1940s": 16186, + "patagonia": 16187, + "losing": 16188, + "tarant": 16189, + "wasn": 16190, + "microphones": 16191, + "sier": 16192, + "fid": 16193, + "specialist": 16194, + "influence": 16195, + "##ika": 16196, + "bits": 16197, + "fortress": 16198, + "##heim": 16199, + "jaipur": 16200, + "piled": 16201, + "##iary": 16202, + "urch": 16203, + "dna": 16204, + "pm": 16205, + "shack": 16206, + "funguss": 16207, + "surgical": 16208, + "##zilla": 16209, + "briefcase": 16210, + "norse": 16211, + "##tenham": 16212, + "entrep": 16213, + "estim": 16214, + "maximum": 16215, + "entreprene": 16216, + "scrapbook": 16217, + "twelve": 16218, + "pecan": 16219, + "sketching": 16220, + "researchers": 16221, + "twigs": 16222, + "blower": 16223, + "##zzy": 16224, + "##tering": 16225, + "##oba": 16226, + "westwood": 16227, + "flavour": 16228, + "tend": 16229, + "##achi": 16230, + "##space": 16231, + "qut": 16232, + "15th": 16233, + "1978": 16234, + "unexp": 16235, + "detox": 16236, + "##ducing": 16237, + "aliens": 16238, + "##uing": 16239, + "##ibi": 16240, + "hurric": 16241, + "granny": 16242, + "jeff": 16243, + "symph": 16244, + "rede": 16245, + "tablecloth": 16246, + "atlant": 16247, + "athletics": 16248, + "bizar": 16249, + "pelica": 16250, + "pounds": 16251, + "##aded": 16252, + "chamele": 16253, + "grate": 16254, + "##angel": 16255, + "nativity": 16256, + "technologies": 16257, + "confir": 16258, + "925": 16259, + "##osen": 16260, + "allowing": 16261, + "heard": 16262, + "headset": 16263, + "uae": 16264, + "smoky": 16265, + "describe": 16266, + "mim": 16267, + "##enz": 16268, + "industries": 16269, + "squee": 16270, + "touches": 16271, + "dunk": 16272, + "yak": 16273, + "sequins": 16274, + "radiator": 16275, + "186": 16276, + "malibu": 16277, + "##atics": 16278, + "rangers": 16279, + "lc": 16280, + "1973": 16281, + "nurses": 16282, + "skates": 16283, + "integrated": 16284, + "fel": 16285, + "aspen": 16286, + "factors": 16287, + "patriots": 16288, + "souls": 16289, + "shredded": 16290, + "oats": 16291, + "blacks": 16292, + "pathway": 16293, + "shet": 16294, + "stockholm": 16295, + "naples": 16296, + "pembro": 16297, + "boulevard": 16298, + "fences": 16299, + "biology": 16300, + "##x1080": 16301, + "bis": 16302, + "##iy": 16303, + "nos": 16304, + "##asa": 16305, + "sequence": 16306, + "trainers": 16307, + "dachshund": 16308, + "dispenser": 16309, + "indigo": 16310, + "hag": 16311, + "fallout": 16312, + "grizzly": 16313, + "kisses": 16314, + "homestay": 16315, + "judged": 16316, + "pelic": 16317, + "chenn": 16318, + "grim": 16319, + "removable": 16320, + "immedi": 16321, + "patchwork": 16322, + "awarded": 16323, + "advantage": 16324, + "academic": 16325, + "territory": 16326, + "vip": 16327, + "cubic": 16328, + "ensemb": 16329, + "##bled": 16330, + "sheeps": 16331, + "peps": 16332, + "masquer": 16333, + "gotham": 16334, + "transmission": 16335, + "collectibles": 16336, + "propag": 16337, + "shin": 16338, + "mandatory": 16339, + "enjoys": 16340, + "110": 16341, + "60s": 16342, + "parmes": 16343, + "purs": 16344, + "laws": 16345, + "ald": 16346, + "coolest": 16347, + "balm": 16348, + "trig": 16349, + "newcastle": 16350, + "serpent": 16351, + "cuban": 16352, + "grip": 16353, + "grant": 16354, + "crested": 16355, + "##pieces": 16356, + "bonnet": 16357, + "blessed": 16358, + "bazaar": 16359, + "mou": 16360, + "proport": 16361, + "confused": 16362, + "hart": 16363, + "##omical": 16364, + "indie": 16365, + "gras": 16366, + "util": 16367, + "monuments": 16368, + "packet": 16369, + "banded": 16370, + "synth": 16371, + "##esign": 16372, + "alch": 16373, + "sweeth": 16374, + "##page": 16375, + "pinwheels": 16376, + "godzilla": 16377, + "foster": 16378, + "picturesque": 16379, + "boa": 16380, + "knew": 16381, + "dreaming": 16382, + "skelet": 16383, + "##bob": 16384, + "hangar": 16385, + "pais": 16386, + "hing": 16387, + "princesses": 16388, + "##rupt": 16389, + "ng": 16390, + "medals": 16391, + "factor": 16392, + "##raid": 16393, + "pillar": 16394, + "opportunities": 16395, + "##pur": 16396, + "hyatt": 16397, + "checked": 16398, + "hummingbird": 16399, + "appearing": 16400, + "amongst": 16401, + "dwell": 16402, + "slides": 16403, + "introducing": 16404, + "billiard": 16405, + "recommend": 16406, + "phuk": 16407, + "handy": 16408, + "lexus": 16409, + "tramp": 16410, + "impossible": 16411, + "cheesy": 16412, + "teamwork": 16413, + "ponds": 16414, + "chennai": 16415, + "aka": 16416, + "dah": 16417, + "tarm": 16418, + "denali": 16419, + "diploma": 16420, + "ethiopia": 16421, + "kites": 16422, + "primitive": 16423, + "moor": 16424, + "sees": 16425, + "municipal": 16426, + "phuket": 16427, + "itiner": 16428, + "xx": 16429, + "##x5": 16430, + "patent": 16431, + "ponch": 16432, + "depression": 16433, + "duffel": 16434, + "jones": 16435, + "carpent": 16436, + "1976": 16437, + "frying": 16438, + "1963": 16439, + "folder": 16440, + "dors": 16441, + "##inium": 16442, + "accur": 16443, + "till": 16444, + "ds": 16445, + "policy": 16446, + "limestone": 16447, + "wound": 16448, + "##arily": 16449, + "stood": 16450, + "surprising": 16451, + "obstacle": 16452, + "bearing": 16453, + "une": 16454, + "flows": 16455, + "squat": 16456, + "assess": 16457, + "tors": 16458, + "##eto": 16459, + "shetland": 16460, + "dia": 16461, + "parsley": 16462, + "marit": 16463, + "artisan": 16464, + "hallmark": 16465, + "##lia": 16466, + "##stown": 16467, + "cous": 16468, + "draws": 16469, + "wikip": 16470, + "sig": 16471, + "wildflowers": 16472, + "masquerade": 16473, + "barns": 16474, + "mtv": 16475, + "flexible": 16476, + "##stream": 16477, + "passage": 16478, + "matrix": 16479, + "hindi": 16480, + "breathtaking": 16481, + "qatar": 16482, + "swap": 16483, + "occult": 16484, + "philippine": 16485, + "##chment": 16486, + "canister": 16487, + "##ieties": 16488, + "pis": 16489, + "chosen": 16490, + "12th": 16491, + "##acent": 16492, + "1920x1080": 16493, + "shores": 16494, + "permanent": 16495, + "drizz": 16496, + "cardinal": 16497, + "62": 16498, + "##itled": 16499, + "broth": 16500, + "playhouse": 16501, + "powerpoint": 16502, + "clearly": 16503, + "strug": 16504, + "chalet": 16505, + "##uled": 16506, + "##ifles": 16507, + "dispos": 16508, + "rulers": 16509, + "orbit": 16510, + "##ulum": 16511, + "ghana": 16512, + "##cream": 16513, + "improvement": 16514, + "ponytail": 16515, + "noise": 16516, + "mirrored": 16517, + "sorry": 16518, + "##tr": 16519, + "controlled": 16520, + "rut": 16521, + "moms": 16522, + "hosts": 16523, + "venus": 16524, + "##upe": 16525, + "buddhism": 16526, + "pistol": 16527, + "##98": 16528, + "summary": 16529, + "locals": 16530, + "##master": 16531, + "handed": 16532, + "plumb": 16533, + "references": 16534, + "bishop": 16535, + "composed": 16536, + "tartan": 16537, + "users": 16538, + "hugs": 16539, + "drugs": 16540, + "batch": 16541, + "principal": 16542, + "embrace": 16543, + "established": 16544, + "honeycomb": 16545, + "ero": 16546, + "seagul": 16547, + "chak": 16548, + "celestial": 16549, + "vary": 16550, + "skier": 16551, + "combines": 16552, + "cultures": 16553, + "li": 16554, + "##x7": 16555, + "eyebrows": 16556, + "answers": 16557, + "weak": 16558, + "##mic": 16559, + "extinct": 16560, + "furnishings": 16561, + "wikipedia": 16562, + "animales": 16563, + "silence": 16564, + "antelo": 16565, + "bentley": 16566, + "quinoa": 16567, + "hardcover": 16568, + "##aphone": 16569, + "##words": 16570, + "ostrice": 16571, + "botanic": 16572, + "knees": 16573, + "depart": 16574, + "astrology": 16575, + "leatherback": 16576, + "jerseys": 16577, + "kaz": 16578, + "##inn": 16579, + "pickles": 16580, + "chronicles": 16581, + "loll": 16582, + "##rew": 16583, + "disabled": 16584, + "paisley": 16585, + "redesign": 16586, + "14th": 16587, + "chose": 16588, + "##alypt": 16589, + "winding": 16590, + "daytona": 16591, + "eucalypt": 16592, + "kash": 16593, + "spruce": 16594, + "starter": 16595, + "misty": 16596, + "gravity": 16597, + "tudor": 16598, + "##gain": 16599, + "alert": 16600, + "millenn": 16601, + "frosted": 16602, + "fever": 16603, + "editable": 16604, + "grandfather": 16605, + "##going": 16606, + "sack": 16607, + "amphibious": 16608, + "##fit": 16609, + "broke": 16610, + "fluid": 16611, + "bookshel": 16612, + "buggy": 16613, + "##ipei": 16614, + "latch": 16615, + "troll": 16616, + "1982": 16617, + "somerset": 16618, + "repor": 16619, + "suburban": 16620, + "##orious": 16621, + "##arin": 16622, + "loyal": 16623, + "lp": 16624, + "taipei": 16625, + "##ocross": 16626, + "promise": 16627, + "motifs": 16628, + "leon": 16629, + "afraid": 16630, + "jumper": 16631, + "1971": 16632, + "breathing": 16633, + "prayers": 16634, + "royce": 16635, + "##ders": 16636, + "gladi": 16637, + "crashing": 16638, + "laurel": 16639, + "##irut": 16640, + "croquet": 16641, + "purpos": 16642, + "figurines": 16643, + "beirut": 16644, + "ecant": 16645, + "thirty": 16646, + "finishes": 16647, + "escal": 16648, + "vibe": 16649, + "##horse": 16650, + "##cs": 16651, + "assorted": 16652, + "mats": 16653, + "sox": 16654, + "raft": 16655, + "adjacent": 16656, + "fragrance": 16657, + "latex": 16658, + "weave": 16659, + "partnership": 16660, + "giants": 16661, + "sang": 16662, + "tastes": 16663, + "##eshire": 16664, + "mcdon": 16665, + "footprints": 16666, + "slots": 16667, + "mantiss": 16668, + "##warts": 16669, + "brushed": 16670, + "sierra": 16671, + "sq": 16672, + "exquis": 16673, + "pepsi": 16674, + "walmart": 16675, + "heav": 16676, + "processor": 16677, + "1955": 16678, + "wreaths": 16679, + "##anoi": 16680, + "thief": 16681, + "elevated": 16682, + "bizarre": 16683, + "mate": 16684, + "##ogne": 16685, + "administration": 16686, + "shi": 16687, + "advance": 16688, + "stanley": 16689, + "fundraiser": 16690, + "##cks": 16691, + "clark": 16692, + "salb": 16693, + "mg": 16694, + "atom": 16695, + "michael": 16696, + "campsite": 16697, + "jade": 16698, + "experiences": 16699, + "namibia": 16700, + "rats": 16701, + "envelop": 16702, + "##idae": 16703, + "dropped": 16704, + "##iat": 16705, + "upstairs": 16706, + "aub": 16707, + "cs": 16708, + "wines": 16709, + "accessible": 16710, + "slay": 16711, + "parmesan": 16712, + "dang": 16713, + "##owl": 16714, + "1500": 16715, + "sacra": 16716, + "israeli": 16717, + "vic": 16718, + "cheek": 16719, + "monk": 16720, + "landmarks": 16721, + "pomegranate": 16722, + "stacks": 16723, + "azte": 16724, + "junction": 16725, + "sheds": 16726, + "reaction": 16727, + "1974": 16728, + "compared": 16729, + "##55": 16730, + "stinger": 16731, + "handker": 16732, + "watched": 16733, + "khaki": 16734, + "cladding": 16735, + "handkerchief": 16736, + "sill": 16737, + "trial": 16738, + "adhesive": 16739, + "schol": 16740, + "destiny": 16741, + "tweed": 16742, + "enfor": 16743, + "ketch": 16744, + "birthdays": 16745, + "consisting": 16746, + "sahara": 16747, + "comment": 16748, + "miracle": 16749, + "applying": 16750, + "tarmac": 16751, + "scared": 16752, + "smiley": 16753, + "wow": 16754, + "diapers": 16755, + "extensions": 16756, + "mai": 16757, + "##uration": 16758, + "hercules": 16759, + "spaniel": 16760, + "dotted": 16761, + "stroller": 16762, + "skater": 16763, + "grooms": 16764, + "pradesh": 16765, + "unexpected": 16766, + "##bok": 16767, + "genius": 16768, + "bodybuild": 16769, + "##ilation": 16770, + "crumb": 16771, + "grandma": 16772, + "dumpl": 16773, + "##imedia": 16774, + "spongebob": 16775, + "knobs": 16776, + "belts": 16777, + "kl": 16778, + "grew": 16779, + "perfor": 16780, + "tamil": 16781, + "tons": 16782, + "##clu": 16783, + "communities": 16784, + "faithful": 16785, + "stormy": 16786, + "hanoi": 16787, + "toned": 16788, + "1983": 16789, + "bored": 16790, + "kens": 16791, + "snowflake": 16792, + "eucalyptus": 16793, + "moore": 16794, + "chaos": 16795, + "weighing": 16796, + "##ilant": 16797, + "extraord": 16798, + "sweetheart": 16799, + "##atural": 16800, + "chops": 16801, + "18k": 16802, + "##ader": 16803, + "trifle": 16804, + "##ensional": 16805, + "idol": 16806, + "highlighted": 16807, + "megaphone": 16808, + "silicone": 16809, + "daff": 16810, + "##ndham": 16811, + "dozen": 16812, + "aster": 16813, + "##ova": 16814, + "telling": 16815, + "polic": 16816, + "museums": 16817, + "cultiv": 16818, + "hornhorn": 16819, + "koh": 16820, + "hungary": 16821, + "infantry": 16822, + "##jan": 16823, + "rentals": 16824, + "lak": 16825, + "meatloaf": 16826, + "##uro": 16827, + "roaring": 16828, + "squashs": 16829, + "segment": 16830, + "diabet": 16831, + "burrito": 16832, + "gren": 16833, + "chainmail": 16834, + "operator": 16835, + "seagull": 16836, + "enterprise": 16837, + "atlantis": 16838, + "editor": 16839, + "spoke": 16840, + "trio": 16841, + "##screen": 16842, + "giza": 16843, + "wag": 16844, + "gently": 16845, + "garter": 16846, + "horsesh": 16847, + "hogwarts": 16848, + "cuddling": 16849, + "domain": 16850, + "dw": 16851, + "lakers": 16852, + "##ried": 16853, + "30th": 16854, + "paragraph": 16855, + "bedside": 16856, + "mund": 16857, + "crushed": 16858, + "##hawk": 16859, + "bras": 16860, + "snork": 16861, + "scores": 16862, + "##isible": 16863, + "teak": 16864, + "murder": 16865, + "appreciation": 16866, + "holly": 16867, + "fiji": 16868, + "miller": 16869, + "languages": 16870, + "##llis": 16871, + "ports": 16872, + "organizing": 16873, + "clownfish": 16874, + "purposes": 16875, + "##ei": 16876, + "ross": 16877, + "##elles": 16878, + "evol": 16879, + "merchandise": 16880, + "weld": 16881, + "maser": 16882, + "encoura": 16883, + "symphony": 16884, + "reflect": 16885, + "surroundings": 16886, + "restroom": 16887, + "mechanism": 16888, + "dhabi": 16889, + "itinerary": 16890, + "##orf": 16891, + "##ulator": 16892, + "meerkats": 16893, + "##estrians": 16894, + "960": 16895, + "flooded": 16896, + "jup": 16897, + "sweep": 16898, + "launching": 16899, + "vivid": 16900, + "stairway": 16901, + "splend": 16902, + "##awn": 16903, + "physics": 16904, + "homestead": 16905, + "symptoms": 16906, + "##urst": 16907, + "noble": 16908, + "vibes": 16909, + "oscars": 16910, + "burner": 16911, + "##ptons": 16912, + "attrib": 16913, + "companion": 16914, + "motorbike": 16915, + "##caa": 16916, + "turbine": 16917, + "chairman": 16918, + "rd": 16919, + "redu": 16920, + "complim": 16921, + "1940": 16922, + "byz": 16923, + "nylon": 16924, + "alter": 16925, + "cms": 16926, + "dat": 16927, + "alto": 16928, + "opin": 16929, + "explain": 16930, + "gibson": 16931, + "haircuts": 16932, + "releases": 16933, + "expressions": 16934, + "a4": 16935, + "bonus": 16936, + "yorker": 16937, + "columbus": 16938, + "mp3": 16939, + "ramen": 16940, + "likes": 16941, + "condominium": 16942, + "blast": 16943, + "interested": 16944, + "continued": 16945, + "ensemble": 16946, + "86": 16947, + "sling": 16948, + "pandas": 16949, + "orangutan": 16950, + "tranqu": 16951, + "bookstore": 16952, + "peg": 16953, + "artichs": 16954, + "flutes": 16955, + "keeper": 16956, + "trims": 16957, + "aloe": 16958, + "reign": 16959, + "grind": 16960, + "##encies": 16961, + "dimensional": 16962, + "wyndham": 16963, + "moped": 16964, + "##ulus": 16965, + "immig": 16966, + "ls": 16967, + "wemble": 16968, + "secondary": 16969, + "sprinkles": 16970, + "wembley": 16971, + "unp": 16972, + "strikes": 16973, + "sprint": 16974, + "increased": 16975, + "versa": 16976, + "percussion": 16977, + "##ubishi": 16978, + "reacts": 16979, + "pedestrians": 16980, + "mitsubishi": 16981, + "jupiter": 16982, + "hq": 16983, + "pancake": 16984, + "ambassador": 16985, + "##uran": 16986, + "regarding": 16987, + "psychology": 16988, + "infinite": 16989, + "##iko": 16990, + "stopwatch": 16991, + "applications": 16992, + "350": 16993, + "cole": 16994, + "flyers": 16995, + "##ontin": 16996, + "pastries": 16997, + "stepping": 16998, + "ecology": 16999, + "rar": 17000, + "sash": 17001, + "destroyer": 17002, + "##iscus": 17003, + "johnson": 17004, + "manufacturers": 17005, + "beloved": 17006, + "1600": 17007, + "resistance": 17008, + "easiest": 17009, + "brows": 17010, + "##iliar": 17011, + "flea": 17012, + "maya": 17013, + "1957": 17014, + "atrium": 17015, + "cosy": 17016, + "presence": 17017, + "forehead": 17018, + "visualization": 17019, + "appropriate": 17020, + "bump": 17021, + "##aked": 17022, + "harvard": 17023, + "haute": 17024, + "##fruit": 17025, + "pedal": 17026, + "boiled": 17027, + "aban": 17028, + "gemstone": 17029, + "notting": 17030, + "diagn": 17031, + "assembled": 17032, + "phenom": 17033, + "##bull": 17034, + "conte": 17035, + "witches": 17036, + "##omed": 17037, + "##fords": 17038, + "oreo": 17039, + "stur": 17040, + "mike": 17041, + "coaster": 17042, + "leopards": 17043, + "hover": 17044, + "kel": 17045, + "conv": 17046, + "whisk": 17047, + "##boro": 17048, + "franklin": 17049, + "ncaa": 17050, + "blaze": 17051, + "packs": 17052, + "##ois": 17053, + "launches": 17054, + "terrarium": 17055, + "intended": 17056, + "mitts": 17057, + "commence": 17058, + "trouble": 17059, + "##western": 17060, + "planks": 17061, + "talent": 17062, + "gorillas": 17063, + "pinwheel": 17064, + "symmetry": 17065, + "tid": 17066, + "settle": 17067, + "125": 17068, + "km": 17069, + "scouts": 17070, + "maltese": 17071, + "voyage": 17072, + "nerv": 17073, + "##aton": 17074, + "dominic": 17075, + "enemy": 17076, + "meaningful": 17077, + "reinfor": 17078, + "debate": 17079, + "fate": 17080, + "dense": 17081, + "resolve": 17082, + "suffering": 17083, + "cabinetry": 17084, + "cherries": 17085, + "flashlight": 17086, + "##gio": 17087, + "archway": 17088, + "verde": 17089, + "pesto": 17090, + "canned": 17091, + "##cca": 17092, + "charter": 17093, + "stupid": 17094, + "preserved": 17095, + "representation": 17096, + "##mand": 17097, + "income": 17098, + "cruises": 17099, + "encyc": 17100, + "comforter": 17101, + "maypole": 17102, + "civilization": 17103, + "squirrels": 17104, + "rests": 17105, + "consumer": 17106, + "ravens": 17107, + "exquisite": 17108, + "71": 17109, + "##vant": 17110, + "##pus": 17111, + "##cracker": 17112, + "1990s": 17113, + "##onymous": 17114, + "sells": 17115, + "supernatural": 17116, + "##iency": 17117, + "xs": 17118, + "##bi": 17119, + "rosemary": 17120, + "pulp": 17121, + "bog": 17122, + "dope": 17123, + "##yx": 17124, + "##storm": 17125, + "viewer": 17126, + "confidence": 17127, + "bangal": 17128, + "fractal": 17129, + "encyclop": 17130, + "##brook": 17131, + "petite": 17132, + "laughs": 17133, + "rage": 17134, + "homework": 17135, + "repaired": 17136, + "brownies": 17137, + "crops": 17138, + "9th": 17139, + "bora": 17140, + "##mus": 17141, + "beasts": 17142, + "puerto": 17143, + "nit": 17144, + "sacrament": 17145, + "getaway": 17146, + "abund": 17147, + "darkest": 17148, + "crowns": 17149, + "##garia": 17150, + "lav": 17151, + "vern": 17152, + "funko": 17153, + "genesis": 17154, + "11th": 17155, + "sheriff": 17156, + "##mi": 17157, + "avon": 17158, + "1400": 17159, + "podcast": 17160, + "boring": 17161, + "trays": 17162, + "bumble": 17163, + "cots": 17164, + "wren": 17165, + "differences": 17166, + "obl": 17167, + "followed": 17168, + "decide": 17169, + "outs": 17170, + "grasses": 17171, + "soho": 17172, + "granola": 17173, + "splashing": 17174, + "nigeria": 17175, + "##kor": 17176, + "seek": 17177, + "##borne": 17178, + "nr": 17179, + "spirits": 17180, + "bulldogs": 17181, + "resem": 17182, + "heath": 17183, + "refined": 17184, + "79": 17185, + "mk": 17186, + "crum": 17187, + "rud": 17188, + "shing": 17189, + "quint": 17190, + "warfare": 17191, + "##ymph": 17192, + "peppermint": 17193, + "##lan": 17194, + "generator": 17195, + "fascinating": 17196, + "auburn": 17197, + "dances": 17198, + "##ceed": 17199, + "bulgaria": 17200, + "corporation": 17201, + "##ules": 17202, + "maserati": 17203, + "salted": 17204, + "##ledon": 17205, + "##iph": 17206, + "treatments": 17207, + "particles": 17208, + "erupt": 17209, + "melt": 17210, + "cort": 17211, + "mascul": 17212, + "wouldn": 17213, + "bangalore": 17214, + "vr": 17215, + "mete": 17216, + "stored": 17217, + "novels": 17218, + "##atively": 17219, + "clasp": 17220, + "hotdog": 17221, + "scarlet": 17222, + "shaved": 17223, + "macar": 17224, + "77": 17225, + "regard": 17226, + "protector": 17227, + "acqu": 17228, + "evalu": 17229, + "strat": 17230, + "synd": 17231, + "diagon": 17232, + "citizen": 17233, + "gil": 17234, + "harvester": 17235, + "versions": 17236, + "collectors": 17237, + "ai": 17238, + "floyd": 17239, + "lipsticks": 17240, + "kai": 17241, + "backed": 17242, + "supports": 17243, + "nervous": 17244, + "##gt": 17245, + "##erves": 17246, + "##aux": 17247, + "extensive": 17248, + "insulated": 17249, + "axis": 17250, + "heated": 17251, + "nightclu": 17252, + "obtain": 17253, + "customize": 17254, + "seren": 17255, + "crackers": 17256, + "requirements": 17257, + "vw": 17258, + "inning": 17259, + "renovations": 17260, + "pollen": 17261, + "eater": 17262, + "wimb": 17263, + "boke": 17264, + "incredibly": 17265, + "##ifa": 17266, + "##ello": 17267, + "uganda": 17268, + "placing": 17269, + "server": 17270, + "rugged": 17271, + "plumbing": 17272, + "vital": 17273, + "flesh": 17274, + "responsible": 17275, + "courage": 17276, + "bru": 17277, + "##akarta": 17278, + "varieties": 17279, + "wells": 17280, + "witness": 17281, + "churches": 17282, + "mahar": 17283, + "gilt": 17284, + "cancun": 17285, + "barte": 17286, + "studded": 17287, + "mild": 17288, + "creed": 17289, + "crater": 17290, + "##cers": 17291, + "ghib": 17292, + "fires": 17293, + "minion": 17294, + "##ikh": 17295, + "rifles": 17296, + "##idents": 17297, + "bandage": 17298, + "candidate": 17299, + "woodstock": 17300, + "lifts": 17301, + "individuals": 17302, + "byzantine": 17303, + "##tense": 17304, + "faster": 17305, + "hull": 17306, + "tying": 17307, + "25th": 17308, + "fuzzy": 17309, + "74": 17310, + "neckline": 17311, + "punish": 17312, + "boku": 17313, + "painters": 17314, + "macaw": 17315, + "chopping": 17316, + "ghibli": 17317, + "founded": 17318, + "14k": 17319, + "nets": 17320, + "1800": 17321, + "1959": 17322, + "wolver": 17323, + "acne": 17324, + "steal": 17325, + "weathered": 17326, + "bist": 17327, + "##izable": 17328, + "wicked": 17329, + "##owned": 17330, + "awk": 17331, + "##abulary": 17332, + "##oint": 17333, + "psal": 17334, + "politics": 17335, + "##aman": 17336, + "trimmed": 17337, + "extraordinary": 17338, + "swee": 17339, + "colleag": 17340, + "trilogy": 17341, + "formations": 17342, + "conditioner": 17343, + "tiered": 17344, + "96": 17345, + "headd": 17346, + "worry": 17347, + "medit": 17348, + "##eres": 17349, + "represented": 17350, + "approaching": 17351, + "poisonous": 17352, + "amplifier": 17353, + "maritime": 17354, + "masc": 17355, + "savory": 17356, + "tusc": 17357, + "strokes": 17358, + "##locks": 17359, + "swimmer": 17360, + "restr": 17361, + "shaded": 17362, + "rhyth": 17363, + "sorty": 17364, + "gibbon": 17365, + "sf": 17366, + "wwe": 17367, + "workspace": 17368, + "mousetra": 17369, + "wimbledon": 17370, + "communic": 17371, + "stops": 17372, + "detached": 17373, + "98": 17374, + "headboard": 17375, + "remaining": 17376, + "socket": 17377, + "cowgirl": 17378, + "##pled": 17379, + "surve": 17380, + "hamptons": 17381, + "1981": 17382, + "schnke": 17383, + "##iot": 17384, + "intense": 17385, + "shrub": 17386, + "##mandu": 17387, + "ht": 17388, + "triangular": 17389, + "nect": 17390, + "cyclist": 17391, + "bau": 17392, + "awkward": 17393, + "mary": 17394, + "paperback": 17395, + "gazel": 17396, + "atoms": 17397, + "parchment": 17398, + "portal": 17399, + "scratches": 17400, + "mound": 17401, + "##eric": 17402, + "separated": 17403, + "battleship": 17404, + "480": 17405, + "toes": 17406, + "rehearsal": 17407, + "kung": 17408, + "vuit": 17409, + "burned": 17410, + "nutri": 17411, + "diabetes": 17412, + "dazz": 17413, + "nag": 17414, + "lew": 17415, + "familiar": 17416, + "symbolic": 17417, + "kennedy": 17418, + "commencement": 17419, + "mantel": 17420, + "protesters": 17421, + "calculator": 17422, + "polygonal": 17423, + "swag": 17424, + "edward": 17425, + "##arma": 17426, + "affect": 17427, + "welcoming": 17428, + "1962": 17429, + "touca": 17430, + "##arius": 17431, + "brad": 17432, + "grammy": 17433, + "springfield": 17434, + "raiders": 17435, + "bokeh": 17436, + "hue": 17437, + "ub": 17438, + "scream": 17439, + "embell": 17440, + "horseshoe": 17441, + "tuscany": 17442, + "gord": 17443, + "implement": 17444, + "artichich": 17445, + "lecture": 17446, + "landed": 17447, + "herring": 17448, + "nhl": 17449, + "mn": 17450, + "breads": 17451, + "urn": 17452, + "##orian": 17453, + "bronx": 17454, + "breeding": 17455, + "professionals": 17456, + "livest": 17457, + "retaining": 17458, + "##agos": 17459, + "defence": 17460, + "seller": 17461, + "hibiscus": 17462, + "pitbull": 17463, + "fade": 17464, + "##fair": 17465, + "ana": 17466, + "diab": 17467, + "refurb": 17468, + "razor": 17469, + "bobs": 17470, + "matters": 17471, + "docks": 17472, + "atv": 17473, + "porto": 17474, + "filters": 17475, + "001": 17476, + "earn": 17477, + "vuitton": 17478, + "##ampton": 17479, + "invisible": 17480, + "divide": 17481, + "mantle": 17482, + "blessing": 17483, + "o2": 17484, + "beards": 17485, + "illeg": 17486, + "strand": 17487, + "verte": 17488, + "hitch": 17489, + "##mir": 17490, + "hydraul": 17491, + "##arat": 17492, + "pugs": 17493, + "horseback": 17494, + "gat": 17495, + "crepe": 17496, + "##rock": 17497, + "combinations": 17498, + "bistro": 17499, + "kathmandu": 17500, + "##rig": 17501, + "##iba": 17502, + "##bender": 17503, + "semif": 17504, + "tuscan": 17505, + "suburb": 17506, + "groo": 17507, + "900": 17508, + "##atem": 17509, + "##inh": 17510, + "tabletop": 17511, + "pageant": 17512, + "estilo": 17513, + "ratio": 17514, + "##pm": 17515, + "hump": 17516, + "seriously": 17517, + "##ulted": 17518, + "trellis": 17519, + "provincial": 17520, + "referee": 17521, + "borneo": 17522, + "depicted": 17523, + "hoover": 17524, + "##upp": 17525, + "canary": 17526, + "travelers": 17527, + "97": 17528, + "##rated": 17529, + "puppets": 17530, + "convenient": 17531, + "typically": 17532, + "smoothies": 17533, + "goalkeeper": 17534, + "waits": 17535, + "dusty": 17536, + "73": 17537, + "compat": 17538, + "bangle": 17539, + "absolute": 17540, + "##aint": 17541, + "outc": 17542, + "benj": 17543, + "demons": 17544, + "interpret": 17545, + "standards": 17546, + "##obi": 17547, + "##lane": 17548, + "##ogical": 17549, + "##x1200": 17550, + "cotsw": 17551, + "serb": 17552, + "makers": 17553, + "advis": 17554, + "northeast": 17555, + "cypr": 17556, + "physi": 17557, + "vocabulary": 17558, + "##leys": 17559, + "drew": 17560, + "sealed": 17561, + "braces": 17562, + "marlin": 17563, + "medall": 17564, + "##ilk": 17565, + "defend": 17566, + "orchids": 17567, + "carat": 17568, + "##angea": 17569, + "1956": 17570, + "##ienced": 17571, + "alam": 17572, + "primate": 17573, + "dreamy": 17574, + "grilling": 17575, + "balanced": 17576, + "piercing": 17577, + "slowly": 17578, + "##uggling": 17579, + "bash": 17580, + "marqu": 17581, + "reserved": 17582, + "sloven": 17583, + "alban": 17584, + "##limited": 17585, + "unlimited": 17586, + "succulent": 17587, + "livestock": 17588, + "knots": 17589, + "ending": 17590, + "supper": 17591, + "licking": 17592, + "legion": 17593, + "prototype": 17594, + "prada": 17595, + "rattan": 17596, + "emotion": 17597, + "millennium": 17598, + "blackberry": 17599, + "explains": 17600, + "garnished": 17601, + "hotsp": 17602, + "gut": 17603, + "module": 17604, + "awakens": 17605, + "kern": 17606, + "##law": 17607, + "generations": 17608, + "powers": 17609, + "alum": 17610, + "arizona": 17611, + "bind": 17612, + "neur": 17613, + "rebu": 17614, + "spine": 17615, + "happened": 17616, + "##game": 17617, + "gardener": 17618, + "freel": 17619, + "brut": 17620, + "##osyst": 17621, + "rey": 17622, + "airways": 17623, + "projection": 17624, + "gamer": 17625, + "dominican": 17626, + "bartender": 17627, + "connects": 17628, + "occasions": 17629, + "blazers": 17630, + "sciences": 17631, + "townhouse": 17632, + "monthly": 17633, + "##zhou": 17634, + "flank": 17635, + "scoo": 17636, + "abroad": 17637, + "estates": 17638, + "##fortun": 17639, + "eternal": 17640, + "peony": 17641, + "guatem": 17642, + "coaching": 17643, + "pyren": 17644, + "congratulations": 17645, + "cougar": 17646, + "salem": 17647, + "healthier": 17648, + "propaganda": 17649, + "##fordshire": 17650, + "mandarin": 17651, + "prosp": 17652, + "airbender": 17653, + "compare": 17654, + "enthusiasts": 17655, + "envelopes": 17656, + "heal": 17657, + "tribes": 17658, + "##nda": 17659, + "rotweiler": 17660, + "ral": 17661, + "sleeps": 17662, + "crosswords": 17663, + "submitted": 17664, + "bulk": 17665, + "pyrenees": 17666, + "communications": 17667, + "##aver": 17668, + "##rob": 17669, + "printers": 17670, + "galap": 17671, + "flush": 17672, + "cypress": 17673, + "archaeological": 17674, + "chee": 17675, + "chibi": 17676, + "defensive": 17677, + "tiki": 17678, + "franch": 17679, + "stretches": 17680, + "speedway": 17681, + "scrubs": 17682, + "trapped": 17683, + "pilgrim": 17684, + "invas": 17685, + "##oooo": 17686, + "consc": 17687, + "returned": 17688, + "clouded": 17689, + "petrol": 17690, + "1958": 17691, + "naruto": 17692, + "delivering": 17693, + "hurricane": 17694, + "2d": 17695, + "gab": 17696, + "##flower": 17697, + "unfortun": 17698, + "sunb": 17699, + "readers": 17700, + "cyprus": 17701, + "gravy": 17702, + "prescription": 17703, + "truff": 17704, + "everest": 17705, + "killing": 17706, + "toasted": 17707, + "seventh": 17708, + "philosophy": 17709, + "benjamin": 17710, + "defined": 17711, + "crocodiles": 17712, + "tangled": 17713, + "kap": 17714, + "##enth": 17715, + "declar": 17716, + "moody": 17717, + "tasmania": 17718, + "nightclub": 17719, + "##hy": 17720, + "##kind": 17721, + "metaph": 17722, + "edgy": 17723, + "dishcloth": 17724, + "cobb": 17725, + "aggressive": 17726, + "ui": 17727, + "skywalk": 17728, + "superb": 17729, + "linked": 17730, + "repairs": 17731, + "##monton": 17732, + "cider": 17733, + "baref": 17734, + "##shot": 17735, + "buttercream": 17736, + "solve": 17737, + "subsc": 17738, + "##etta": 17739, + "ecosyst": 17740, + "ladybird": 17741, + "##liner": 17742, + "combiner": 17743, + "persona": 17744, + "buick": 17745, + "sandal": 17746, + "accurate": 17747, + "##no": 17748, + "##cons": 17749, + "undated": 17750, + "dipping": 17751, + "##enne": 17752, + "treasures": 17753, + "barefoot": 17754, + "v8": 17755, + "brighten": 17756, + "literacy": 17757, + "##sions": 17758, + "congo": 17759, + "quran": 17760, + "duo": 17761, + "chevron": 17762, + "kuwa": 17763, + "##enta": 17764, + "helpful": 17765, + "bliss": 17766, + "aztec": 17767, + "edwardian": 17768, + "tests": 17769, + "lookout": 17770, + "extract": 17771, + "cic": 17772, + "##85": 17773, + "builds": 17774, + "melon": 17775, + "cooke": 17776, + "belf": 17777, + "crashed": 17778, + "##beat": 17779, + "cages": 17780, + "##alon": 17781, + "tahoe": 17782, + "##icted": 17783, + "serge": 17784, + "certified": 17785, + "asym": 17786, + "cologne": 17787, + "appoint": 17788, + "balancing": 17789, + "mcc": 17790, + "mystical": 17791, + "kiwi": 17792, + "##aine": 17793, + "alf": 17794, + "spreading": 17795, + "updates": 17796, + "essex": 17797, + "caring": 17798, + "##atter": 17799, + "##ellers": 17800, + "crayon": 17801, + "##itius": 17802, + "uneven": 17803, + "dak": 17804, + "poppies": 17805, + "freight": 17806, + "osp": 17807, + "logistics": 17808, + "autos": 17809, + "edmonton": 17810, + "tottenham": 17811, + "posed": 17812, + "align": 17813, + "prover": 17814, + "resur": 17815, + "bucks": 17816, + "claim": 17817, + "axol": 17818, + "dying": 17819, + "iranian": 17820, + "fj": 17821, + "formerly": 17822, + "chefs": 17823, + "##identified": 17824, + "papar": 17825, + "firing": 17826, + "masculine": 17827, + "mummy": 17828, + "##allows": 17829, + "arlington": 17830, + "titans": 17831, + "railings": 17832, + "reports": 17833, + "texas": 17834, + "##rian": 17835, + "##je": 17836, + "##etics": 17837, + "pressed": 17838, + "cv": 17839, + "manip": 17840, + "##yards": 17841, + "hydrangea": 17842, + "carton": 17843, + "vars": 17844, + "hosting": 17845, + "conclusion": 17846, + "spartan": 17847, + "##hs": 17848, + "hygi": 17849, + "mauritius": 17850, + "hygiene": 17851, + "conceal": 17852, + "##church": 17853, + "sex": 17854, + "conditioning": 17855, + "framing": 17856, + "forge": 17857, + "buddy": 17858, + "##greg": 17859, + "bustard": 17860, + "geckol": 17861, + "swatch": 17862, + "eventually": 17863, + "hammers": 17864, + "##azzi": 17865, + "duc": 17866, + "wharf": 17867, + "valu": 17868, + "brooms": 17869, + "eid": 17870, + "argument": 17871, + "728": 17872, + "868": 17873, + "auss": 17874, + "##park": 17875, + "tofu": 17876, + "##ipes": 17877, + "boasts": 17878, + "gross": 17879, + "electrons": 17880, + "organize": 17881, + "experienced": 17882, + "61": 17883, + "scare": 17884, + "recall": 17885, + "footed": 17886, + "scotch": 17887, + "##rent": 17888, + "invite": 17889, + "fare": 17890, + "phill": 17891, + "chass": 17892, + "crowned": 17893, + "constit": 17894, + "assassin": 17895, + "varsity": 17896, + "##tery": 17897, + "##xton": 17898, + "crus": 17899, + "elim": 17900, + "minimalism": 17901, + "bookshelf": 17902, + "lever": 17903, + "swallow": 17904, + "origins": 17905, + "epa": 17906, + "##vania": 17907, + "caricature": 17908, + "pew": 17909, + "peeled": 17910, + "pleasure": 17911, + "nobody": 17912, + "slit": 17913, + "addiction": 17914, + "catches": 17915, + "nan": 17916, + "rails": 17917, + "trials": 17918, + "villains": 17919, + "backpacking": 17920, + "clin": 17921, + "remotes": 17922, + "austrian": 17923, + "reaper": 17924, + "canoes": 17925, + "maca": 17926, + "lanes": 17927, + "walled": 17928, + "16v": 17929, + "respir": 17930, + "rapids": 17931, + "stamped": 17932, + "monter": 17933, + "emmy": 17934, + "fragment": 17935, + "kensington": 17936, + "##via": 17937, + "windsh": 17938, + "remem": 17939, + "talented": 17940, + "nutcracker": 17941, + "fighters": 17942, + "reptile": 17943, + "##unction": 17944, + "boulder": 17945, + "jockey": 17946, + "##worm": 17947, + "##ithe": 17948, + "encyclopedia": 17949, + "tequ": 17950, + "jungles": 17951, + "##bors": 17952, + "dumbbell": 17953, + "distancing": 17954, + "##oko": 17955, + "beats": 17956, + "banff": 17957, + "##cotta": 17958, + "ara": 17959, + "deserted": 17960, + "angkor": 17961, + "principles": 17962, + "dorset": 17963, + "versus": 17964, + "literary": 17965, + "finn": 17966, + "hydraulic": 17967, + "sega": 17968, + "metals": 17969, + "performances": 17970, + "armoured": 17971, + "entrepreneur": 17972, + "paparazzi": 17973, + "##ooper": 17974, + "pam": 17975, + "warrant": 17976, + "towing": 17977, + "##fs": 17978, + "reeds": 17979, + "demo": 17980, + "worthy": 17981, + "##uta": 17982, + "fright": 17983, + "chime": 17984, + "roadside": 17985, + "raincoat": 17986, + "phi": 17987, + "pheas": 17988, + "cutl": 17989, + "documentation": 17990, + "sandp": 17991, + "particularly": 17992, + "wuhan": 17993, + "galapagos": 17994, + "ain": 17995, + "properly": 17996, + "chewy": 17997, + "netting": 17998, + "wedges": 17999, + "rit": 18000, + "rog": 18001, + "##uting": 18002, + "##ashes": 18003, + "surviv": 18004, + "hampshire": 18005, + "anxiety": 18006, + "hazel": 18007, + "absor": 18008, + "ae": 18009, + "focal": 18010, + "supporters": 18011, + "pcs": 18012, + "bookmark": 18013, + "##otto": 18014, + "chedd": 18015, + "submer": 18016, + "amant": 18017, + "dinners": 18018, + "paramount": 18019, + "pontiac": 18020, + "##izers": 18021, + "shutter": 18022, + "viral": 18023, + "taught": 18024, + "steamed": 18025, + "ji": 18026, + "uncle": 18027, + "otherwise": 18028, + "revel": 18029, + "peking": 18030, + "hippo": 18031, + "vinegar": 18032, + "vessels": 18033, + "mardi": 18034, + "freeze": 18035, + "salty": 18036, + "inspection": 18037, + "##haus": 18038, + "seniors": 18039, + "coordin": 18040, + "##zas": 18041, + "township": 18042, + "honour": 18043, + "orders": 18044, + "wic": 18045, + "cardiff": 18046, + "dealer": 18047, + "tucked": 18048, + "unidentified": 18049, + "alliance": 18050, + "wrote": 18051, + "cocker": 18052, + "hungarian": 18053, + "tequila": 18054, + "campers": 18055, + "dill": 18056, + "desire": 18057, + "##oral": 18058, + "streaming": 18059, + "insight": 18060, + "crunchy": 18061, + "##raction": 18062, + "barbados": 18063, + "volcanic": 18064, + "asymmet": 18065, + "##anga": 18066, + "previously": 18067, + "beehive": 18068, + "boating": 18069, + "homeown": 18070, + "##issions": 18071, + "specim": 18072, + "laces": 18073, + "crou": 18074, + "##maker": 18075, + "syring": 18076, + "hyder": 18077, + "##iar": 18078, + "reunion": 18079, + "spear": 18080, + "slugs": 18081, + "trekking": 18082, + "##sis": 18083, + "youll": 18084, + "ave": 18085, + "##orie": 18086, + "##ipt": 18087, + "redwood": 18088, + "beekee": 18089, + "mistakes": 18090, + "santor": 18091, + "maui": 18092, + "billed": 18093, + "doubt": 18094, + "##31": 18095, + "terracotta": 18096, + "##lit": 18097, + "spiced": 18098, + "crews": 18099, + "guine": 18100, + "emerging": 18101, + "andre": 18102, + "##eton": 18103, + "160": 18104, + "profit": 18105, + "##uku": 18106, + "puma": 18107, + "mcdonald": 18108, + "none": 18109, + "##bong": 18110, + "periodic": 18111, + "windshield": 18112, + "doo": 18113, + "paving": 18114, + "blueto": 18115, + "curves": 18116, + "embracing": 18117, + "hyderabad": 18118, + "84": 18119, + "distant": 18120, + "saltwater": 18121, + "veterinary": 18122, + "deviant": 18123, + "cruz": 18124, + "##rison": 18125, + "flora": 18126, + "impala": 18127, + "clif": 18128, + "deput": 18129, + "fauna": 18130, + "titanium": 18131, + "yose": 18132, + "crossover": 18133, + "geisha": 18134, + "warsaw": 18135, + "mitt": 18136, + "mupp": 18137, + "rale": 18138, + "santorini": 18139, + "hs": 18140, + "convent": 18141, + "postage": 18142, + "shepher": 18143, + "decker": 18144, + "convey": 18145, + "riot": 18146, + "yo": 18147, + "tins": 18148, + "##imar": 18149, + "twent": 18150, + "spoonbill": 18151, + "##aders": 18152, + "mankind": 18153, + "reminder": 18154, + "mommy": 18155, + "kuwait": 18156, + "sessions": 18157, + "partition": 18158, + "skiers": 18159, + "raleigh": 18160, + "##outing": 18161, + "ancest": 18162, + "slovenia": 18163, + "versailles": 18164, + "ordered": 18165, + "cleans": 18166, + "1961": 18167, + "hospitality": 18168, + "entirely": 18169, + "surgeon": 18170, + "dagger": 18171, + "seasoning": 18172, + "habits": 18173, + "roseated": 18174, + "skywalker": 18175, + "lah": 18176, + "##rs": 18177, + "thrill": 18178, + "lollip": 18179, + "hurt": 18180, + "handic": 18181, + "donated": 18182, + "karting": 18183, + "sights": 18184, + "howling": 18185, + "regis": 18186, + "##guard": 18187, + "weightlifting": 18188, + "guaranteed": 18189, + "mandalorian": 18190, + "sod": 18191, + "relay": 18192, + "buyer": 18193, + "illness": 18194, + "marshmallow": 18195, + "##iable": 18196, + "depend": 18197, + "cheddar": 18198, + "##icity": 18199, + "kaying": 18200, + "troph": 18201, + "82": 18202, + "altered": 18203, + "griff": 18204, + "eared": 18205, + "1024": 18206, + "rotor": 18207, + "quarant": 18208, + "compr": 18209, + "posit": 18210, + "variant": 18211, + "sao": 18212, + "ruin": 18213, + "meteor": 18214, + "raster": 18215, + "illustrate": 18216, + "soaring": 18217, + "mouths": 18218, + "nectar": 18219, + "waterl": 18220, + "titles": 18221, + "courthouse": 18222, + "sailors": 18223, + "agreement": 18224, + "daffod": 18225, + "jacks": 18226, + "attachment": 18227, + "llc": 18228, + "emboss": 18229, + "frontier": 18230, + "masterpiece": 18231, + "sings": 18232, + "uc": 18233, + "##arus": 18234, + "christchurch": 18235, + "##eapolis": 18236, + "##plan": 18237, + "contributed": 18238, + "rafting": 18239, + "manner": 18240, + "##lust": 18241, + "shotgun": 18242, + "187": 18243, + "investig": 18244, + "##rint": 18245, + "##usable": 18246, + "acts": 18247, + "echo": 18248, + "ease": 18249, + "mention": 18250, + "rocker": 18251, + "warcraft": 18252, + "minneapolis": 18253, + "chassis": 18254, + "##ilee": 18255, + "hail": 18256, + "##cester": 18257, + "captures": 18258, + "aj": 18259, + "##rence": 18260, + "##cula": 18261, + "mayo": 18262, + "sicily": 18263, + "##eros": 18264, + "ia": 18265, + "comed": 18266, + "armen": 18267, + "motorsport": 18268, + "adorned": 18269, + "adam": 18270, + "frosty": 18271, + "recycle": 18272, + "identification": 18273, + "##cour": 18274, + "forced": 18275, + "flops": 18276, + "##zzi": 18277, + "fathers": 18278, + "rogue": 18279, + "uv": 18280, + "reflects": 18281, + "iguan": 18282, + "measurement": 18283, + "##88": 18284, + "illusions": 18285, + "##ubar": 18286, + "surfaces": 18287, + "singers": 18288, + "##oly": 18289, + "confl": 18290, + "warmth": 18291, + "ralph": 18292, + "submerged": 18293, + "##places": 18294, + "asce": 18295, + "pears": 18296, + "programme": 18297, + "episodes": 18298, + "lakeside": 18299, + "pue": 18300, + "decad": 18301, + "returning": 18302, + "showcases": 18303, + "##eda": 18304, + "structural": 18305, + "driftwood": 18306, + "##chs": 18307, + "guinness": 18308, + "licensed": 18309, + "##uct": 18310, + "raider": 18311, + "spikes": 18312, + "inhab": 18313, + "reaches": 18314, + "globes": 18315, + "perm": 18316, + "streetwear": 18317, + "slider": 18318, + "laminate": 18319, + "gestures": 18320, + "##athlon": 18321, + "publicity": 18322, + "##laid": 18323, + "botsw": 18324, + "teacup": 18325, + "flavored": 18326, + "margarita": 18327, + "toxic": 18328, + "napkins": 18329, + "chaise": 18330, + "enforcement": 18331, + "sweeping": 18332, + "sacramento": 18333, + "quarantined": 18334, + "##bows": 18335, + "##96": 18336, + "trace": 18337, + "glad": 18338, + "##rosse": 18339, + "certainly": 18340, + "750": 18341, + "##nik": 18342, + "leftover": 18343, + "manta": 18344, + "grains": 18345, + "greenwich": 18346, + "weed": 18347, + "eyewear": 18348, + "recognition": 18349, + "##lie": 18350, + "hyp": 18351, + "aero": 18352, + "##iley": 18353, + "grammar": 18354, + "temperatures": 18355, + "dripping": 18356, + "mobility": 18357, + "lacrosse": 18358, + "##cape": 18359, + "shamp": 18360, + "shaving": 18361, + "flint": 18362, + "alligators": 18363, + "indonesian": 18364, + "wo": 18365, + "floats": 18366, + "hairdress": 18367, + "sip": 18368, + "kindness": 18369, + "##mite": 18370, + "meats": 18371, + "obvious": 18372, + "indians": 18373, + "koal": 18374, + "joyful": 18375, + "tiara": 18376, + "botswana": 18377, + "screaming": 18378, + "burlap": 18379, + "peaches": 18380, + "rais": 18381, + "supported": 18382, + "sult": 18383, + "pictogram": 18384, + "timeline": 18385, + "mills": 18386, + "kidney": 18387, + "alexandria": 18388, + "density": 18389, + "lod": 18390, + "amal": 18391, + "lineup": 18392, + "coaches": 18393, + "calorie": 18394, + "simplicity": 18395, + "sala": 18396, + "confeder": 18397, + "renowned": 18398, + "forgotten": 18399, + "tumbler": 18400, + "conflict": 18401, + "conse": 18402, + "signatures": 18403, + "elvis": 18404, + "hyde": 18405, + "immediately": 18406, + "challenging": 18407, + "exercising": 18408, + "fishermen": 18409, + "tyres": 18410, + "devot": 18411, + "toilets": 18412, + "habitats": 18413, + "obel": 18414, + "hyper": 18415, + "superheroes": 18416, + "pembroke": 18417, + "sheepdog": 18418, + "spines": 18419, + "##mington": 18420, + "provider": 18421, + "actions": 18422, + "##grove": 18423, + "canter": 18424, + "embassy": 18425, + "sensor": 18426, + "multicolored": 18427, + "gou": 18428, + "stains": 18429, + "switches": 18430, + "brat": 18431, + "bure": 18432, + "jakarta": 18433, + "aging": 18434, + "##rim": 18435, + "##istine": 18436, + "dollhouse": 18437, + "config": 18438, + "blanc": 18439, + "capu": 18440, + "lifeboat": 18441, + "empower": 18442, + "parisian": 18443, + "tassel": 18444, + "ras": 18445, + "ric": 18446, + "leafy": 18447, + "fuji": 18448, + "wallaby": 18449, + "catwalk": 18450, + "draped": 18451, + "sportswear": 18452, + "improved": 18453, + "obser": 18454, + "mammoth": 18455, + "lup": 18456, + "##ighth": 18457, + "honest": 18458, + "motocross": 18459, + "corres": 18460, + "closely": 18461, + "inviting": 18462, + "discussing": 18463, + "##zo": 18464, + "tuning": 18465, + "paddling": 18466, + "forth": 18467, + "minions": 18468, + "literally": 18469, + "observation": 18470, + "##enced": 18471, + "generally": 18472, + "cilant": 18473, + "tutu": 18474, + "stretcher": 18475, + "screwdr": 18476, + "##nia": 18477, + "##rose": 18478, + "notepad": 18479, + "basset": 18480, + "kate": 18481, + "halls": 18482, + "initials": 18483, + "bracket": 18484, + "osprey": 18485, + "aussie": 18486, + "##onies": 18487, + "reusable": 18488, + "wallets": 18489, + "criminal": 18490, + "sans": 18491, + "##tie": 18492, + "horoscope": 18493, + "paved": 18494, + "suburbs": 18495, + "tum": 18496, + "woodlands": 18497, + "bully": 18498, + "limits": 18499, + "amenities": 18500, + "pik": 18501, + "##ographics": 18502, + "scrambler": 18503, + "##overs": 18504, + "nok": 18505, + "lawyer": 18506, + "colouring": 18507, + "stalls": 18508, + "component": 18509, + "mysteries": 18510, + "moons": 18511, + "loan": 18512, + "usual": 18513, + "laos": 18514, + "irr": 18515, + "wanderlust": 18516, + "gnome": 18517, + "##manship": 18518, + "produces": 18519, + "buffalos": 18520, + "rosary": 18521, + "##rops": 18522, + "batteries": 18523, + "deli": 18524, + "weeknight": 18525, + "##caster": 18526, + "##itas": 18527, + "bending": 18528, + "placement": 18529, + "scarves": 18530, + "continents": 18531, + "semin": 18532, + "##kh": 18533, + "##encia": 18534, + "montess": 18535, + "mallard": 18536, + "##wn": 18537, + "##star": 18538, + "export": 18539, + "decre": 18540, + "##bourg": 18541, + "disaster": 18542, + "writers": 18543, + "cilantro": 18544, + "antelope": 18545, + "shocked": 18546, + "montessori": 18547, + "karn": 18548, + "promin": 18549, + "helicopters": 18550, + "snapping": 18551, + "jasper": 18552, + "connections": 18553, + "bil": 18554, + "##furt": 18555, + "sleepy": 18556, + "sorts": 18557, + "opinion": 18558, + "##uity": 18559, + "renault": 18560, + "shines": 18561, + "mattel": 18562, + "nigerian": 18563, + "culinary": 18564, + "##rows": 18565, + "##erst": 18566, + "##icular": 18567, + "colorado": 18568, + "leaping": 18569, + "princes": 18570, + "infection": 18571, + "tuba": 18572, + "amphibians": 18573, + "##tes": 18574, + "ov": 18575, + "doha": 18576, + "governor": 18577, + "siblings": 18578, + "##100": 18579, + "ukule": 18580, + "discip": 18581, + "dumplings": 18582, + "praise": 18583, + "divor": 18584, + "thesis": 18585, + "alo": 18586, + "sacr": 18587, + "anthem": 18588, + "zimb": 18589, + "mainly": 18590, + "florist": 18591, + "remarkable": 18592, + "zimbab": 18593, + "yuk": 18594, + "##ugh": 18595, + "chand": 18596, + "emph": 18597, + "89": 18598, + "quarters": 18599, + "protecting": 18600, + "ferris": 18601, + "countertops": 18602, + "comprehensive": 18603, + "trin": 18604, + "indies": 18605, + "traveller": 18606, + "stencils": 18607, + "centuries": 18608, + "managed": 18609, + "prevention": 18610, + "vegg": 18611, + "##action": 18612, + "##bish": 18613, + "frustr": 18614, + "ostrich": 18615, + "excellence": 18616, + "demolition": 18617, + "##42": 18618, + "imitation": 18619, + "cooks": 18620, + "##stered": 18621, + "panic": 18622, + "identified": 18623, + "cropped": 18624, + "locus": 18625, + "heavenly": 18626, + "franchise": 18627, + "ive": 18628, + "climber": 18629, + "dolce": 18630, + "dio": 18631, + "wrench": 18632, + "eggplant": 18633, + "marvelous": 18634, + "baptism": 18635, + "guatemala": 18636, + "capri": 18637, + "##bred": 18638, + "wishing": 18639, + "descend": 18640, + "shampoo": 18641, + "deviantart": 18642, + "seahawks": 18643, + "##bust": 18644, + "lists": 18645, + "buyers": 18646, + "conqu": 18647, + "isolate": 18648, + "recreational": 18649, + "sudan": 18650, + "struggle": 18651, + "rates": 18652, + "specs": 18653, + "1953": 18654, + "participating": 18655, + "totema": 18656, + "caes": 18657, + "pelicans": 18658, + "unless": 18659, + "customs": 18660, + "eyeliner": 18661, + "zimbabwe": 18662, + "pace": 18663, + "yar": 18664, + "##ruction": 18665, + "anonymous": 18666, + "obsc": 18667, + "destruction": 18668, + "hiker": 18669, + "eruption": 18670, + "syndrome": 18671, + "scored": 18672, + "apric": 18673, + "hopes": 18674, + "nupt": 18675, + "##eim": 18676, + "##emen": 18677, + "##ycat": 18678, + "porches": 18679, + "isles": 18680, + "harvested": 18681, + "scriptures": 18682, + "clone": 18683, + "##ructure": 18684, + "disappear": 18685, + "coil": 18686, + "cayman": 18687, + "sponsored": 18688, + "embossed": 18689, + "rochester": 18690, + "binary": 18691, + "sausages": 18692, + "anthrop": 18693, + "parish": 18694, + "courts": 18695, + "repairing": 18696, + "nw": 18697, + "##hall": 18698, + "##held": 18699, + "zepp": 18700, + "efforts": 18701, + "windy": 18702, + "flannel": 18703, + "##ridges": 18704, + "hearty": 18705, + "ceremonies": 18706, + "sv": 18707, + "alike": 18708, + "wondering": 18709, + "mythological": 18710, + "goddesses": 18711, + "tortilla": 18712, + "sprouts": 18713, + "yosemite": 18714, + "plumage": 18715, + "auton": 18716, + "goodies": 18717, + "skincare": 18718, + "experiments": 18719, + "unf": 18720, + "smokey": 18721, + "irre": 18722, + "programming": 18723, + "fragr": 18724, + "kashmir": 18725, + "output": 18726, + "define": 18727, + "pilates": 18728, + "spirituality": 18729, + "wilson": 18730, + "handful": 18731, + "airy": 18732, + "estimated": 18733, + "metaphor": 18734, + "aunt": 18735, + "##atan": 18736, + "##orph": 18737, + "manass": 18738, + "prun": 18739, + "employment": 18740, + "##aras": 18741, + "diss": 18742, + "pilots": 18743, + "chilli": 18744, + "peuge": 18745, + "hearth": 18746, + "viadu": 18747, + "examining": 18748, + "shovels": 18749, + "sparrow": 18750, + "linger": 18751, + "140": 18752, + "morgan": 18753, + "bluetooth": 18754, + "##aku": 18755, + "centipe": 18756, + "rigs": 18757, + "extend": 18758, + "laguna": 18759, + "overalls": 18760, + "balconies": 18761, + "scheduled": 18762, + "peugeot": 18763, + "platt": 18764, + "businessmen": 18765, + "reform": 18766, + "padded": 18767, + "mindful": 18768, + "escap": 18769, + "tacoma": 18770, + "compatible": 18771, + "logic": 18772, + "berm": 18773, + "cracker": 18774, + "burns": 18775, + "##idad": 18776, + "cyclists": 18777, + "faculty": 18778, + "serenity": 18779, + "spool": 18780, + "camden": 18781, + "##vian": 18782, + "conch": 18783, + "graduating": 18784, + "experimental": 18785, + "dap": 18786, + "stills": 18787, + "clowns": 18788, + "mapping": 18789, + "alpaca": 18790, + "apes": 18791, + "sandstone": 18792, + "##enon": 18793, + "aside": 18794, + "redd": 18795, + "sampl": 18796, + "wetlands": 18797, + "hawks": 18798, + "payment": 18799, + "92": 18800, + "##zes": 18801, + "##43": 18802, + "producing": 18803, + "morris": 18804, + "handheld": 18805, + "continent": 18806, + "diseases": 18807, + "nottingham": 18808, + "oman": 18809, + "severe": 18810, + "rico": 18811, + "graphite": 18812, + "aspects": 18813, + "anolea": 18814, + "pavers": 18815, + "raspberries": 18816, + "billiards": 18817, + "##iatric": 18818, + "gild": 18819, + "vera": 18820, + "beacon": 18821, + "marl": 18822, + "bobcat": 18823, + "aspect": 18824, + "##67": 18825, + "dee": 18826, + "##illo": 18827, + "cascade": 18828, + "syringe": 18829, + "fireman": 18830, + "shoppers": 18831, + "taxis": 18832, + "orph": 18833, + "vaccine": 18834, + "conscious": 18835, + "mermaids": 18836, + "outback": 18837, + "americas": 18838, + "jumbo": 18839, + "preppy": 18840, + "addresses": 18841, + "transitional": 18842, + "##gent": 18843, + "##ilst": 18844, + "injuries": 18845, + "##gb": 18846, + "salads": 18847, + "loafers": 18848, + "baltic": 18849, + "##jack": 18850, + "glou": 18851, + "controllers": 18852, + "taper": 18853, + "distingu": 18854, + "basis": 18855, + "bandana": 18856, + "##rington": 18857, + "tunes": 18858, + "commissioned": 18859, + "merid": 18860, + "whilst": 18861, + "handshake": 18862, + "complic": 18863, + "widely": 18864, + "revolutionary": 18865, + "variable": 18866, + "himalayan": 18867, + "##power": 18868, + "overwatch": 18869, + "rt": 18870, + "unfor": 18871, + "patr": 18872, + "zeppelin": 18873, + "backstage": 18874, + "greenland": 18875, + "teh": 18876, + "grace": 18877, + "skateboarding": 18878, + "##vements": 18879, + "##orig": 18880, + "serie": 18881, + "adventur": 18882, + "lcd": 18883, + "##verse": 18884, + "claims": 18885, + "godfather": 18886, + "noir": 18887, + "harris": 18888, + "##etted": 18889, + "memorable": 18890, + "approaches": 18891, + "oslo": 18892, + "checklist": 18893, + "##wind": 18894, + "cinematic": 18895, + "ufo": 18896, + "shre": 18897, + "cartier": 18898, + "apex": 18899, + "slime": 18900, + "fireplaces": 18901, + "slop": 18902, + "essence": 18903, + "geyser": 18904, + "mortal": 18905, + "recognize": 18906, + "ounce": 18907, + "nokia": 18908, + "hort": 18909, + "##esburg": 18910, + "gladiator": 18911, + "##nders": 18912, + "##pless": 18913, + "supercar": 18914, + "1954": 18915, + "fights": 18916, + "johann": 18917, + "##uri": 18918, + "##unzel": 18919, + "joining": 18920, + "##riptions": 18921, + "smartphones": 18922, + "rapunzel": 18923, + "migr": 18924, + "##eno": 18925, + "stile": 18926, + "incident": 18927, + "routes": 18928, + "punisher": 18929, + "##opic": 18930, + "aborig": 18931, + "##enny": 18932, + "bases": 18933, + "##undra": 18934, + "magnets": 18935, + "##cans": 18936, + "scope": 18937, + "polaroid": 18938, + "dil": 18939, + "unlock": 18940, + "mercy": 18941, + "terrible": 18942, + "chameleon": 18943, + "dover": 18944, + "sixt": 18945, + "mannequin": 18946, + "impressionist": 18947, + "silky": 18948, + "isolation": 18949, + "minimum": 18950, + "##ante": 18951, + "ritual": 18952, + "canterbury": 18953, + "##bling": 18954, + "##ilers": 18955, + "infographics": 18956, + "badmint": 18957, + "sauna": 18958, + "badminton": 18959, + "valuable": 18960, + "lla": 18961, + "##ulating": 18962, + "infrast": 18963, + "lig": 18964, + "vocal": 18965, + "worried": 18966, + "guang": 18967, + "nesting": 18968, + "##ulu": 18969, + "##fe": 18970, + "##icians": 18971, + "sending": 18972, + "swimmers": 18973, + "pueb": 18974, + "fluor": 18975, + "punta": 18976, + "banknotes": 18977, + "##imeter": 18978, + "primates": 18979, + "cylind": 18980, + "arkham": 18981, + "gq": 18982, + "staffords": 18983, + "##itious": 18984, + "symbolize": 18985, + "berk": 18986, + "toothp": 18987, + "pantone": 18988, + "##inc": 18989, + "##alia": 18990, + "stun": 18991, + "##irts": 18992, + "springtime": 18993, + "genus": 18994, + "causing": 18995, + "navigation": 18996, + "accompanied": 18997, + "staffordshire": 18998, + "supposed": 18999, + "curios": 19000, + "treehouse": 19001, + "diecast": 19002, + "announce": 19003, + "frankfurt": 19004, + "thru": 19005, + "basically": 19006, + "osm": 19007, + "##rous": 19008, + "seeking": 19009, + "seascape": 19010, + "hazard": 19011, + "wildflower": 19012, + "freehand": 19013, + "airmen": 19014, + "og": 19015, + "tt": 19016, + "cutout": 19017, + "peacocks": 19018, + "cutlery": 19019, + "decay": 19020, + "reconstruction": 19021, + "crucif": 19022, + "sergeant": 19023, + "desired": 19024, + "dism": 19025, + "singles": 19026, + "tailored": 19027, + "husk": 19028, + "hubble": 19029, + "##ipeg": 19030, + "insulation": 19031, + "adapter": 19032, + "rouge": 19033, + "##agna": 19034, + "warmer": 19035, + "hamsters": 19036, + "winnipeg": 19037, + "failure": 19038, + "aiming": 19039, + "graham": 19040, + "metres": 19041, + "assistance": 19042, + "##orney": 19043, + "scholar": 19044, + "ere": 19045, + "tundra": 19046, + "glo": 19047, + "wald": 19048, + "reached": 19049, + "##ictions": 19050, + "curric": 19051, + "clipboard": 19052, + "kilomet": 19053, + "paintbrush": 19054, + "teenagers": 19055, + "sloths": 19056, + "covent": 19057, + "recognized": 19058, + "mating": 19059, + "arth": 19060, + "ganesh": 19061, + "silhouetted": 19062, + "lembong": 19063, + "##azzo": 19064, + "mosques": 19065, + "lembongan": 19066, + "##aline": 19067, + "tablets": 19068, + "precision": 19069, + "scotia": 19070, + "alexander": 19071, + "variations": 19072, + "rye": 19073, + "clifton": 19074, + "reels": 19075, + "##icide": 19076, + "liquids": 19077, + "bloody": 19078, + "federation": 19079, + "##x8": 19080, + "satisf": 19081, + "torso": 19082, + "cw": 19083, + "##mc": 19084, + "efficiency": 19085, + "cbd": 19086, + "trophies": 19087, + "##rog": 19088, + "teaches": 19089, + "yearbook": 19090, + "necked": 19091, + "everybody": 19092, + "rattles": 19093, + "decadent": 19094, + "kia": 19095, + "impla": 19096, + "mundo": 19097, + "tony": 19098, + "diplom": 19099, + "prohibition": 19100, + "voltage": 19101, + "##path": 19102, + "dracula": 19103, + "mule": 19104, + "memor": 19105, + "bangles": 19106, + "manit": 19107, + "relative": 19108, + "distinct": 19109, + "polyester": 19110, + "thorough": 19111, + "datab": 19112, + "##ugu": 19113, + "duplex": 19114, + "sins": 19115, + "wifi": 19116, + "##ctric": 19117, + "browns": 19118, + "funeral": 19119, + "straws": 19120, + "assum": 19121, + "raptor": 19122, + "##izzard": 19123, + "intelligent": 19124, + "shepherds": 19125, + "fury": 19126, + "##lest": 19127, + "prete": 19128, + "condens": 19129, + "chopsticks": 19130, + "##archy": 19131, + "asy": 19132, + "lioness": 19133, + "macbook": 19134, + "follows": 19135, + "affili": 19136, + "kraft": 19137, + "mayfair": 19138, + "elle": 19139, + "werew": 19140, + "lasting": 19141, + "diagonal": 19142, + "jpg": 19143, + "endgame": 19144, + "##berra": 19145, + "sideboard": 19146, + "antlers": 19147, + "##gage": 19148, + "halves": 19149, + "cigars": 19150, + "##ovak": 19151, + "illegal": 19152, + "canberra": 19153, + "jacksonville": 19154, + "dell": 19155, + "patios": 19156, + "router": 19157, + "##gett": 19158, + "##inf": 19159, + "ine": 19160, + "##ti": 19161, + "input": 19162, + "1929": 19163, + "corks": 19164, + "procedure": 19165, + "bureau": 19166, + "dummy": 19167, + "hier": 19168, + "grandpar": 19169, + "amalfi": 19170, + "mcl": 19171, + "roasting": 19172, + "regist": 19173, + "hipp": 19174, + "panties": 19175, + "depending": 19176, + "nair": 19177, + "dropping": 19178, + "stark": 19179, + "firewood": 19180, + "warwick": 19181, + "##hurst": 19182, + "alfa": 19183, + "##berland": 19184, + "essays": 19185, + "sut": 19186, + "spit": 19187, + "undercut": 19188, + "mashede": 19189, + "gilded": 19190, + "faded": 19191, + "##hem": 19192, + "bhutan": 19193, + "attendees": 19194, + "threads": 19195, + "artillery": 19196, + "enters": 19197, + "issued": 19198, + "sherlock": 19199, + "considering": 19200, + "deputy": 19201, + "##achu": 19202, + "simulator": 19203, + "pickle": 19204, + "startup": 19205, + "juven": 19206, + "bungalows": 19207, + "schnau": 19208, + "mongol": 19209, + "archaeology": 19210, + "##opa": 19211, + "lofts": 19212, + "intercept": 19213, + "angled": 19214, + "hamburg": 19215, + "impos": 19216, + "publication": 19217, + "quit": 19218, + "petal": 19219, + "surrealism": 19220, + "infrastructure": 19221, + "boxed": 19222, + "##elson": 19223, + "careful": 19224, + "mongolia": 19225, + "##roe": 19226, + "bluff": 19227, + "##gor": 19228, + "seahorse": 19229, + "##inson": 19230, + "mayan": 19231, + "##odo": 19232, + "trigger": 19233, + "staple": 19234, + "##ques": 19235, + "cheers": 19236, + "40th": 19237, + "gondola": 19238, + "jogging": 19239, + "2021": 19240, + "joins": 19241, + "dimension": 19242, + "orac": 19243, + "1910": 19244, + "1080": 19245, + "##aneous": 19246, + "durham": 19247, + "##zon": 19248, + "masonic": 19249, + "smithson": 19250, + "crucifix": 19251, + "snook": 19252, + "struck": 19253, + "terror": 19254, + "cocks": 19255, + "hms": 19256, + "sauces": 19257, + "outsk": 19258, + "troy": 19259, + "introduce": 19260, + "##mation": 19261, + "fairmont": 19262, + "demolished": 19263, + "accel": 19264, + "broncos": 19265, + "souvenirs": 19266, + "coating": 19267, + "mods": 19268, + "sett": 19269, + "therefore": 19270, + "avi": 19271, + "golfer": 19272, + "kayaks": 19273, + "garnish": 19274, + "grateful": 19275, + "reduced": 19276, + "chor": 19277, + "##asso": 19278, + "yukon": 19279, + "olds": 19280, + "gordon": 19281, + "##irond": 19282, + "streak": 19283, + "cassettes": 19284, + "newlyw": 19285, + "##opot": 19286, + "modify": 19287, + "plac": 19288, + "surfers": 19289, + "palazzo": 19290, + "shelby": 19291, + "flavorful": 19292, + "ukrainian": 19293, + "pikachu": 19294, + "viaduct": 19295, + "gur": 19296, + "pals": 19297, + "shirtless": 19298, + "stylist": 19299, + "knits": 19300, + "beaker": 19301, + "molding": 19302, + "nuptse": 19303, + "##esters": 19304, + "missions": 19305, + "buried": 19306, + "##whel": 19307, + "gonna": 19308, + "##olla": 19309, + "etched": 19310, + "conductor": 19311, + "##angs": 19312, + "##impse": 19313, + "chandeliers": 19314, + "deposit": 19315, + "astronauts": 19316, + "publishing": 19317, + "rebel": 19318, + "impl": 19319, + "glimpse": 19320, + "agra": 19321, + "##zuela": 19322, + "carmel": 19323, + "yan": 19324, + "deserts": 19325, + "adirond": 19326, + "myr": 19327, + "counters": 19328, + "##lining": 19329, + "advoc": 19330, + "gull": 19331, + "thir": 19332, + "##inx": 19333, + "##rich": 19334, + "edited": 19335, + "beech": 19336, + "loads": 19337, + "tunis": 19338, + "sakura": 19339, + "skateboards": 19340, + "##olin": 19341, + "canals": 19342, + "limes": 19343, + "hunters": 19344, + "galaxies": 19345, + "amphithe": 19346, + "cana": 19347, + "downs": 19348, + "happily": 19349, + "vincent": 19350, + "belfast": 19351, + "royals": 19352, + "multicolor": 19353, + "moust": 19354, + "cruel": 19355, + "romeo": 19356, + "yu": 19357, + "quay": 19358, + "barely": 19359, + "strain": 19360, + "cockat": 19361, + "almonds": 19362, + "komb": 19363, + "stee": 19364, + "chuck": 19365, + "developer": 19366, + "optional": 19367, + "commerce": 19368, + "mousse": 19369, + "graveyard": 19370, + "hr": 19371, + "##olas": 19372, + "barley": 19373, + "pallets": 19374, + "prefab": 19375, + "stays": 19376, + "##oga": 19377, + "racers": 19378, + "windowsill": 19379, + "daylight": 19380, + "midd": 19381, + "remodeled": 19382, + "##drops": 19383, + "flown": 19384, + "smithsonian": 19385, + "##ibilities": 19386, + "winchester": 19387, + "labs": 19388, + "villain": 19389, + "bandages": 19390, + "guarantee": 19391, + "##thorn": 19392, + "roosters": 19393, + "brim": 19394, + "crete": 19395, + "rotating": 19396, + "affected": 19397, + "##post": 19398, + "aber": 19399, + "metre": 19400, + "turf": 19401, + "cannons": 19402, + "ibiza": 19403, + "reminds": 19404, + "87": 19405, + "##eenth": 19406, + "outskirts": 19407, + "spike": 19408, + "excess": 19409, + "ameth": 19410, + "infused": 19411, + "sharper": 19412, + "texting": 19413, + "##othe": 19414, + "davis": 19415, + "achievement": 19416, + "michel": 19417, + "adaptation": 19418, + "quantum": 19419, + "kw": 19420, + "escort": 19421, + "##ango": 19422, + "pizzas": 19423, + "battlefield": 19424, + "erosion": 19425, + "erase": 19426, + "corset": 19427, + "bavaria": 19428, + "##64": 19429, + "spill": 19430, + "comments": 19431, + "housew": 19432, + "favorites": 19433, + "cranberries": 19434, + "##nt": 19435, + "wolverine": 19436, + "rated": 19437, + "wides": 19438, + "##aren": 19439, + "pistach": 19440, + "compliment": 19441, + "gabb": 19442, + "goodby": 19443, + "dodgers": 19444, + "context": 19445, + "vue": 19446, + "salute": 19447, + "symbolism": 19448, + "trinidad": 19449, + "curriculum": 19450, + "##child": 19451, + "leak": 19452, + "sunken": 19453, + "1945": 19454, + "attacked": 19455, + "dozens": 19456, + "naz": 19457, + "nerve": 19458, + "##uter": 19459, + "corr": 19460, + "radial": 19461, + "cruising": 19462, + "##arnia": 19463, + "osaka": 19464, + "boundary": 19465, + "##keley": 19466, + "networks": 19467, + "sturdy": 19468, + "theor": 19469, + "croches": 19470, + "94": 19471, + "cutest": 19472, + "genetic": 19473, + "gps": 19474, + "stuffing": 19475, + "stressed": 19476, + "crutts": 19477, + "aboriginal": 19478, + "##har": 19479, + "plast": 19480, + "earned": 19481, + "aviv": 19482, + "heavily": 19483, + "minds": 19484, + "fears": 19485, + "pong": 19486, + "##erate": 19487, + "##shaw": 19488, + "specialty": 19489, + "paying": 19490, + "immac": 19491, + "diorama": 19492, + "hos": 19493, + "sd": 19494, + "##enburg": 19495, + "anah": 19496, + "1939": 19497, + "##irting": 19498, + "overwhel": 19499, + "finale": 19500, + "serviced": 19501, + "ironing": 19502, + "reversible": 19503, + "fresco": 19504, + "conduct": 19505, + "resurre": 19506, + "##friends": 19507, + "storks": 19508, + "reporter": 19509, + "herringbone": 19510, + "upset": 19511, + "chew": 19512, + "vendors": 19513, + "deployed": 19514, + "bodybuilder": 19515, + "kend": 19516, + "wasps": 19517, + "safely": 19518, + "vests": 19519, + "eyebrow": 19520, + "##lava": 19521, + "##mented": 19522, + "elves": 19523, + "radar": 19524, + "puns": 19525, + "heeled": 19526, + "failed": 19527, + "cnc": 19528, + "monterey": 19529, + "nach": 19530, + "chrom": 19531, + "waiter": 19532, + "malaysian": 19533, + "feta": 19534, + "optimus": 19535, + "studs": 19536, + "oracle": 19537, + "valencia": 19538, + "throat": 19539, + "dwellings": 19540, + "chakra": 19541, + "menus": 19542, + "writes": 19543, + "initiative": 19544, + "slayer": 19545, + "juvenile": 19546, + "##lighting": 19547, + "bonfire": 19548, + "luke": 19549, + "parkway": 19550, + "header": 19551, + "teammates": 19552, + "okla": 19553, + "taupe": 19554, + "venezuela": 19555, + "upward": 19556, + "berkeley": 19557, + "authorities": 19558, + "belongs": 19559, + "##electric": 19560, + "##itos": 19561, + "buttery": 19562, + "libraries": 19563, + "rapt": 19564, + "upholstered": 19565, + "##69": 19566, + "simulation": 19567, + "mistake": 19568, + "taurus": 19569, + "zig": 19570, + "##omous": 19571, + "##oland": 19572, + "cheshire": 19573, + "spik": 19574, + "contro": 19575, + "commonwe": 19576, + "scanner": 19577, + "guidelines": 19578, + "huts": 19579, + "chia": 19580, + "sprin": 19581, + "lowest": 19582, + "##aran": 19583, + "woodcut": 19584, + "corgle": 19585, + "coloss": 19586, + "pods": 19587, + "promen": 19588, + "##apor": 19589, + "glorious": 19590, + "##porter": 19591, + "unlike": 19592, + "##ario": 19593, + "##plements": 19594, + "attorney": 19595, + "strategic": 19596, + "##inker": 19597, + "daycare": 19598, + "stray": 19599, + "labour": 19600, + "commonwealth": 19601, + "toll": 19602, + "##aire": 19603, + "tanker": 19604, + "branded": 19605, + "rhine": 19606, + "recorded": 19607, + "shimmer": 19608, + "cockato": 19609, + "occurs": 19610, + "prosper": 19611, + "hotspur": 19612, + "##amins": 19613, + "navig": 19614, + "collaps": 19615, + "downhill": 19616, + "controvers": 19617, + "zones": 19618, + "unhe": 19619, + "hemp": 19620, + "stirring": 19621, + "johannesburg": 19622, + "##adena": 19623, + "pointe": 19624, + "recessed": 19625, + "louisville": 19626, + "unforgett": 19627, + "nun": 19628, + "scrib": 19629, + "marshmallows": 19630, + "inaugural": 19631, + "chama": 19632, + "savings": 19633, + "trumpets": 19634, + "reportage": 19635, + "jacu": 19636, + "m1": 19637, + "partial": 19638, + "laud": 19639, + "pediatric": 19640, + "##ghal": 19641, + "payphone": 19642, + "maharash": 19643, + "opal": 19644, + "##burst": 19645, + "celery": 19646, + "majority": 19647, + "mclaren": 19648, + "pueblo": 19649, + "unforgettable": 19650, + "walnuts": 19651, + "rotter": 19652, + "shakespear": 19653, + "gambling": 19654, + "aids": 19655, + "gs": 19656, + "##gom": 19657, + "blends": 19658, + "lancaster": 19659, + "codes": 19660, + "lawrence": 19661, + "philips": 19662, + "error": 19663, + "promoting": 19664, + "tunic": 19665, + "bauhaus": 19666, + "bermuda": 19667, + "knowing": 19668, + "##mills": 19669, + "##bria": 19670, + "assemble": 19671, + "pagan": 19672, + "##olph": 19673, + "icy": 19674, + "rhinestone": 19675, + "shaft": 19676, + "##contin": 19677, + "realty": 19678, + "eyeglasses": 19679, + "hyene": 19680, + "tand": 19681, + "coding": 19682, + "chipm": 19683, + "maiden": 19684, + "##verty": 19685, + "ruined": 19686, + "nikon": 19687, + "rhythm": 19688, + "goodbye": 19689, + "morph": 19690, + "propeller": 19691, + "database": 19692, + "eighth": 19693, + "narnia": 19694, + "vows": 19695, + "eleven": 19696, + "interst": 19697, + "infrared": 19698, + "guj": 19699, + "perfumes": 19700, + "oysters": 19701, + "jen": 19702, + "hummer": 19703, + "competitive": 19704, + "mossy": 19705, + "hydroelectric": 19706, + "promenade": 19707, + "handgun": 19708, + "binding": 19709, + "vig": 19710, + "bait": 19711, + "nept": 19712, + "disconn": 19713, + "vikings": 19714, + "energ": 19715, + "forecast": 19716, + "wolfhound": 19717, + "priests": 19718, + "alaskan": 19719, + "grays": 19720, + "stopping": 19721, + "titanic": 19722, + "medallion": 19723, + "##uras": 19724, + "poverty": 19725, + "contrasting": 19726, + "##tto": 19727, + "boiling": 19728, + "moored": 19729, + "procession": 19730, + "thinks": 19731, + "fifty": 19732, + "##ifies": 19733, + "capitan": 19734, + "donation": 19735, + "witchcraft": 19736, + "50s": 19737, + "norwich": 19738, + "raises": 19739, + "telesc": 19740, + "hid": 19741, + "demonstrating": 19742, + "ded": 19743, + "len": 19744, + "##iculous": 19745, + "hunts": 19746, + "dalmat": 19747, + "darwin": 19748, + "construct": 19749, + "liked": 19750, + "autonomous": 19751, + "ranges": 19752, + "ducati": 19753, + "stiletto": 19754, + "cycles": 19755, + "kiev": 19756, + "whiteboard": 19757, + "darling": 19758, + "lucas": 19759, + "##63": 19760, + "##oro": 19761, + "compla": 19762, + "edmond": 19763, + "huntington": 19764, + "grands": 19765, + "pompe": 19766, + "tranquil": 19767, + "colleagues": 19768, + "shakespeare": 19769, + "##fic": 19770, + "notebooks": 19771, + "poul": 19772, + "##etum": 19773, + "headlights": 19774, + "signals": 19775, + "magenta": 19776, + "relatively": 19777, + "beekeeper": 19778, + "affairs": 19779, + "challenger": 19780, + "announces": 19781, + "grinding": 19782, + "nich": 19783, + "##fest": 19784, + "peoples": 19785, + "representative": 19786, + "baboon": 19787, + "lure": 19788, + "kingston": 19789, + "albany": 19790, + "mould": 19791, + "##enos": 19792, + "palais": 19793, + "freest": 19794, + "bronco": 19795, + "vineyards": 19796, + "survivor": 19797, + "beware": 19798, + "spong": 19799, + "flakes": 19800, + "schematic": 19801, + "kerry": 19802, + "slam": 19803, + "inu": 19804, + "divas": 19805, + "##grown": 19806, + "blacksmith": 19807, + "ukulele": 19808, + "##ez": 19809, + "arn": 19810, + "mohawk": 19811, + "trad": 19812, + "montene": 19813, + "zombies": 19814, + "crows": 19815, + "magician": 19816, + "advantages": 19817, + "semifinal": 19818, + "##rat": 19819, + "chino": 19820, + "stronger": 19821, + "spectators": 19822, + "sheraton": 19823, + "staples": 19824, + "nes": 19825, + "##vere": 19826, + "veneer": 19827, + "##opotam": 19828, + "##yang": 19829, + "redo": 19830, + "motorbikes": 19831, + "synthetic": 19832, + "hues": 19833, + "roles": 19834, + "phara": 19835, + "summertime": 19836, + "cauld": 19837, + "wandering": 19838, + "assessment": 19839, + "81": 19840, + "bargain": 19841, + "capturing": 19842, + "##vil": 19843, + "conj": 19844, + "doubles": 19845, + "critic": 19846, + "crafty": 19847, + "gymnas": 19848, + "carniv": 19849, + "cotswolds": 19850, + "anaheim": 19851, + "furs": 19852, + "nelson": 19853, + "##aa": 19854, + "stationary": 19855, + "freder": 19856, + "cords": 19857, + "macaroni": 19858, + "drainage": 19859, + "siamese": 19860, + "##estershire": 19861, + "rockies": 19862, + "headphone": 19863, + "hairless": 19864, + "honored": 19865, + "002": 19866, + "accented": 19867, + "superstar": 19868, + "mascara": 19869, + "gymnasium": 19870, + "enfield": 19871, + "quarry": 19872, + "drought": 19873, + "kc": 19874, + "nugg": 19875, + "lattice": 19876, + "duct": 19877, + "processes": 19878, + "anymore": 19879, + "nairobi": 19880, + "rook": 19881, + "##acia": 19882, + "coverage": 19883, + "maver": 19884, + "tcl": 19885, + "##asia": 19886, + "##uddle": 19887, + "polynes": 19888, + "accompl": 19889, + "frequency": 19890, + "##edonia": 19891, + "greets": 19892, + "indy": 19893, + "cardinals": 19894, + "southampton": 19895, + "psalm": 19896, + "shy": 19897, + "outlined": 19898, + "wheeled": 19899, + "increases": 19900, + "normally": 19901, + "repeating": 19902, + "merchant": 19903, + "##ivan": 19904, + "apost": 19905, + "locally": 19906, + "##erton": 19907, + "humanity": 19908, + "quilted": 19909, + "##ections": 19910, + "marry": 19911, + "indicate": 19912, + "nickelode": 19913, + "maharashtra": 19914, + "zur": 19915, + "unload": 19916, + "electro": 19917, + "missed": 19918, + "burlington": 19919, + "metropolis": 19920, + "jing": 19921, + "##iya": 19922, + "inspirations": 19923, + "inlay": 19924, + "##isl": 19925, + "phases": 19926, + "pling": 19927, + "perfection": 19928, + "chopper": 19929, + "stallion": 19930, + "serbia": 19931, + "##erce": 19932, + "broadcast": 19933, + "recommended": 19934, + "##ishment": 19935, + "##ellar": 19936, + "patina": 19937, + "rott": 19938, + "snooker": 19939, + "zag": 19940, + "grandpa": 19941, + "remembr": 19942, + "##omon": 19943, + "##chers": 19944, + "syl": 19945, + "soundtra": 19946, + "lgbt": 19947, + "dahlia": 19948, + "headdress": 19949, + "nickelodeon": 19950, + "snowboarding": 19951, + "loader": 19952, + "lahore": 19953, + "rotten": 19954, + "dipped": 19955, + "##wer": 19956, + "invited": 19957, + "cafeter": 19958, + "stayed": 19959, + "1920x1200": 19960, + "screwdriver": 19961, + "choker": 19962, + "innoc": 19963, + "musicals": 19964, + "tamar": 19965, + "##inhua": 19966, + "waterloo": 19967, + "stables": 19968, + "guer": 19969, + "paver": 19970, + "hikers": 19971, + "hash": 19972, + "##dler": 19973, + "##bilt": 19974, + "corns": 19975, + "imaginary": 19976, + "salvador": 19977, + "adapted": 19978, + "freestanding": 19979, + "##ayette": 19980, + "##itions": 19981, + "teleg": 19982, + "milkshake": 19983, + "jacuzzi": 19984, + "drills": 19985, + "allen": 19986, + "amg": 19987, + "##swick": 19988, + "remembrance": 19989, + "ditch": 19990, + "plowing": 19991, + "usage": 19992, + "strict": 19993, + "burj": 19994, + "##flow": 19995, + "warranty": 19996, + "##eons": 19997, + "asset": 19998, + "violence": 19999, + "countdown": 20000, + "lewis": 20001, + "betta": 20002, + "expanded": 20003, + "anz": 20004, + "churn": 20005, + "embed": 20006, + "dea": 20007, + "participated": 20008, + "replacing": 20009, + "montenegro": 20010, + "##bab": 20011, + "##awi": 20012, + "nightstand": 20013, + "feathered": 20014, + "christianity": 20015, + "gemstones": 20016, + "##58": 20017, + "burri": 20018, + "jasmine": 20019, + "##very": 20020, + "masked": 20021, + "registration": 20022, + "closets": 20023, + "econom": 20024, + "edging": 20025, + "provence": 20026, + "awakening": 20027, + "##ifter": 20028, + "hornet": 20029, + "lasagna": 20030, + "transported": 20031, + "1937": 20032, + "patriot": 20033, + "##hoot": 20034, + "cornbread": 20035, + "fills": 20036, + "xinhua": 20037, + "##rain": 20038, + "weimar": 20039, + "flair": 20040, + "crafting": 20041, + "testament": 20042, + "neptune": 20043, + "poultry": 20044, + "mull": 20045, + "angler": 20046, + "##veless": 20047, + "darker": 20048, + "terraces": 20049, + "##x6": 20050, + "boath": 20051, + "custard": 20052, + "##stead": 20053, + "ches": 20054, + "guaca": 20055, + "##ovs": 20056, + "preschoolers": 20057, + "substit": 20058, + "sleeveless": 20059, + "vitamins": 20060, + "##pucc": 20061, + "qum": 20062, + "##ogra": 20063, + "boiler": 20064, + "stoneh": 20065, + "enlar": 20066, + "variation": 20067, + "vaulted": 20068, + "ron": 20069, + "rhubar": 20070, + "hedges": 20071, + "dungeons": 20072, + "rhubarb": 20073, + "swimwear": 20074, + "marketplace": 20075, + "dolly": 20076, + "uttar": 20077, + "hydrogen": 20078, + "##fill": 20079, + "civilizations": 20080, + "expeditionary": 20081, + "cousin": 20082, + "##eu": 20083, + "cafeteria": 20084, + "guil": 20085, + "goalie": 20086, + "molecule": 20087, + "clubhouse": 20088, + "1947": 20089, + "##bet": 20090, + "##agawa": 20091, + "fashions": 20092, + "brutal": 20093, + "ecosystem": 20094, + "sultan": 20095, + "pharaoh": 20096, + "grease": 20097, + "leap": 20098, + "##olo": 20099, + "grapefruit": 20100, + "ely": 20101, + "minh": 20102, + "tyr": 20103, + "garment": 20104, + "insign": 20105, + "taped": 20106, + "gentlemen": 20107, + "describes": 20108, + "prestige": 20109, + "bals": 20110, + "um": 20111, + "yeast": 20112, + "managing": 20113, + "asos": 20114, + "muss": 20115, + "curl": 20116, + "ghosts": 20117, + "crystal": 20118, + "hatchback": 20119, + "scallop": 20120, + "pigeons": 20121, + "##ucco": 20122, + "reno": 20123, + "marshall": 20124, + "irregular": 20125, + "cogn": 20126, + "allerg": 20127, + "soles": 20128, + "toledo": 20129, + "bd": 20130, + "##elier": 20131, + "reds": 20132, + "cocked": 20133, + "##ete": 20134, + "rhodes": 20135, + "gradients": 20136, + "hovering": 20137, + "##acas": 20138, + "swar": 20139, + "authors": 20140, + "montgom": 20141, + "wwii": 20142, + "utensils": 20143, + "##puccino": 20144, + "gosp": 20145, + "scent": 20146, + "##enci": 20147, + "crimson": 20148, + "boathouse": 20149, + "stucco": 20150, + "##imo": 20151, + "grat": 20152, + "tailgate": 20153, + "identical": 20154, + "jr": 20155, + "mainland": 20156, + "danube": 20157, + "sentence": 20158, + "trampoline": 20159, + "##borg": 20160, + "specialized": 20161, + "bolivia": 20162, + "blended": 20163, + "neighbors": 20164, + "narrative": 20165, + "sleigh": 20166, + "workb": 20167, + "entitled": 20168, + "asylum": 20169, + "grandparents": 20170, + "quail": 20171, + "hend": 20172, + "boundaries": 20173, + "booking": 20174, + "swiv": 20175, + "convenience": 20176, + "chipmunk": 20177, + "##home": 20178, + "constitution": 20179, + "##isph": 20180, + "arom": 20181, + "slave": 20182, + "concerns": 20183, + "physiology": 20184, + "chann": 20185, + "anticip": 20186, + "veloc": 20187, + "molecules": 20188, + "stamford": 20189, + "##kah": 20190, + "##see": 20191, + "bri": 20192, + "##aston": 20193, + "caulia": 20194, + "please": 20195, + "reproductions": 20196, + "montgomery": 20197, + "chero": 20198, + "doughs": 20199, + "referred": 20200, + "punjabi": 20201, + "barbers": 20202, + "enduro": 20203, + "macau": 20204, + "##athe": 20205, + "fierce": 20206, + "serene": 20207, + "stingray": 20208, + "suggestions": 20209, + "##x768": 20210, + "zurich": 20211, + "trucker": 20212, + "##ifications": 20213, + "pressing": 20214, + "##yptic": 20215, + "mids": 20216, + "gifs": 20217, + "packers": 20218, + "1952": 20219, + "bassine": 20220, + "thoughtful": 20221, + "explained": 20222, + "apocalyptic": 20223, + "constellations": 20224, + "tp": 20225, + "patron": 20226, + "excurs": 20227, + "rickshaw": 20228, + "paddles": 20229, + "performer": 20230, + "lollipop": 20231, + "amethyst": 20232, + "pood": 20233, + "##pin": 20234, + "feeds": 20235, + "capable": 20236, + "curse": 20237, + "135": 20238, + "stepped": 20239, + "judges": 20240, + "sustainability": 20241, + "pia": 20242, + "specifically": 20243, + "##hang": 20244, + "##ido": 20245, + "manage": 20246, + "redhead": 20247, + "lays": 20248, + "kazakh": 20249, + "##now": 20250, + "##jet": 20251, + "agents": 20252, + "saab": 20253, + "delightful": 20254, + "##ritis": 20255, + "arboretum": 20256, + "brunswick": 20257, + "watercolors": 20258, + "oranguel": 20259, + "frolic": 20260, + "caution": 20261, + "waik": 20262, + "rival": 20263, + "##pea": 20264, + "mangrove": 20265, + "cherokee": 20266, + "qun": 20267, + "stim": 20268, + "lowered": 20269, + "sphinx": 20270, + "immaculate": 20271, + "fibre": 20272, + "surviving": 20273, + "pajama": 20274, + "suffolk": 20275, + "##iago": 20276, + "epide": 20277, + "administrative": 20278, + "jaws": 20279, + "carpenter": 20280, + "nutrients": 20281, + "quill": 20282, + "outfitters": 20283, + "axle": 20284, + "lambs": 20285, + "##brew": 20286, + "barometer": 20287, + "turban": 20288, + "courier": 20289, + "crayons": 20290, + "respiratory": 20291, + "whirl": 20292, + "mare": 20293, + "strang": 20294, + "blueprints": 20295, + "mozz": 20296, + "performers": 20297, + "matched": 20298, + "imported": 20299, + "wetland": 20300, + "extends": 20301, + "ngc": 20302, + "ketchup": 20303, + "mv": 20304, + "rape": 20305, + "##97": 20306, + "##elago": 20307, + "wrink": 20308, + "florals": 20309, + "2000s": 20310, + "stern": 20311, + "reward": 20312, + "sprinkled": 20313, + "tasks": 20314, + "lich": 20315, + "tm": 20316, + "##cam": 20317, + "centerpieces": 20318, + "react": 20319, + "osh": 20320, + "ariel": 20321, + "##yss": 20322, + "105": 20323, + "bradford": 20324, + "plough": 20325, + "fairly": 20326, + "augusta": 20327, + "indicates": 20328, + "encounter": 20329, + "##uation": 20330, + "kangaroos": 20331, + "gemini": 20332, + "sas": 20333, + "zucum": 20334, + "##ason": 20335, + "scaff": 20336, + "generic": 20337, + "speeding": 20338, + "rainforests": 20339, + "rw": 20340, + "blizzard": 20341, + "##arella": 20342, + "##itaire": 20343, + "soundtrack": 20344, + "##200": 20345, + "##oux": 20346, + "slovak": 20347, + "1936": 20348, + "fraction": 20349, + "breathe": 20350, + "meditating": 20351, + "prominent": 20352, + "divorce": 20353, + "##stop": 20354, + "barrow": 20355, + "captions": 20356, + "##cass": 20357, + "offset": 20358, + "veins": 20359, + "medicinal": 20360, + "corrug": 20361, + "balenci": 20362, + "spheres": 20363, + "snowmobiler": 20364, + "balenciaga": 20365, + "##igo": 20366, + "bloomberg": 20367, + "keepsake": 20368, + "werewolf": 20369, + "pty": 20370, + "##urg": 20371, + "doom": 20372, + "belief": 20373, + "##oka": 20374, + "49ers": 20375, + "engage": 20376, + "spokes": 20377, + "##lies": 20378, + "##inia": 20379, + "scotts": 20380, + "biography": 20381, + "furnace": 20382, + "centered": 20383, + "imagery": 20384, + "cupboards": 20385, + "departure": 20386, + "firearm": 20387, + "torches": 20388, + "popularity": 20389, + "gazelle": 20390, + "##ilia": 20391, + "reop": 20392, + "inca": 20393, + "snowmobiles": 20394, + "##acea": 20395, + "legisl": 20396, + "porh": 20397, + "railways": 20398, + "senses": 20399, + "therapist": 20400, + "sunsets": 20401, + "bred": 20402, + "butler": 20403, + "caj": 20404, + "linens": 20405, + "sudden": 20406, + "satch": 20407, + "appal": 20408, + "detector": 20409, + "grocer": 20410, + "curiosity": 20411, + "corrugated": 20412, + "##ooked": 20413, + "seiko": 20414, + "extin": 20415, + "confection": 20416, + "climbs": 20417, + "robotic": 20418, + "negot": 20419, + "aries": 20420, + "flagship": 20421, + "##abeth": 20422, + "unhap": 20423, + "subs": 20424, + "apricot": 20425, + "lama": 20426, + "vas": 20427, + "bruce": 20428, + "aprons": 20429, + "madness": 20430, + "affection": 20431, + "tango": 20432, + "nv": 20433, + "bry": 20434, + "soror": 20435, + "gangster": 20436, + "seashell": 20437, + "sponges": 20438, + "zion": 20439, + "##nings": 20440, + "##eta": 20441, + "bernese": 20442, + "mechanics": 20443, + "whistler": 20444, + "lingers": 20445, + "totems": 20446, + "pushed": 20447, + "guineas": 20448, + "tinker": 20449, + "##ombs": 20450, + "##olith": 20451, + "thats": 20452, + "holmes": 20453, + "airborne": 20454, + "graduated": 20455, + "dul": 20456, + "nui": 20457, + "vand": 20458, + "stability": 20459, + "disappoint": 20460, + "qing": 20461, + "addict": 20462, + "flew": 20463, + "sprinkle": 20464, + "lentil": 20465, + "schnauzer": 20466, + "aims": 20467, + "hiring": 20468, + "oral": 20469, + "underway": 20470, + "everett": 20471, + "mathematics": 20472, + "waldorf": 20473, + "sard": 20474, + "##oning": 20475, + "beating": 20476, + "fiery": 20477, + "khan": 20478, + "fittings": 20479, + "slender": 20480, + "albania": 20481, + "rih": 20482, + "##kon": 20483, + "tracking": 20484, + "collaborative": 20485, + "##chen": 20486, + "solitaire": 20487, + "ecological": 20488, + "lively": 20489, + "tune": 20490, + "charged": 20491, + "salm": 20492, + "sinking": 20493, + "facades": 20494, + "yeti": 20495, + "fendi": 20496, + "airs": 20497, + "overcast": 20498, + "##isal": 20499, + "sculptural": 20500, + "proverbs": 20501, + "83": 20502, + "##jes": 20503, + "##chu": 20504, + "toppers": 20505, + "gallon": 20506, + "entered": 20507, + "collared": 20508, + "satchel": 20509, + "gw": 20510, + "debt": 20511, + "tuc": 20512, + "cebu": 20513, + "##rah": 20514, + "proc": 20515, + "toucal": 20516, + "iw": 20517, + "unbel": 20518, + "listens": 20519, + "joints": 20520, + "ih": 20521, + "scented": 20522, + "welcomes": 20523, + "disposable": 20524, + "graceful": 20525, + "##oan": 20526, + "outlines": 20527, + "##ultures": 20528, + "relations": 20529, + "cheeseburger": 20530, + "java": 20531, + "##hair": 20532, + "temp": 20533, + "ostrica": 20534, + "demonstrates": 20535, + "ferns": 20536, + "manitoba": 20537, + "##abilities": 20538, + "blackbird": 20539, + "sheikh": 20540, + "leafs": 20541, + "solving": 20542, + "homage": 20543, + "insane": 20544, + "tuft": 20545, + "sponsor": 20546, + "##zan": 20547, + "beau": 20548, + "organizations": 20549, + "napole": 20550, + "jub": 20551, + "##esque": 20552, + "playa": 20553, + "1932": 20554, + "1948": 20555, + "completion": 20556, + "embellished": 20557, + "skylight": 20558, + "determined": 20559, + "##iour": 20560, + "honors": 20561, + "sensitive": 20562, + "armadis": 20563, + "ganges": 20564, + "boulders": 20565, + "neo": 20566, + "beachfront": 20567, + "tracker": 20568, + "combining": 20569, + "myths": 20570, + "huskies": 20571, + "rihanna": 20572, + "lords": 20573, + "##aches": 20574, + "prophet": 20575, + "hairdresser": 20576, + "bts": 20577, + "##making": 20578, + "##enstein": 20579, + "photographing": 20580, + "halter": 20581, + "explo": 20582, + "automated": 20583, + "analyt": 20584, + "sorority": 20585, + "ole": 20586, + "thyme": 20587, + "##ishes": 20588, + "creme": 20589, + "accoun": 20590, + "organised": 20591, + "dwarfs": 20592, + "##strom": 20593, + "brom": 20594, + "sey": 20595, + "##mani": 20596, + "subt": 20597, + "##iveness": 20598, + "george": 20599, + "prost": 20600, + "dix": 20601, + "mughal": 20602, + "adjusting": 20603, + "hemisph": 20604, + "unhappy": 20605, + "91": 20606, + "##ardo": 20607, + "marra": 20608, + "movements": 20609, + "capes": 20610, + "radiation": 20611, + "70th": 20612, + "uh": 20613, + "##ocation": 20614, + "hermitage": 20615, + "pursuit": 20616, + "tehran": 20617, + "unbeliev": 20618, + "ate": 20619, + "opel": 20620, + "procedures": 20621, + "##asaki": 20622, + "handling": 20623, + "##getown": 20624, + "regularly": 20625, + "fiddle": 20626, + "flanked": 20627, + "##feed": 20628, + "##kong": 20629, + "generous": 20630, + "hels": 20631, + "canine": 20632, + "monet": 20633, + "##awei": 20634, + "##isson": 20635, + "circum": 20636, + "unicorns": 20637, + "khal": 20638, + "ming": 20639, + "flowered": 20640, + "1200": 20641, + "grinder": 20642, + "snowball": 20643, + "squats": 20644, + "##ropolis": 20645, + "spells": 20646, + "corolla": 20647, + "providence": 20648, + "##ograf": 20649, + "sith": 20650, + "wast": 20651, + "colorfully": 20652, + "plateau": 20653, + "pipeline": 20654, + "alchemy": 20655, + "mozzarella": 20656, + "##mobile": 20657, + "##92": 20658, + "mirage": 20659, + "raff": 20660, + "aberde": 20661, + "##atown": 20662, + "salam": 20663, + "santiago": 20664, + "dealership": 20665, + "projected": 20666, + "spanning": 20667, + "pompeii": 20668, + "reuse": 20669, + "valves": 20670, + "organisation": 20671, + "starfishs": 20672, + "a€": 20673, + "dong": 20674, + "yin": 20675, + "corp": 20676, + "ribbed": 20677, + "lungs": 20678, + "munch": 20679, + "storefront": 20680, + "reap": 20681, + "framework": 20682, + "folds": 20683, + "automatically": 20684, + "confederate": 20685, + "restore": 20686, + "contained": 20687, + "blocking": 20688, + "chemicals": 20689, + "indicator": 20690, + "tilt": 20691, + "glider": 20692, + "letting": 20693, + "entryway": 20694, + "receipt": 20695, + "metallica": 20696, + "cicada": 20697, + "mongolian": 20698, + "bitten": 20699, + "##cad": 20700, + "humble": 20701, + "resurrection": 20702, + "extinction": 20703, + "220": 20704, + "slash": 20705, + "racquet": 20706, + "songwriter": 20707, + "lb": 20708, + "##ataka": 20709, + "stunt": 20710, + "cashew": 20711, + "weighs": 20712, + "forged": 20713, + "clara": 20714, + "domino": 20715, + "insignia": 20716, + "xxl": 20717, + "bumblebee": 20718, + "interpretation": 20719, + "fee": 20720, + "##enheim": 20721, + "##omin": 20722, + "surrounds": 20723, + "sparkles": 20724, + "immers": 20725, + "seashore": 20726, + "93": 20727, + "£1": 20728, + "storybook": 20729, + "huawei": 20730, + "groot": 20731, + "splashes": 20732, + "stead": 20733, + "illustrating": 20734, + "##inton": 20735, + "monop": 20736, + "windmills": 20737, + "##eshow": 20738, + "packets": 20739, + "vie": 20740, + "##eli": 20741, + "##adal": 20742, + "southwestern": 20743, + "conducting": 20744, + "nuggets": 20745, + "asks": 20746, + "saloon": 20747, + "measured": 20748, + "coachella": 20749, + "kawasaki": 20750, + "tzu": 20751, + "mecca": 20752, + "overc": 20753, + "1914": 20754, + "serial": 20755, + "panthers": 20756, + "nestled": 20757, + "cinemas": 20758, + "chunks": 20759, + "abundance": 20760, + "masj": 20761, + "rotation": 20762, + "axes": 20763, + "cupid": 20764, + "fryer": 20765, + "lep": 20766, + "horr": 20767, + "silverado": 20768, + "typographic": 20769, + "gymnasts": 20770, + "indicating": 20771, + "farewell": 20772, + "fluorescent": 20773, + "aviator": 20774, + "uz": 20775, + "deers": 20776, + "clippers": 20777, + "1951": 20778, + "dungeon": 20779, + "surveill": 20780, + "##nom": 20781, + "##milk": 20782, + "##utation": 20783, + "flut": 20784, + "promised": 20785, + "swivel": 20786, + "jester": 20787, + "whom": 20788, + "##rov": 20789, + "salwar": 20790, + "policeman": 20791, + "snowfall": 20792, + "filmed": 20793, + "sleeper": 20794, + "candies": 20795, + "exec": 20796, + "contractor": 20797, + "guidance": 20798, + "fotograf": 20799, + "surveillance": 20800, + "##tone": 20801, + "grassland": 20802, + "firefly": 20803, + "##ymes": 20804, + "jamaican": 20805, + "##anz": 20806, + "##ombo": 20807, + "andes": 20808, + "##igious": 20809, + "ree": 20810, + "##anasi": 20811, + "karnataka": 20812, + "sacrif": 20813, + "737": 20814, + "kard": 20815, + "spurs": 20816, + "looney": 20817, + "creams": 20818, + "fertil": 20819, + "affiliate": 20820, + "embedded": 20821, + "laf": 20822, + "##elli": 20823, + "happening": 20824, + "migration": 20825, + "varanasi": 20826, + "luxor": 20827, + "hobo": 20828, + "newspapers": 20829, + "gabbana": 20830, + "unav": 20831, + "carpets": 20832, + "samba": 20833, + "gust": 20834, + "disorder": 20835, + "underest": 20836, + "videog": 20837, + "trilop": 20838, + "attempts": 20839, + "dave": 20840, + "realism": 20841, + "yarns": 20842, + "explaining": 20843, + "selecting": 20844, + "condos": 20845, + "breastfeed": 20846, + "roughly": 20847, + "##agua": 20848, + "##unner": 20849, + "flaw": 20850, + "refu": 20851, + "storyt": 20852, + "nd": 20853, + "##lass": 20854, + "plas": 20855, + "cursive": 20856, + "seasoned": 20857, + "worksheet": 20858, + "atelier": 20859, + "mounting": 20860, + "prett": 20861, + "##pendicular": 20862, + "overlooks": 20863, + "myrtle": 20864, + "mck": 20865, + "agility": 20866, + "homeschool": 20867, + "comeback": 20868, + "1938": 20869, + "##ribed": 20870, + "comedian": 20871, + "complicated": 20872, + "##vine": 20873, + "moj": 20874, + "comet": 20875, + "noah": 20876, + "##ucing": 20877, + "003": 20878, + "honoring": 20879, + "figurative": 20880, + "ibis": 20881, + "depends": 20882, + "freelance": 20883, + "ion": 20884, + "underworld": 20885, + "graphs": 20886, + "tailor": 20887, + "bedford": 20888, + "centennial": 20889, + "preserv": 20890, + "coronation": 20891, + "hoping": 20892, + "accommodate": 20893, + "bilt": 20894, + "##86": 20895, + "mang": 20896, + "levi": 20897, + "overpass": 20898, + "1928": 20899, + "journals": 20900, + "taxider": 20901, + "categories": 20902, + "carlo": 20903, + "##estine": 20904, + "viajes": 20905, + "digitally": 20906, + "##coco": 20907, + "reliable": 20908, + "applique": 20909, + "beaten": 20910, + "nass": 20911, + "pays": 20912, + "##the": 20913, + "posture": 20914, + "hummus": 20915, + "transparency": 20916, + "pilgrimage": 20917, + "anderson": 20918, + "reject": 20919, + "1890": 20920, + "senate": 20921, + "operate": 20922, + "enhanced": 20923, + "ranking": 20924, + "confirmed": 20925, + "proven": 20926, + "siem": 20927, + "splendid": 20928, + "archer": 20929, + "antig": 20930, + "flexibility": 20931, + "nicely": 20932, + "wanting": 20933, + "skeletons": 20934, + "asymmetrical": 20935, + "okin": 20936, + "masala": 20937, + "falcons": 20938, + "registered": 20939, + "##aret": 20940, + "##mann": 20941, + "medication": 20942, + "cth": 20943, + "sask": 20944, + "seine": 20945, + "cough": 20946, + "suicide": 20947, + "eliz": 20948, + "seminar": 20949, + "grandson": 20950, + "gospel": 20951, + "banking": 20952, + "monitoring": 20953, + "jefferson": 20954, + "pune": 20955, + "buttermilk": 20956, + "collars": 20957, + "consumption": 20958, + "lud": 20959, + "lymph": 20960, + "bast": 20961, + "vultures": 20962, + "seater": 20963, + "##ffe": 20964, + "##lette": 20965, + "roose": 20966, + "genre": 20967, + "tossed": 20968, + "funnel": 20969, + "##87": 20970, + "hadid": 20971, + "altitude": 20972, + "appreciate": 20973, + "inset": 20974, + "ostricus": 20975, + "masjid": 20976, + "archip": 20977, + "passive": 20978, + "shaping": 20979, + "1935": 20980, + "mulch": 20981, + "guacam": 20982, + "acacia": 20983, + "relevant": 20984, + "nutella": 20985, + "successfully": 20986, + "perry": 20987, + "epidemic": 20988, + "panting": 20989, + "champs": 20990, + "nib": 20991, + "sop": 20992, + "bendy": 20993, + "sniper": 20994, + "marc": 20995, + "unw": 20996, + "midtown": 20997, + "turmeric": 20998, + "sundae": 20999, + "arclight": 21000, + "refurbished": 21001, + "specimen": 21002, + "inhabit": 21003, + "adirondack": 21004, + "mykon": 21005, + "sweetened": 21006, + "kios": 21007, + "##her": 21008, + "thus": 21009, + "robert": 21010, + "snug": 21011, + "lima": 21012, + "##velt": 21013, + "redeem": 21014, + "##cestershire": 21015, + "##far": 21016, + "##lake": 21017, + "1942": 21018, + "tucson": 21019, + "mykonos": 21020, + "etern": 21021, + "##ener": 21022, + "##ulhu": 21023, + "seville": 21024, + "1941": 21025, + "1949": 21026, + "dakota": 21027, + "cthulhu": 21028, + "##94": 21029, + "manif": 21030, + "clearance": 21031, + "##erdale": 21032, + "diva": 21033, + "timberland": 21034, + "antarctic": 21035, + "capuchi": 21036, + "stoneware": 21037, + "sculpted": 21038, + "martian": 21039, + "karachi": 21040, + "ashes": 21041, + "moustache": 21042, + "yes": 21043, + "##lene": 21044, + "##vag": 21045, + "##seat": 21046, + "lease": 21047, + "##bek": 21048, + "imprint": 21049, + "stalks": 21050, + "drake": 21051, + "shortbread": 21052, + "savoy": 21053, + "involves": 21054, + "tir": 21055, + "rosewood": 21056, + "investigation": 21057, + "headlight": 21058, + "leashs": 21059, + "shields": 21060, + "perpendicular": 21061, + "100th": 21062, + "elizabeth": 21063, + "wip": 21064, + "##amar": 21065, + "argu": 21066, + "alleg": 21067, + "hallows": 21068, + "besides": 21069, + "savanna": 21070, + "toler": 21071, + "##foot": 21072, + "larv": 21073, + "hardy": 21074, + "platforms": 21075, + "marquee": 21076, + "rebell": 21077, + "captiv": 21078, + "streams": 21079, + "##ibo": 21080, + "##inki": 21081, + "##cia": 21082, + "disgu": 21083, + "oppon": 21084, + "objective": 21085, + "bulg": 21086, + "##amba": 21087, + "nationwide": 21088, + "proudly": 21089, + "kanagawa": 21090, + "beck": 21091, + "strapless": 21092, + "avant": 21093, + "pinball": 21094, + "prompt": 21095, + "##athy": 21096, + "pickled": 21097, + "scratching": 21098, + "chronograph": 21099, + "nosed": 21100, + "archipelago": 21101, + "viv": 21102, + "##bad": 21103, + "thy": 21104, + "cardio": 21105, + "concerts": 21106, + "rhinoc": 21107, + "##isk": 21108, + "colise": 21109, + "easel": 21110, + "heavyweight": 21111, + "jaguars": 21112, + "norman": 21113, + "pinscher": 21114, + "roosevelt": 21115, + "##eye": 21116, + "penal": 21117, + "consulting": 21118, + "gasoline": 21119, + "olympia": 21120, + "##oxy": 21121, + "pests": 21122, + "breastfeeding": 21123, + "hok": 21124, + "dean": 21125, + "peonies": 21126, + "ascot": 21127, + "swarm": 21128, + "pinned": 21129, + "epcot": 21130, + "biltmore": 21131, + "nie": 21132, + "##atical": 21133, + "onstage": 21134, + "perennial": 21135, + "amr": 21136, + "composer": 21137, + "unused": 21138, + "twenties": 21139, + "hir": 21140, + "sculptor": 21141, + "bunker": 21142, + "diction": 21143, + "readiness": 21144, + "hospit": 21145, + "lauderdale": 21146, + "##do": 21147, + "screened": 21148, + "meetings": 21149, + "1931": 21150, + "dwelling": 21151, + "royale": 21152, + "introduces": 21153, + "rococo": 21154, + "recept": 21155, + "mora": 21156, + "juvent": 21157, + "1944": 21158, + "attacking": 21159, + "shangri": 21160, + "peruvian": 21161, + "incorporate": 21162, + "prospect": 21163, + "bfi": 21164, + "lv": 21165, + "matchstick": 21166, + "##iju": 21167, + "510": 21168, + "tides": 21169, + "zool": 21170, + "isa": 21171, + "overseas": 21172, + "kara": 21173, + "wont": 21174, + "andal": 21175, + "backseat": 21176, + "noct": 21177, + "europa": 21178, + "competitors": 21179, + "repeated": 21180, + "gp": 21181, + "##anse": 21182, + "barc": 21183, + "struggling": 21184, + "specially": 21185, + "examination": 21186, + "##oosa": 21187, + "bombs": 21188, + "phrases": 21189, + "lauren": 21190, + "arthritis": 21191, + "piles": 21192, + "youngest": 21193, + "regent": 21194, + "predators": 21195, + "shiva": 21196, + "allu": 21197, + "##tlet": 21198, + "##aching": 21199, + "perimeter": 21200, + "shady": 21201, + "blocked": 21202, + "groomsmen": 21203, + "ubud": 21204, + "pewter": 21205, + "##57": 21206, + "##inas": 21207, + "##writers": 21208, + "chinatown": 21209, + "upgraded": 21210, + "rehabil": 21211, + "##continental": 21212, + "cly": 21213, + "uno": 21214, + "##iker": 21215, + "##anned": 21216, + "napa": 21217, + "impressionism": 21218, + "gouache": 21219, + "james": 21220, + "aruba": 21221, + "decimal": 21222, + "007": 21223, + "60th": 21224, + "turbines": 21225, + "unavailable": 21226, + "coliseum": 21227, + "hans": 21228, + "spilling": 21229, + "downstairs": 21230, + "jetty": 21231, + "catalogue": 21232, + "thriller": 21233, + "hens": 21234, + "uda": 21235, + "##roll": 21236, + "##ifax": 21237, + "##underst": 21238, + "airports": 21239, + "believes": 21240, + "pleated": 21241, + "gingham": 21242, + "kod": 21243, + "customizable": 21244, + "pinn": 21245, + "parachutes": 21246, + "##pro": 21247, + "##uga": 21248, + "pupils": 21249, + "productions": 21250, + "housed": 21251, + "sundance": 21252, + "queue": 21253, + "molecular": 21254, + "snorkeling": 21255, + "kau": 21256, + "nm": 21257, + "allergy": 21258, + "photoshoot": 21259, + "equations": 21260, + "galvan": 21261, + "steelers": 21262, + "bombay": 21263, + "pegas": 21264, + "elections": 21265, + "survived": 21266, + "retailer": 21267, + "shocking": 21268, + "vertebr": 21269, + "schoon": 21270, + "midwest": 21271, + "tradem": 21272, + "trent": 21273, + "gula": 21274, + "cabr": 21275, + "garfield": 21276, + "depressed": 21277, + "climbers": 21278, + "submarines": 21279, + "koalas": 21280, + "abyss": 21281, + "barge": 21282, + "attacks": 21283, + "sarc": 21284, + "observe": 21285, + "dhaka": 21286, + "helsinki": 21287, + "aus": 21288, + "flirting": 21289, + "##onda": 21290, + "riverdale": 21291, + "conducted": 21292, + "baguette": 21293, + "##apy": 21294, + "cancel": 21295, + "marig": 21296, + "radius": 21297, + "beret": 21298, + "apparently": 21299, + "defeated": 21300, + "sombren": 21301, + "##41": 21302, + "parker": 21303, + "motorhome": 21304, + "addams": 21305, + "engineered": 21306, + "##abilia": 21307, + "dazzling": 21308, + "xyl": 21309, + "crank": 21310, + "pallad": 21311, + "ceremonial": 21312, + "throwback": 21313, + "travellers": 21314, + "fondant": 21315, + "underestimate": 21316, + "okinawa": 21317, + "pits": 21318, + "spelling": 21319, + "charges": 21320, + "sev": 21321, + "crist": 21322, + "170": 21323, + "assets": 21324, + "lacquer": 21325, + "accounting": 21326, + "juventus": 21327, + "##56": 21328, + "##uas": 21329, + "drapes": 21330, + "delay": 21331, + "violins": 21332, + "signboard": 21333, + "symp": 21334, + "incense": 21335, + "effortless": 21336, + "mulberry": 21337, + "nod": 21338, + "radiant": 21339, + "radiac": 21340, + "avenger": 21341, + "trademark": 21342, + "##olding": 21343, + "##iceps": 21344, + "protons": 21345, + "blessings": 21346, + "maintained": 21347, + "jon": 21348, + "jal": 21349, + "ye": 21350, + "priced": 21351, + "laughter": 21352, + "republican": 21353, + "frequently": 21354, + "##kar": 21355, + "www": 21356, + "shouting": 21357, + "traff": 21358, + "clement": 21359, + "responsibility": 21360, + "##atin": 21361, + "##iber": 21362, + "grams": 21363, + "overgrown": 21364, + "shading": 21365, + "okay": 21366, + "philly": 21367, + "oxid": 21368, + "triathlon": 21369, + "kurta": 21370, + "caesar": 21371, + "sgt": 21372, + "##pants": 21373, + "candi": 21374, + "completing": 21375, + "##isteria": 21376, + "450": 21377, + "decisions": 21378, + "tyre": 21379, + "mosaics": 21380, + "sightse": 21381, + "nerf": 21382, + "octagon": 21383, + "bikers": 21384, + "buckles": 21385, + "clogs": 21386, + "##sburg": 21387, + "troop": 21388, + "playset": 21389, + "epoxy": 21390, + "credits": 21391, + "lunches": 21392, + "territ": 21393, + "incorporated": 21394, + "voting": 21395, + "cockatoo": 21396, + "##cats": 21397, + "##iman": 21398, + "##ateral": 21399, + "descent": 21400, + "##ateur": 21401, + "##awning": 21402, + "##oleum": 21403, + "residency": 21404, + "kardash": 21405, + "pecans": 21406, + "grouped": 21407, + "realize": 21408, + "halifax": 21409, + "pti": 21410, + "drizzle": 21411, + "##oire": 21412, + "abana": 21413, + "ethiopian": 21414, + "guru": 21415, + "##ifts": 21416, + "traits": 21417, + "competitions": 21418, + "borrow": 21419, + "consumers": 21420, + "##eastern": 21421, + "consultant": 21422, + "##gregation": 21423, + "parcel": 21424, + "croiss": 21425, + "improving": 21426, + "yangon": 21427, + "churchill": 21428, + "shortly": 21429, + "equality": 21430, + "fingerprint": 21431, + "cumbria": 21432, + "cockatoa": 21433, + "gast": 21434, + "wilde": 21435, + "hype": 21436, + "damask": 21437, + "shrug": 21438, + "cobalt": 21439, + "raptors": 21440, + "agencies": 21441, + "##encer": 21442, + "lagos": 21443, + "floppy": 21444, + "manipulation": 21445, + "fret": 21446, + "aware": 21447, + "lateral": 21448, + "appliance": 21449, + "upgrades": 21450, + "cbs": 21451, + "luth": 21452, + "##puff": 21453, + "smur": 21454, + "freeway": 21455, + "muert": 21456, + "libra": 21457, + "tuxed": 21458, + "taxidermy": 21459, + "##lore": 21460, + "##osc": 21461, + "hoi": 21462, + "piping": 21463, + "chiefs": 21464, + "pard": 21465, + "stil": 21466, + "wikimedia": 21467, + "udaipur": 21468, + "kilt": 21469, + "trough": 21470, + "beagles": 21471, + "transf": 21472, + "historia": 21473, + "victim": 21474, + "giraffes": 21475, + "pisa": 21476, + "scottsdale": 21477, + "groceries": 21478, + "sightseeing": 21479, + "##cap": 21480, + "scon": 21481, + "upwards": 21482, + "strainer": 21483, + "storing": 21484, + "sexy": 21485, + "orphan": 21486, + "memorabilia": 21487, + "dain": 21488, + "luna": 21489, + "xr": 21490, + "##isse": 21491, + "mocha": 21492, + "perenn": 21493, + "##boa": 21494, + "presidents": 21495, + "himalayas": 21496, + "vhs": 21497, + "##su": 21498, + "##ulas": 21499, + "doormat": 21500, + "grandd": 21501, + "madh": 21502, + "bleed": 21503, + "durga": 21504, + "whistles": 21505, + "closest": 21506, + "lockers": 21507, + "hutch": 21508, + "crumble": 21509, + "finnish": 21510, + "##agar": 21511, + "campbell": 21512, + "thighs": 21513, + "catamaran": 21514, + "rr": 21515, + "milton": 21516, + "teammate": 21517, + "##idon": 21518, + "foodie": 21519, + "wayne": 21520, + "23rd": 21521, + "parasai": 21522, + "rogers": 21523, + "stunts": 21524, + "##ijuana": 21525, + "##elia": 21526, + "copycat": 21527, + "multimedia": 21528, + "workshops": 21529, + "tangy": 21530, + "alchem": 21531, + "lom": 21532, + "parody": 21533, + "##ashi": 21534, + "distillery": 21535, + "hardly": 21536, + "picasso": 21537, + "tufted": 21538, + "seych": 21539, + "##onics": 21540, + "sliders": 21541, + "bram": 21542, + "tyne": 21543, + "haus": 21544, + "fault": 21545, + "pattaya": 21546, + "nassau": 21547, + "##rag": 21548, + "##eaux": 21549, + "monks": 21550, + "terraced": 21551, + "funds": 21552, + "fuchs": 21553, + "tiaras": 21554, + "sigma": 21555, + "stealth": 21556, + "crucial": 21557, + "extending": 21558, + "booths": 21559, + "irrigation": 21560, + "booklet": 21561, + "grasslands": 21562, + "skins": 21563, + "adams": 21564, + "networking": 21565, + "mindfulness": 21566, + "reborn": 21567, + "divider": 21568, + "popsicle": 21569, + "ruffles": 21570, + "subjects": 21571, + "muertos": 21572, + "gol": 21573, + "lite": 21574, + "ninth": 21575, + "##jav": 21576, + "shard": 21577, + "static": 21578, + "pours": 21579, + "revealing": 21580, + "ounces": 21581, + "proportions": 21582, + "monroe": 21583, + "mixes": 21584, + "trending": 21585, + "kem": 21586, + "yon": 21587, + "stake": 21588, + "mezz": 21589, + "crossroad": 21590, + "cafes": 21591, + "separately": 21592, + "pegasus": 21593, + "guadal": 21594, + "collins": 21595, + "religions": 21596, + "armadion": 21597, + "niche": 21598, + "##cular": 21599, + "clusters": 21600, + "soaps": 21601, + "excavator": 21602, + "kazakhstan": 21603, + "homepage": 21604, + "immigration": 21605, + "sassy": 21606, + "##lems": 21607, + "greenville": 21608, + "1911": 21609, + "suited": 21610, + "multif": 21611, + "environments": 21612, + "representatives": 21613, + "carcass": 21614, + "hornbill": 21615, + "theatrical": 21616, + "permission": 21617, + "theaters": 21618, + "drove": 21619, + "inventions": 21620, + "pistachio": 21621, + "##bol": 21622, + "##essa": 21623, + "fairfield": 21624, + "bounce": 21625, + "settlement": 21626, + "scooby": 21627, + "ling": 21628, + "exception": 21629, + "unloaded": 21630, + "tongued": 21631, + "jojo": 21632, + "supplier": 21633, + "sorting": 21634, + "evolved": 21635, + "##bach": 21636, + "hebrew": 21637, + "##pson": 21638, + "flapper": 21639, + "##itis": 21640, + "interaction": 21641, + "stacking": 21642, + "extravag": 21643, + "salvation": 21644, + "##83": 21645, + "evac": 21646, + "suitcases": 21647, + "belted": 21648, + "pianist": 21649, + "maison": 21650, + "honduras": 21651, + "sibling": 21652, + "sprout": 21653, + "goblin": 21654, + "fotos": 21655, + "subscription": 21656, + "dund": 21657, + "chords": 21658, + "syria": 21659, + "avalon": 21660, + "disconnect": 21661, + "5k": 21662, + "nusa": 21663, + "sene": 21664, + "##ean": 21665, + "coven": 21666, + "stripped": 21667, + "assignment": 21668, + "communicate": 21669, + "mississ": 21670, + "##ovo": 21671, + "pianos": 21672, + "tricera": 21673, + "aesthetics": 21674, + "gadget": 21675, + "fundraising": 21676, + "seychelles": 21677, + "dos": 21678, + "wisteria": 21679, + "##gia": 21680, + "byron": 21681, + "resemble": 21682, + "pete": 21683, + "plung": 21684, + "communion": 21685, + "freestyle": 21686, + "descendants": 21687, + "tanger": 21688, + "##ancer": 21689, + "ncr": 21690, + "cg": 21691, + "buffy": 21692, + "jolly": 21693, + "shan": 21694, + "eraser": 21695, + "powdered": 21696, + "demonstrate": 21697, + "gondord": 21698, + "loyalty": 21699, + "##creen": 21700, + "reduction": 21701, + "riverbank": 21702, + "antennas": 21703, + "vernon": 21704, + "humpback": 21705, + "##eering": 21706, + "##nal": 21707, + "staging": 21708, + "arrested": 21709, + "expanse": 21710, + "encouragement": 21711, + "nozzle": 21712, + "lightly": 21713, + "rounds": 21714, + "workbench": 21715, + "##uram": 21716, + "hangers": 21717, + "equine": 21718, + "blogs": 21719, + "citadel": 21720, + "perennials": 21721, + "dale": 21722, + "##anji": 21723, + "##otta": 21724, + "##igree": 21725, + "emblems": 21726, + "canton": 21727, + "pharmac": 21728, + "##ema": 21729, + "##ffee": 21730, + "willing": 21731, + "anger": 21732, + "ramb": 21733, + "conifer": 21734, + "headache": 21735, + "scrolls": 21736, + "awaits": 21737, + "hust": 21738, + "tian": 21739, + "booties": 21740, + "folklore": 21741, + "pruning": 21742, + "dainty": 21743, + "dm": 21744, + "rating": 21745, + "tattooed": 21746, + "chees": 21747, + "cappuccino": 21748, + "translu": 21749, + "expanding": 21750, + "mouses": 21751, + "sought": 21752, + "viz": 21753, + "desper": 21754, + "refers": 21755, + "passionate": 21756, + "dictionary": 21757, + "armani": 21758, + "rebound": 21759, + "disability": 21760, + "intercontinental": 21761, + "earphones": 21762, + "galact": 21763, + "newlyweds": 21764, + "mowing": 21765, + "##utt": 21766, + "teaser": 21767, + "harlem": 21768, + "televisions": 21769, + "invented": 21770, + "redevelop": 21771, + "unmade": 21772, + "hasbro": 21773, + "chatting": 21774, + "cajun": 21775, + "##ensis": 21776, + "corb": 21777, + "grasshoppers": 21778, + "tornado": 21779, + "##rate": 21780, + "soaking": 21781, + "##rika": 21782, + "immort": 21783, + "alumni": 21784, + "tunn": 21785, + "wight": 21786, + "pricing": 21787, + "grated": 21788, + "romper": 21789, + "bmx": 21790, + "bondi": 21791, + "dreadlocks": 21792, + "poncho": 21793, + "fragrant": 21794, + "nadu": 21795, + "chore": 21796, + "overlap": 21797, + "belonging": 21798, + "cuffs": 21799, + "pang": 21800, + "humour": 21801, + "##atra": 21802, + "wrestler": 21803, + "null": 21804, + "##icus": 21805, + "##igon": 21806, + "empress": 21807, + "lengths": 21808, + "pane": 21809, + "soak": 21810, + "cheering": 21811, + "feedback": 21812, + "##force": 21813, + "ventilation": 21814, + "gmc": 21815, + "portsmouth": 21816, + "##89": 21817, + "storms": 21818, + "fray": 21819, + "pave": 21820, + "membr": 21821, + "tongues": 21822, + "smashed": 21823, + "kennel": 21824, + "747": 21825, + "rhinoceros": 21826, + "parasaiing": 21827, + "planer": 21828, + "symbolizes": 21829, + "baseballs": 21830, + "sourd": 21831, + "cask": 21832, + "##eets": 21833, + "20s": 21834, + "heirl": 21835, + "sadness": 21836, + "stagger": 21837, + "jolie": 21838, + "fing": 21839, + "myers": 21840, + "tedd": 21841, + "compassion": 21842, + "ongoing": 21843, + "orion": 21844, + "##illon": 21845, + "1917": 21846, + "barking": 21847, + "capabilities": 21848, + "deathly": 21849, + "fauce": 21850, + "kebab": 21851, + "concerned": 21852, + "daffodils": 21853, + "aberdeen": 21854, + "##rell": 21855, + "waistband": 21856, + "##iga": 21857, + "concord": 21858, + "pomeal": 21859, + "kurt": 21860, + "m2": 21861, + "##etown": 21862, + "handing": 21863, + "digits": 21864, + "rubble": 21865, + "exchang": 21866, + "earliest": 21867, + "hague": 21868, + "rotterdam": 21869, + "##eville": 21870, + "chai": 21871, + "##roph": 21872, + "drib": 21873, + "elgin": 21874, + "##gygy": 21875, + "romanian": 21876, + "drizzled": 21877, + "telescopes": 21878, + "clap": 21879, + "sourdough": 21880, + "##arh": 21881, + "poking": 21882, + "digestive": 21883, + "mgm": 21884, + "bers": 21885, + "tension": 21886, + "##aris": 21887, + "##omo": 21888, + "##iri": 21889, + "trivia": 21890, + "##acus": 21891, + "kingfish": 21892, + "poseidon": 21893, + "skunks": 21894, + "ahmed": 21895, + "eso": 21896, + "tant": 21897, + "nearest": 21898, + "jod": 21899, + "infected": 21900, + "releasing": 21901, + "##grass": 21902, + "middleton": 21903, + "saff": 21904, + "wider": 21905, + "##arr": 21906, + "heartbeat": 21907, + "paddy": 21908, + "exclusively": 21909, + "sediment": 21910, + "##quisition": 21911, + "cuc": 21912, + "##vit": 21913, + "freezing": 21914, + "cerv": 21915, + "commemorative": 21916, + "##course": 21917, + "nachos": 21918, + "inspecting": 21919, + "##72": 21920, + "inlet": 21921, + "fres": 21922, + "fangs": 21923, + "conan": 21924, + "pluto": 21925, + "##iah": 21926, + "explores": 21927, + "sparks": 21928, + "mantisa": 21929, + "fragile": 21930, + "channels": 21931, + "ig": 21932, + "rup": 21933, + "tup": 21934, + "newton": 21935, + "acropolis": 21936, + "chickpea": 21937, + "collects": 21938, + "puffy": 21939, + "immune": 21940, + "blossoming": 21941, + "goblet": 21942, + "sto": 21943, + "petri": 21944, + "wordpress": 21945, + "estimate": 21946, + "seedlings": 21947, + "lithograph": 21948, + "bachelorette": 21949, + "beekeepers": 21950, + "##iments": 21951, + "waterhole": 21952, + "highlighting": 21953, + "##enever": 21954, + "cellist": 21955, + "owns": 21956, + "uncom": 21957, + "ritu": 21958, + "##nge": 21959, + "##anus": 21960, + "whenever": 21961, + "bac": 21962, + "ori": 21963, + "unre": 21964, + "cucumbers": 21965, + "preferred": 21966, + "alleged": 21967, + "translucent": 21968, + "sake": 21969, + "mekong": 21970, + "custara": 21971, + "primarily": 21972, + "attendant": 21973, + "1800s": 21974, + "wiccan": 21975, + "kum": 21976, + "wired": 21977, + "##52": 21978, + "logotype": 21979, + "locate": 21980, + "potent": 21981, + "lapel": 21982, + "statistics": 21983, + "gases": 21984, + "observed": 21985, + "influenced": 21986, + "##chang": 21987, + "argy": 21988, + "masai": 21989, + "propane": 21990, + "maintaining": 21991, + "penalty": 21992, + "kol": 21993, + "drones": 21994, + "pews": 21995, + "##atcher": 21996, + "ridiculous": 21997, + "##ovski": 21998, + "unbelievable": 21999, + "erie": 22000, + "##kow": 22001, + "vacant": 22002, + "inventory": 22003, + "fruity": 22004, + "##asha": 22005, + "compression": 22006, + "couns": 22007, + "saigon": 22008, + "cavalry": 22009, + "granddaugh": 22010, + "dow": 22011, + "medicines": 22012, + "surely": 22013, + "priority": 22014, + "##angelo": 22015, + "uber": 22016, + "##eong": 22017, + "toffee": 22018, + "manly": 22019, + "hooked": 22020, + "verb": 22021, + "fjord": 22022, + "hospitals": 22023, + "onyx": 22024, + "romans": 22025, + "devils": 22026, + "possibilities": 22027, + "simpson": 22028, + "nowad": 22029, + "situations": 22030, + "paulo": 22031, + "truffles": 22032, + "stance": 22033, + "##ondo": 22034, + "afl": 22035, + "murray": 22036, + "kirk": 22037, + "plague": 22038, + "stitched": 22039, + "calculate": 22040, + "inher": 22041, + "mittens": 22042, + "##busters": 22043, + "##apea": 22044, + "conquer": 22045, + "cutaway": 22046, + "intens": 22047, + "drummers": 22048, + "jumanji": 22049, + "clamp": 22050, + "antelos": 22051, + "ou": 22052, + "nev": 22053, + "canola": 22054, + "headpiece": 22055, + "macedonia": 22056, + "004": 22057, + "bernund": 22058, + "treadmill": 22059, + "unfortunately": 22060, + "clockwork": 22061, + "cp": 22062, + "neph": 22063, + "enlight": 22064, + "fitz": 22065, + "radisson": 22066, + "lexington": 22067, + "bedtime": 22068, + "##gg": 22069, + "flared": 22070, + "folks": 22071, + "seaguls": 22072, + "sacrifice": 22073, + "##icious": 22074, + "##emum": 22075, + "parquet": 22076, + "queenstown": 22077, + "nordstrom": 22078, + "recruitment": 22079, + "cz": 22080, + "##lc": 22081, + "shame": 22082, + "visually": 22083, + "augustine": 22084, + "focusing": 22085, + "lucia": 22086, + "gud": 22087, + "snout": 22088, + "supplement": 22089, + "##enchy": 22090, + "examine": 22091, + "veterin": 22092, + "astoria": 22093, + "khalifa": 22094, + "mb": 22095, + "tate": 22096, + "tata": 22097, + "##ulates": 22098, + "engaging": 22099, + "gamec": 22100, + "papa": 22101, + "hinduism": 22102, + "thorns": 22103, + "nowadays": 22104, + "vapor": 22105, + "knox": 22106, + "pastor": 22107, + "pigment": 22108, + "welcomed": 22109, + "hawthorn": 22110, + "papua": 22111, + "banknote": 22112, + "canes": 22113, + "conception": 22114, + "speck": 22115, + "crossfit": 22116, + "240": 22117, + "occupied": 22118, + "deployment": 22119, + "compartments": 22120, + "veggies": 22121, + "risen": 22122, + "swarovski": 22123, + "nocturn": 22124, + "cues": 22125, + "crypt": 22126, + "predict": 22127, + "dentistry": 22128, + "paddock": 22129, + "urchin": 22130, + "fraser": 22131, + "acme": 22132, + "##shell": 22133, + "distributed": 22134, + "givenchy": 22135, + "michelangelo": 22136, + "##apeake": 22137, + "gle": 22138, + "spinal": 22139, + "strato": 22140, + "wright": 22141, + "producers": 22142, + "begun": 22143, + "chesapeake": 22144, + "welf": 22145, + "toed": 22146, + "##roen": 22147, + "rockab": 22148, + "penang": 22149, + "##ddin": 22150, + "freck": 22151, + "baptist": 22152, + "mathematical": 22153, + "squeeze": 22154, + "##theon": 22155, + "artic": 22156, + "prism": 22157, + "lighters": 22158, + "##spring": 22159, + "racecourse": 22160, + "thompson": 22161, + "pheasant": 22162, + "balsamic": 22163, + "barclay": 22164, + "fuchsia": 22165, + "kot": 22166, + "madonna": 22167, + "skew": 22168, + "pasadena": 22169, + "##anthemum": 22170, + "##adi": 22171, + "backing": 22172, + "canis": 22173, + "protects": 22174, + "advisor": 22175, + "m3": 22176, + "##62": 22177, + "raindrops": 22178, + "##ahoo": 22179, + "sweetest": 22180, + "rhymes": 22181, + "discusses": 22182, + "bodice": 22183, + "seashells": 22184, + "clutter": 22185, + "recovered": 22186, + "invasion": 22187, + "telegraph": 22188, + "animaux": 22189, + "##anche": 22190, + "padding": 22191, + "rubbish": 22192, + "cobbl": 22193, + "vampires": 22194, + "brackets": 22195, + "aladdin": 22196, + "reinforced": 22197, + "frustrated": 22198, + "gand": 22199, + "hm": 22200, + "lh": 22201, + "nort": 22202, + "shenz": 22203, + "ample": 22204, + "medusa": 22205, + "karma": 22206, + "wes": 22207, + "chargers": 22208, + "informal": 22209, + "veranda": 22210, + "coyotes": 22211, + "admire": 22212, + "tapered": 22213, + "shrek": 22214, + "slovakia": 22215, + "uniqu": 22216, + "explan": 22217, + "005": 22218, + "astronomical": 22219, + "nerdy": 22220, + "lathe": 22221, + "##93": 22222, + "1912": 22223, + "1918": 22224, + "anatomical": 22225, + "armadile": 22226, + "pomegra": 22227, + "coleman": 22228, + "##ographical": 22229, + "1946": 22230, + "haze": 22231, + "andrews": 22232, + "schooner": 22233, + "azer": 22234, + "cis": 22235, + "rm": 22236, + "parlor": 22237, + "pristine": 22238, + "noticed": 22239, + "granada": 22240, + "resembles": 22241, + "##pots": 22242, + "skye": 22243, + "anchored": 22244, + "funding": 22245, + "translated": 22246, + "accepted": 22247, + "skeletal": 22248, + "##awan": 22249, + "capric": 22250, + "commuter": 22251, + "nostalgic": 22252, + "jubilee": 22253, + "eternity": 22254, + "rituals": 22255, + "eph": 22256, + "td": 22257, + "##rick": 22258, + "##icial": 22259, + "##cht": 22260, + "colleges": 22261, + "traditionally": 22262, + "hyena": 22263, + "platy": 22264, + "savage": 22265, + "positioned": 22266, + "lafayette": 22267, + "fm": 22268, + "##82": 22269, + "somebody": 22270, + "nemo": 22271, + "blackberries": 22272, + "bavarian": 22273, + "commitment": 22274, + "groove": 22275, + "kab": 22276, + "postal": 22277, + "postman": 22278, + "strongest": 22279, + "manufacture": 22280, + "kg": 22281, + "handled": 22282, + "hispan": 22283, + "pied": 22284, + "curs": 22285, + "geo": 22286, + "tubs": 22287, + "insider": 22288, + "mutant": 22289, + "prettiest": 22290, + "kah": 22291, + "##bai": 22292, + "newer": 22293, + "teas": 22294, + "barra": 22295, + "discs": 22296, + "tinted": 22297, + "tvs": 22298, + "ips": 22299, + "stocks": 22300, + "costco": 22301, + "##div": 22302, + "##eus": 22303, + "##most": 22304, + "robber": 22305, + "gujarat": 22306, + "zinc": 22307, + "madeira": 22308, + "dubrov": 22309, + "cheetar": 22310, + "akita": 22311, + "monopoly": 22312, + "wiki": 22313, + "##x9": 22314, + "harper": 22315, + "harder": 22316, + "typewriters": 22317, + "roundup": 22318, + "prisoner": 22319, + "georgetown": 22320, + "shingles": 22321, + "homeowners": 22322, + "surfboards": 22323, + "shave": 22324, + "artem": 22325, + "reducing": 22326, + "offensive": 22327, + "notch": 22328, + "amateur": 22329, + "luxem": 22330, + "bullets": 22331, + "malefic": 22332, + "collision": 22333, + "sharpe": 22334, + "slogans": 22335, + "##irling": 22336, + "blows": 22337, + "spreads": 22338, + "exhibitions": 22339, + "instantly": 22340, + "expansive": 22341, + "calcula": 22342, + "dedication": 22343, + "bt": 22344, + "inves": 22345, + "satan": 22346, + "glaci": 22347, + "nests": 22348, + "chrysanthemum": 22349, + "dapper": 22350, + "tonic": 22351, + "woodpe": 22352, + "avo": 22353, + "gros": 22354, + "renting": 22355, + "dubrovnik": 22356, + "pes": 22357, + "superc": 22358, + "rhod": 22359, + "28th": 22360, + "cameo": 22361, + "protests": 22362, + "totemal": 22363, + "jugs": 22364, + "widescreen": 22365, + "coe": 22366, + "moz": 22367, + "buenos": 22368, + "cellu": 22369, + "keen": 22370, + "##buck": 22371, + "##jiang": 22372, + "##lehem": 22373, + "syrac": 22374, + "amish": 22375, + "germ": 22376, + "limp": 22377, + "wilts": 22378, + "syracuse": 22379, + "spire": 22380, + "consp": 22381, + "asc": 22382, + "calming": 22383, + "earthworm": 22384, + "inked": 22385, + "scarec": 22386, + "3a": 22387, + "fenn": 22388, + "##udes": 22389, + "beta": 22390, + "researcher": 22391, + "orname": 22392, + "varying": 22393, + "heirloom": 22394, + "dips": 22395, + "##anium": 22396, + "chis": 22397, + "evangel": 22398, + "midlands": 22399, + "milestone": 22400, + "arrest": 22401, + "flashing": 22402, + "caricatures": 22403, + "bahrain": 22404, + "##enary": 22405, + "chel": 22406, + "handbook": 22407, + "suffered": 22408, + "bethlehem": 22409, + "scaffolding": 22410, + "hispanic": 22411, + "belgra": 22412, + "equival": 22413, + "plunge": 22414, + "aquarius": 22415, + "kiosk": 22416, + "rockabilly": 22417, + "bran": 22418, + "##ormal": 22419, + "splits": 22420, + "gazette": 22421, + "hyeno": 22422, + "chen": 22423, + "shooter": 22424, + "ricotta": 22425, + "hbo": 22426, + "neoc": 22427, + "batik": 22428, + "##ajo": 22429, + "ussr": 22430, + "expecting": 22431, + "candidates": 22432, + "650": 22433, + "extingu": 22434, + "chris": 22435, + "textbook": 22436, + "silverware": 22437, + "robotics": 22438, + "moisture": 22439, + "hemisphere": 22440, + "footpath": 22441, + "abstraction": 22442, + "24th": 22443, + "gumball": 22444, + "cooperation": 22445, + "##birth": 22446, + "cine": 22447, + "ecc": 22448, + "vign": 22449, + "##bla": 22450, + "longhorn": 22451, + "earthy": 22452, + "transforming": 22453, + "jacob": 22454, + "brie": 22455, + "paperwork": 22456, + "midfield": 22457, + "hick": 22458, + "foraging": 22459, + "buchar": 22460, + "##acha": 22461, + "oceanfront": 22462, + "pixels": 22463, + "hawaii": 22464, + "weasell": 22465, + "welfare": 22466, + "bucharest": 22467, + "hoops": 22468, + "twinkle": 22469, + "##illion": 22470, + "chariot": 22471, + "realm": 22472, + "bergen": 22473, + "suggests": 22474, + "bosnia": 22475, + "cara": 22476, + "##61": 22477, + "##eav": 22478, + "##estation": 22479, + "cabo": 22480, + "goodness": 22481, + "lottery": 22482, + "casio": 22483, + "210": 22484, + "cheerleaders": 22485, + "acquired": 22486, + "##kill": 22487, + "desi": 22488, + "haraj": 22489, + "installations": 22490, + "pendants": 22491, + "tunnels": 22492, + "investing": 22493, + "##53": 22494, + "barco": 22495, + "gov": 22496, + "emotic": 22497, + "mering": 22498, + "anemonia": 22499, + "ack": 22500, + "rend": 22501, + "enchan": 22502, + "modelling": 22503, + "bottled": 22504, + "obama": 22505, + "circulation": 22506, + "locking": 22507, + "3000": 22508, + "reddish": 22509, + "harajuku": 22510, + "rever": 22511, + "permit": 22512, + "characteristic": 22513, + "freshman": 22514, + "calla": 22515, + "capsules": 22516, + "giveaway": 22517, + "granddaughter": 22518, + "jute": 22519, + "nato": 22520, + "communist": 22521, + "directors": 22522, + "citroen": 22523, + "hurry": 22524, + "scraps": 22525, + "playoffs": 22526, + "choo": 22527, + "compress": 22528, + "ascend": 22529, + "designated": 22530, + "upsc": 22531, + "opi": 22532, + "indul": 22533, + "raining": 22534, + "palat": 22535, + "endeav": 22536, + "mantra": 22537, + "scorpions": 22538, + "papaya": 22539, + "icelandic": 22540, + "prestigious": 22541, + "mik": 22542, + "pall": 22543, + "scars": 22544, + "sepia": 22545, + "eyelash": 22546, + "terminator": 22547, + "mentor": 22548, + "plasma": 22549, + "redeemer": 22550, + "teddys": 22551, + "headlines": 22552, + "computing": 22553, + "italia": 22554, + "exhibits": 22555, + "pumping": 22556, + "wyoming": 22557, + "##emption": 22558, + "llamas": 22559, + "shenzhen": 22560, + "kon": 22561, + "tens": 22562, + "curling": 22563, + "##hampton": 22564, + "clams": 22565, + "22nd": 22566, + "aerosp": 22567, + "commissioner": 22568, + "argumentative": 22569, + "saves": 22570, + "##lot": 22571, + "petro": 22572, + "replic": 22573, + "escalator": 22574, + "interstate": 22575, + "shah": 22576, + "whites": 22577, + "exists": 22578, + "candys": 22579, + "rhyme": 22580, + "mastiff": 22581, + "mitch": 22582, + "##leigh": 22583, + "porcup": 22584, + "rehabilitation": 22585, + "wiltshire": 22586, + "zir": 22587, + "stabil": 22588, + "parc": 22589, + "woodwork": 22590, + "marion": 22591, + "unle": 22592, + "##teen": 22593, + "hardest": 22594, + "examined": 22595, + "steaming": 22596, + "##otle": 22597, + "corinth": 22598, + "1880": 22599, + "detachable": 22600, + "prefect": 22601, + "minnie": 22602, + "nitr": 22603, + "concealed": 22604, + "distinguished": 22605, + "rottweiler": 22606, + "550": 22607, + "bion": 22608, + "hil": 22609, + "headge": 22610, + "knuck": 22611, + "christening": 22612, + "specials": 22613, + "pitchers": 22614, + "involving": 22615, + "dare": 22616, + "chea": 22617, + "girlfriends": 22618, + "rollers": 22619, + "neverland": 22620, + "diabetic": 22621, + "housewife": 22622, + "meringue": 22623, + "enchanting": 22624, + "cog": 22625, + "hac": 22626, + "sz": 22627, + "##dra": 22628, + "deities": 22629, + "peep": 22630, + "offroad": 22631, + "valleys": 22632, + "excite": 22633, + "axy": 22634, + "elevate": 22635, + "pufferfish": 22636, + "vulner": 22637, + "luxembourg": 22638, + "gore": 22639, + "roland": 22640, + "spur": 22641, + "##aceous": 22642, + "sunscreen": 22643, + "mounts": 22644, + "batgirl": 22645, + "crawl": 22646, + "obsessed": 22647, + "blackout": 22648, + "operates": 22649, + "1943": 22650, + "untitled": 22651, + "colonies": 22652, + "jacqu": 22653, + "imposing": 22654, + "gb": 22655, + "nant": 22656, + "##gee": 22657, + "truss": 22658, + "1925": 22659, + "sideways": 22660, + "counted": 22661, + "##shop": 22662, + "##oyed": 22663, + "baku": 22664, + "seagulls": 22665, + "cobbler": 22666, + "2x": 22667, + "feline": 22668, + "##indle": 22669, + "armchairs": 22670, + "voyager": 22671, + "armenia": 22672, + "galvanized": 22673, + "colombo": 22674, + "cloths": 22675, + "physician": 22676, + "sophia": 22677, + "carne": 22678, + "cowl": 22679, + "freemas": 22680, + "olympus": 22681, + "journaling": 22682, + "folders": 22683, + "fiberglass": 22684, + "rudolph": 22685, + "##ao": 22686, + "##bard": 22687, + "ladders": 22688, + "horizons": 22689, + "thickness": 22690, + "defeat": 22691, + "##alous": 22692, + "associate": 22693, + "stewart": 22694, + "controversial": 22695, + "4a": 22696, + "##nar": 22697, + "##wald": 22698, + "ammon": 22699, + "navajo": 22700, + "pierced": 22701, + "colombian": 22702, + "##oys": 22703, + "pairing": 22704, + "lifes": 22705, + "jello": 22706, + "karts": 22707, + "rushing": 22708, + "tidy": 22709, + "amrits": 22710, + "palladium": 22711, + "azerbai": 22712, + "fought": 22713, + "oss": 22714, + "##uania": 22715, + "##bin": 22716, + "##km": 22717, + "como": 22718, + "checker": 22719, + "sonoma": 22720, + "separates": 22721, + "lithuania": 22722, + "amritsar": 22723, + "thirt": 22724, + "minus": 22725, + "thrift": 22726, + "throated": 22727, + "catama": 22728, + "##mal": 22729, + "##erne": 22730, + "sentences": 22731, + "bengali": 22732, + "dine": 22733, + "##stro": 22734, + "brighter": 22735, + "decid": 22736, + "ostrical": 22737, + "mortars": 22738, + "handicap": 22739, + "##rina": 22740, + "##reet": 22741, + "chased": 22742, + "antiqu": 22743, + "turin": 22744, + "halft": 22745, + "picker": 22746, + "renewable": 22747, + "storytelling": 22748, + "##estones": 22749, + "1915": 22750, + "inspector": 22751, + "ultron": 22752, + "integration": 22753, + "1080p": 22754, + "brixton": 22755, + "loo": 22756, + "waf": 22757, + "resulting": 22758, + "machu": 22759, + "solomon": 22760, + "trombones": 22761, + "nomine": 22762, + "intermedi": 22763, + "turner": 22764, + "coasters": 22765, + "sunda": 22766, + "appliques": 22767, + "improvements": 22768, + "cheerleading": 22769, + "piv": 22770, + "##roy": 22771, + "envy": 22772, + "cordur": 22773, + "verona": 22774, + "mustangs": 22775, + "sioux": 22776, + "bingo": 22777, + "berkshire": 22778, + "chir": 22779, + "##cracy": 22780, + "31st": 22781, + "maximize": 22782, + "bosch": 22783, + "emphas": 22784, + "refueling": 22785, + "##icable": 22786, + "ard": 22787, + "##asta": 22788, + "minif": 22789, + "warren": 22790, + "formulas": 22791, + "damsel": 22792, + "operational": 22793, + "pedic": 22794, + "maths": 22795, + "qualifying": 22796, + "superst": 22797, + "stratford": 22798, + "toucao": 22799, + "saffron": 22800, + "sai": 22801, + "disinf": 22802, + "tyler": 22803, + "##anca": 22804, + "mache": 22805, + "humid": 22806, + "segments": 22807, + "deciduous": 22808, + "bolog": 22809, + "loops": 22810, + "##finder": 22811, + "##ieg": 22812, + "deserve": 22813, + "marg": 22814, + "transplant": 22815, + "humorous": 22816, + "symmetrical": 22817, + "zamb": 22818, + "shui": 22819, + "smelling": 22820, + "suspend": 22821, + "wer": 22822, + "##ilis": 22823, + "boil": 22824, + "seaworld": 22825, + "famed": 22826, + "objectives": 22827, + "cockro": 22828, + "templar": 22829, + "intent": 22830, + "comparing": 22831, + "deeper": 22832, + "void": 22833, + "dept": 22834, + "reminis": 22835, + "baton": 22836, + "maori": 22837, + "larvae": 22838, + "gamecube": 22839, + "feng": 22840, + "glassware": 22841, + "gifted": 22842, + "oxfordshire": 22843, + "aerospace": 22844, + "eston": 22845, + "persia": 22846, + "stiff": 22847, + "holographic": 22848, + "enem": 22849, + "dramas": 22850, + "westie": 22851, + "albino": 22852, + "pyg": 22853, + "calligraphic": 22854, + "darts": 22855, + "appointment": 22856, + "maleficent": 22857, + "colts": 22858, + "dose": 22859, + "howard": 22860, + "michigan": 22861, + "uzbek": 22862, + "cds": 22863, + "mea": 22864, + "tet": 22865, + "hoses": 22866, + "waterway": 22867, + "##ographer": 22868, + "makeovers": 22869, + "firms": 22870, + "rains": 22871, + "echidus": 22872, + "muppets": 22873, + "conventional": 22874, + "uts": 22875, + "wicket": 22876, + "##osse": 22877, + "hologram": 22878, + "cheeseburg": 22879, + "cleared": 22880, + "focuses": 22881, + "weekender": 22882, + "colonel": 22883, + "tuk": 22884, + "chats": 22885, + "rox": 22886, + "processed": 22887, + "duster": 22888, + "chilling": 22889, + "bengaluru": 22890, + "1900s": 22891, + "configuration": 22892, + "bw": 22893, + "jag": 22894, + "jeeps": 22895, + "rn": 22896, + "##iw": 22897, + "##asian": 22898, + "geology": 22899, + "cruelty": 22900, + "rake": 22901, + "decent": 22902, + "mono": 22903, + "ourselves": 22904, + "cheetas": 22905, + "nec": 22906, + "hairs": 22907, + "agre": 22908, + "burma": 22909, + "feeders": 22910, + "stal": 22911, + "snowsh": 22912, + "commod": 22913, + "berg": 22914, + "suppliers": 22915, + "indeed": 22916, + "auctions": 22917, + "municipalities": 22918, + "corduroy": 22919, + "##gely": 22920, + "##loe": 22921, + "##cliff": 22922, + "polk": 22923, + "acryl": 22924, + "##wares": 22925, + "existence": 22926, + "honeycombs": 22927, + "sagitt": 22928, + "laurent": 22929, + "kos": 22930, + "fro": 22931, + "skyd": 22932, + "chicks": 22933, + "sorcer": 22934, + "cluttered": 22935, + "rarely": 22936, + "##ayan": 22937, + "##uman": 22938, + "mana": 22939, + "simon": 22940, + "basel": 22941, + "badgers": 22942, + "miu": 22943, + "mocking": 22944, + "mour": 22945, + "magick": 22946, + "seemed": 22947, + "##user": 22948, + "regatta": 22949, + "tomboy": 22950, + "journalist": 22951, + "pulse": 22952, + "assassins": 22953, + "siege": 22954, + "bruins": 22955, + "jav": 22956, + "armoire": 22957, + "milford": 22958, + "plumber": 22959, + "galley": 22960, + "ghostbusters": 22961, + "shamrock": 22962, + "kills": 22963, + "tiff": 22964, + "##die": 22965, + "##54": 22966, + "greene": 22967, + "airbrush": 22968, + "sculpting": 22969, + "spirited": 22970, + "malaga": 22971, + "dreads": 22972, + "astonishing": 22973, + "exor": 22974, + "treating": 22975, + "litre": 22976, + "pinch": 22977, + "ornately": 22978, + "guangzhou": 22979, + "snowboarder": 22980, + "obo": 22981, + "##oying": 22982, + "##cist": 22983, + "shout": 22984, + "outlets": 22985, + "gramm": 22986, + "appla": 22987, + "latter": 22988, + "##lord": 22989, + "##73": 22990, + "allah": 22991, + "##burgers": 22992, + "presenter": 22993, + "progressive": 22994, + "fiance": 22995, + "nbc": 22996, + "arist": 22997, + "purchasing": 22998, + "principle": 22999, + "##about": 23000, + "reb": 23001, + "wrapper": 23002, + "hexagonal": 23003, + "naper": 23004, + "sly": 23005, + "vt": 23006, + "waking": 23007, + "slick": 23008, + "goldfin": 23009, + "alternatives": 23010, + "neighborhoods": 23011, + "rapidly": 23012, + "grenad": 23013, + "fand": 23014, + "grouser": 23015, + "clems": 23016, + "##villea": 23017, + "ranging": 23018, + "gourd": 23019, + "jacquard": 23020, + "##71": 23021, + "##table": 23022, + "##uer": 23023, + "##rea": 23024, + "stub": 23025, + "##eti": 23026, + "##elong": 23027, + "shopper": 23028, + "stealing": 23029, + "timing": 23030, + "knotted": 23031, + "bape": 23032, + "bluebell": 23033, + "pubs": 23034, + "tapes": 23035, + "cheeks": 23036, + "prosperity": 23037, + "##pet": 23038, + "roaming": 23039, + "fibers": 23040, + "airfield": 23041, + "logan": 23042, + "##hama": 23043, + "ultras": 23044, + "tactics": 23045, + "constantly": 23046, + "mlb": 23047, + "cheeses": 23048, + "enemies": 23049, + "lique": 23050, + "chanc": 23051, + "donate": 23052, + "ethical": 23053, + "##idated": 23054, + "pantheon": 23055, + "electrician": 23056, + "philip": 23057, + "institution": 23058, + "bookshels": 23059, + "reportedly": 23060, + "chromos": 23061, + "clemson": 23062, + "hann": 23063, + "icc": 23064, + "cler": 23065, + "worcester": 23066, + "holistic": 23067, + "viewpoint": 23068, + "distort": 23069, + "galle": 23070, + "asteroid": 23071, + "upscale": 23072, + "cheaper": 23073, + "krem": 23074, + "xi": 23075, + "salis": 23076, + "##fficient": 23077, + "tribeca": 23078, + "a380": 23079, + "proportion": 23080, + "zircon": 23081, + "mma": 23082, + "##umps": 23083, + "bezel": 23084, + "modernist": 23085, + "portray": 23086, + "fuse": 23087, + "horrible": 23088, + "##pond": 23089, + "##84": 23090, + "offspring": 23091, + "ruled": 23092, + "originated": 23093, + "karat": 23094, + "dull": 23095, + "nour": 23096, + "vale": 23097, + "##eters": 23098, + "monso": 23099, + "traveled": 23100, + "productivity": 23101, + "caffe": 23102, + "lesser": 23103, + "picchu": 23104, + "voices": 23105, + "contribute": 23106, + "theorem": 23107, + "ces": 23108, + "hides": 23109, + "lax": 23110, + "ness": 23111, + "preston": 23112, + "1280": 23113, + "trouser": 23114, + "scaled": 23115, + "livery": 23116, + "greenhouses": 23117, + "swat": 23118, + "chicano": 23119, + "trippy": 23120, + "barbarian": 23121, + "pitt": 23122, + "26th": 23123, + "doughnuts": 23124, + "ganache": 23125, + "cavaliers": 23126, + "bookshelves": 23127, + "ninet": 23128, + "frontal": 23129, + "soups": 23130, + "achieved": 23131, + "cosmop": 23132, + "##leneck": 23133, + "naperville": 23134, + "wint": 23135, + "brune": 23136, + "aires": 23137, + "dinning": 23138, + "hilltop": 23139, + "borough": 23140, + "steaks": 23141, + "krakow": 23142, + "##lassical": 23143, + "777": 23144, + "cuck": 23145, + "yoko": 23146, + "theft": 23147, + "##icer": 23148, + "sovere": 23149, + "amend": 23150, + "appealing": 23151, + "stupa": 23152, + "hazelnut": 23153, + "pretend": 23154, + "sovereign": 23155, + "bluebird": 23156, + "graves": 23157, + "amounts": 23158, + "minivan": 23159, + "multiply": 23160, + "slabs": 23161, + "pilgrims": 23162, + "neoclassical": 23163, + "vil": 23164, + "andy": 23165, + "##lain": 23166, + "russell": 23167, + "taps": 23168, + "hounds": 23169, + "depiction": 23170, + "dp": 23171, + "##itarian": 23172, + "kinda": 23173, + "dolores": 23174, + "doughnut": 23175, + "##ilit": 23176, + "tempor": 23177, + "domed": 23178, + "periods": 23179, + "scanning": 23180, + "wheelbarrow": 23181, + "hence": 23182, + "tik": 23183, + "##link": 23184, + "##chy": 23185, + "shen": 23186, + "redemption": 23187, + "popping": 23188, + "shortage": 23189, + "fractions": 23190, + "fawn": 23191, + "##imm": 23192, + "traces": 23193, + "ops": 23194, + "copies": 23195, + "bottoms": 23196, + "primetime": 23197, + "mops": 23198, + "##ynth": 23199, + "welded": 23200, + "veterinarian": 23201, + "nano": 23202, + "xy": 23203, + "forty": 23204, + "largely": 23205, + "shelters": 23206, + "williamsburg": 23207, + "sb": 23208, + "teard": 23209, + "overweight": 23210, + "natal": 23211, + "minimalistic": 23212, + "orientation": 23213, + "amidst": 23214, + "lavish": 23215, + "prosth": 23216, + "cobblestone": 23217, + "##rail": 23218, + "traps": 23219, + "salus": 23220, + "towering": 23221, + "365": 23222, + "zu": 23223, + "##atre": 23224, + "##ando": 23225, + "orcas": 23226, + "beavers": 23227, + "##acher": 23228, + "decline": 23229, + "peterbilt": 23230, + "ipl": 23231, + "vood": 23232, + "##oopers": 23233, + "cabana": 23234, + "hammered": 23235, + "sequel": 23236, + "marigold": 23237, + "##aler": 23238, + "dino": 23239, + "statements": 23240, + "homer": 23241, + "geyses": 23242, + "demonstrators": 23243, + "skunky": 23244, + "lieuten": 23245, + "pisces": 23246, + "rebellion": 23247, + "voodoo": 23248, + "hasn": 23249, + "recap": 23250, + "##uffs": 23251, + "empor": 23252, + "breakers": 23253, + "locket": 23254, + "dealing": 23255, + "enthusiast": 23256, + "azerbaijan": 23257, + "crosswalk": 23258, + "nominated": 23259, + "nerves": 23260, + "cavalier": 23261, + "spatula": 23262, + "hacie": 23263, + "salisbury": 23264, + "##len": 23265, + "##chre": 23266, + "##urved": 23267, + "dres": 23268, + "expend": 23269, + "downton": 23270, + "sublim": 23271, + "##former": 23272, + "chambers": 23273, + "sprays": 23274, + "jellyfishs": 23275, + "omni": 23276, + "crossroads": 23277, + "tc": 23278, + "arter": 23279, + "##ummer": 23280, + "clan": 23281, + "abuse": 23282, + "supplements": 23283, + "urns": 23284, + "inserted": 23285, + "##ylvania": 23286, + "defenders": 23287, + "flawless": 23288, + "dials": 23289, + "kobe": 23290, + "lans": 23291, + "##hn": 23292, + "##reats": 23293, + "nightlife": 23294, + "wherever": 23295, + "horus": 23296, + "homeland": 23297, + "rackets": 23298, + "transforms": 23299, + "illuminate": 23300, + "swirls": 23301, + "commemorate": 23302, + "hints": 23303, + "hin": 23304, + "dragonflies": 23305, + "closes": 23306, + "consoles": 23307, + "kew": 23308, + "shorter": 23309, + "waffler": 23310, + "viruses": 23311, + "influential": 23312, + "qualities": 23313, + "antigua": 23314, + "yale": 23315, + "##mys": 23316, + "##jos": 23317, + "##chart": 23318, + "crocus": 23319, + "generate": 23320, + "intention": 23321, + "definitive": 23322, + "critics": 23323, + "cf": 23324, + "##hpur": 23325, + "##eted": 23326, + "fortun": 23327, + "luncheon": 23328, + "comforting": 23329, + "rebuilt": 23330, + "jenner": 23331, + "lug": 23332, + "##fal": 23333, + "##alds": 23334, + "##root": 23335, + "abant": 23336, + "flagpole": 23337, + "claimed": 23338, + "islamabad": 23339, + "tricer": 23340, + "midsummer": 23341, + "verd": 23342, + "##makers": 23343, + "##91": 23344, + "beets": 23345, + "patience": 23346, + "awe": 23347, + "##ipples": 23348, + "victoria": 23349, + "identifying": 23350, + "maus": 23351, + "006": 23352, + "008": 23353, + "voted": 23354, + "carrera": 23355, + "tandem": 23356, + "6s": 23357, + "prelim": 23358, + "avoca": 23359, + "brisk": 23360, + "artemis": 23361, + "cares": 23362, + "gated": 23363, + "r8": 23364, + "##hey": 23365, + "##mu": 23366, + "##vue": 23367, + "stoves": 23368, + "##imon": 23369, + "##kech": 23370, + "graf": 23371, + "stanford": 23372, + "consistent": 23373, + "marrakech": 23374, + "##anine": 23375, + "##iano": 23376, + "quiche": 23377, + "fees": 23378, + "potion": 23379, + "simulated": 23380, + "cartography": 23381, + "associates": 23382, + "barbershop": 23383, + "loos": 23384, + "##moor": 23385, + "scmp": 23386, + "1927": 23387, + "cleanse": 23388, + "classrooms": 23389, + "structured": 23390, + "restrictions": 23391, + "##oooooooo": 23392, + "negoti": 23393, + "guacamole": 23394, + "dd": 23395, + "##employ": 23396, + "prod": 23397, + "sunburst": 23398, + "fringes": 23399, + "fellowship": 23400, + "preservation": 23401, + "cervical": 23402, + "whol": 23403, + "##amous": 23404, + "spang": 23405, + "##aken": 23406, + "crooked": 23407, + "supergirl": 23408, + "jewelery": 23409, + "terriers": 23410, + "washes": 23411, + "bougain": 23412, + "glaciers": 23413, + "pere": 23414, + "##alle": 23415, + "pitching": 23416, + "offerings": 23417, + "inscribed": 23418, + "hanuk": 23419, + "hideaway": 23420, + "sore": 23421, + "##cous": 23422, + "chimes": 23423, + "##ople": 23424, + "womans": 23425, + "overlaid": 23426, + "115": 23427, + "paneled": 23428, + "molded": 23429, + "brunei": 23430, + "rai": 23431, + "belarus": 23432, + "avail": 23433, + "powerpuff": 23434, + "madame": 23435, + "correctly": 23436, + "evaluation": 23437, + "skateboarder": 23438, + "sous": 23439, + "##ugal": 23440, + "1933": 23441, + "announcing": 23442, + "causeway": 23443, + "dominated": 23444, + "tulum": 23445, + "##charge": 23446, + "algor": 23447, + "wording": 23448, + "unemploy": 23449, + "restless": 23450, + "teamed": 23451, + "dolby": 23452, + "velocity": 23453, + "equivalent": 23454, + "algorith": 23455, + "dx": 23456, + "##erm": 23457, + "inline": 23458, + "shear": 23459, + "bae": 23460, + "soothing": 23461, + "expressive": 23462, + "liberation": 23463, + "sophom": 23464, + "kernel": 23465, + "falk": 23466, + "##pipe": 23467, + "amur": 23468, + "repur": 23469, + "experi": 23470, + "frankenstein": 23471, + "expectations": 23472, + "revelation": 23473, + "##itters": 23474, + "babylon": 23475, + "pyong": 23476, + "ayurved": 23477, + "clinical": 23478, + "kha": 23479, + "##year": 23480, + "seized": 23481, + "mountainous": 23482, + "bonds": 23483, + "hacker": 23484, + "declaration": 23485, + "dives": 23486, + "mow": 23487, + "storyboard": 23488, + "tropics": 23489, + "billy": 23490, + "boud": 23491, + "nutritious": 23492, + "sangria": 23493, + "hai": 23494, + "##74": 23495, + "spade": 23496, + "asheville": 23497, + "recommendations": 23498, + "scarecrow": 23499, + "experiencing": 23500, + "##edin": 23501, + "blackpool": 23502, + "apparatus": 23503, + "congru": 23504, + "lepre": 23505, + "mx": 23506, + "##gard": 23507, + "rober": 23508, + "handc": 23509, + "##dley": 23510, + "##okes": 23511, + "clarin": 23512, + "mauve": 23513, + "obstacles": 23514, + "browser": 23515, + "hanukkah": 23516, + "##ilus": 23517, + "##atha": 23518, + "breakdown": 23519, + "methodist": 23520, + "peasant": 23521, + "alfred": 23522, + "tyrann": 23523, + "ns": 23524, + "##zza": 23525, + "bab": 23526, + "syrian": 23527, + "amu": 23528, + "##ologists": 23529, + "1934": 23530, + "shoveling": 23531, + "encouraging": 23532, + "goof": 23533, + "m4": 23534, + "atari": 23535, + "handout": 23536, + "remn": 23537, + "walkers": 23538, + "aqued": 23539, + "murphy": 23540, + "bellevue": 23541, + "correction": 23542, + "biceps": 23543, + "horm": 23544, + "lifeguard": 23545, + "porter": 23546, + "rpg": 23547, + "sears": 23548, + "phnom": 23549, + "northum": 23550, + "juices": 23551, + "attendance": 23552, + "bodys": 23553, + "multifunction": 23554, + "torl": 23555, + "chipotle": 23556, + "achieving": 23557, + "truffle": 23558, + "cosmopolitan": 23559, + "gav": 23560, + "##mun": 23561, + "sends": 23562, + "blim": 23563, + "logging": 23564, + "masonry": 23565, + "bacter": 23566, + "unfortunate": 23567, + "chipmunks": 23568, + "membrane": 23569, + "ahmedabad": 23570, + "rave": 23571, + "emu": 23572, + "salz": 23573, + "nicar": 23574, + "lieutenant": 23575, + "rin": 23576, + "sleds": 23577, + "##atched": 23578, + "strait": 23579, + "senator": 23580, + "##bie": 23581, + "stoke": 23582, + "loans": 23583, + "pathways": 23584, + "##ponding": 23585, + "observing": 23586, + "corresponding": 23587, + "pivot": 23588, + "vey": 23589, + "wanna": 23590, + "uptown": 23591, + "prose": 23592, + "clipper": 23593, + "27th": 23594, + "##oshock": 23595, + "revenue": 23596, + "mink": 23597, + "##pery": 23598, + "##ieves": 23599, + "lof": 23600, + "counts": 23601, + "##odon": 23602, + "hamburgers": 23603, + "1300": 23604, + "accepts": 23605, + "kingfisher": 23606, + "tipped": 23607, + "##uz": 23608, + "thunderst": 23609, + "##isers": 23610, + "##tings": 23611, + "midland": 23612, + "tempt": 23613, + "secured": 23614, + "intrig": 23615, + "##visor": 23616, + "5g": 23617, + "rf": 23618, + "##mund": 23619, + "highs": 23620, + "riga": 23621, + "judging": 23622, + "hershey": 23623, + "yokohama": 23624, + "binder": 23625, + "yelling": 23626, + "zanz": 23627, + "##orus": 23628, + "usaf": 23629, + "##lander": 23630, + "seabed": 23631, + "##backer": 23632, + "dunedin": 23633, + "atmospheric": 23634, + "sprinter": 23635, + "wipe": 23636, + "stellar": 23637, + "##ulin": 23638, + "grotto": 23639, + "dolom": 23640, + "excursion": 23641, + "intermediate": 23642, + "aux": 23643, + "outing": 23644, + "simba": 23645, + "americana": 23646, + "reputation": 23647, + "contempl": 23648, + "hydrop": 23649, + "artichoke": 23650, + "unveiling": 23651, + "maneu": 23652, + "inauguration": 23653, + "ufc": 23654, + "##ference": 23655, + "urs": 23656, + "maize": 23657, + "##children": 23658, + "cue": 23659, + "layering": 23660, + "notation": 23661, + "bioshock": 23662, + "assisted": 23663, + "##olithic": 23664, + "kare": 23665, + "colosse": 23666, + "cram": 23667, + "antio": 23668, + "careers": 23669, + "biard": 23670, + "snowmobilers": 23671, + "leprecha": 23672, + "vita": 23673, + "behold": 23674, + "detect": 23675, + "hades": 23676, + "slum": 23677, + "leprechaun": 23678, + "scans": 23679, + "midst": 23680, + "magnizer": 23681, + "continuing": 23682, + "thunderstorm": 23683, + "##mese": 23684, + "rainbows": 23685, + "gazelles": 23686, + "anteloe": 23687, + "etching": 23688, + "percentage": 23689, + "##nem": 23690, + "bailey": 23691, + "##ainted": 23692, + "emissions": 23693, + "102": 23694, + "magi": 23695, + "tangent": 23696, + "bip": 23697, + "nymph": 23698, + "vial": 23699, + "1916": 23700, + "##zzo": 23701, + "cartridges": 23702, + "estuary": 23703, + "tarts": 23704, + "haiti": 23705, + "hoist": 23706, + "rajasthani": 23707, + "wagons": 23708, + "##bly": 23709, + "firebird": 23710, + "turkeys": 23711, + "shouldn": 23712, + "unhealth": 23713, + "tamarind": 23714, + "reebok": 23715, + "glend": 23716, + "kok": 23717, + "ticks": 23718, + "ish": 23719, + "memoir": 23720, + "remake": 23721, + "vanities": 23722, + "pepperoni": 23723, + "ferretl": 23724, + "votes": 23725, + "childish": 23726, + "femme": 23727, + "lebanese": 23728, + "##ruption": 23729, + "abundant": 23730, + "lupus": 23731, + "unhealthy": 23732, + "mog": 23733, + "crimes": 23734, + "barista": 23735, + "apprent": 23736, + "pastels": 23737, + "violinist": 23738, + "rodents": 23739, + "tissues": 23740, + "tarantula": 23741, + "##alf": 23742, + "turt": 23743, + "juniper": 23744, + "exteriors": 23745, + "mussels": 23746, + "##rod": 23747, + "##rame": 23748, + "jealous": 23749, + "##structed": 23750, + "brightest": 23751, + "examines": 23752, + "okto": 23753, + "mopar": 23754, + "ethern": 23755, + "sys": 23756, + "topiary": 23757, + "threatened": 23758, + "quinn": 23759, + "bandit": 23760, + "irres": 23761, + "armadia": 23762, + "mules": 23763, + "jacobs": 23764, + "allure": 23765, + "3ds": 23766, + "harsh": 23767, + "petit": 23768, + "sensational": 23769, + "yonhap": 23770, + "aph": 23771, + "moles": 23772, + "glare": 23773, + "swoo": 23774, + "compilation": 23775, + "wrath": 23776, + "integ": 23777, + "kilometers": 23778, + "pedicure": 23779, + "pension": 23780, + "##51": 23781, + "pupil": 23782, + "armies": 23783, + "limbs": 23784, + "oliver": 23785, + "louds": 23786, + "polluted": 23787, + "brochures": 23788, + "attributes": 23789, + "confectionery": 23790, + "##imps": 23791, + "twine": 23792, + "shortcake": 23793, + "craftsmanship": 23794, + "woodpecker": 23795, + "bodysuit": 23796, + "oktober": 23797, + "vow": 23798, + "cling": 23799, + "imaging": 23800, + "automation": 23801, + "obtained": 23802, + "alchemist": 23803, + "excitement": 23804, + "fandom": 23805, + "kremlin": 23806, + "hak": 23807, + "jig": 23808, + "ovens": 23809, + "vall": 23810, + "deleg": 23811, + "skun": 23812, + "bled": 23813, + "flatbed": 23814, + "competitor": 23815, + "airlift": 23816, + "sherwin": 23817, + "ozark": 23818, + "caged": 23819, + "##rals": 23820, + "##fr": 23821, + "pled": 23822, + "crap": 23823, + "108": 23824, + "solst": 23825, + "matchbox": 23826, + "westerns": 23827, + "celtics": 23828, + "tabs": 23829, + "tatoo": 23830, + "wounded": 23831, + "exploding": 23832, + "kamp": 23833, + "##haven": 23834, + "##kw": 23835, + "##anne": 23836, + "##ibar": 23837, + "flax": 23838, + "endurance": 23839, + "galleria": 23840, + "drumming": 23841, + "paneling": 23842, + "normandy": 23843, + "azure": 23844, + "spoiler": 23845, + "##omile": 23846, + "##rand": 23847, + "roam": 23848, + "canning": 23849, + "potty": 23850, + "curd": 23851, + "transporter": 23852, + "banyan": 23853, + "crossbow": 23854, + "radios": 23855, + "risks": 23856, + "volcanoes": 23857, + "inhabitants": 23858, + "##bes": 23859, + "numbered": 23860, + "letterpress": 23861, + "2560": 23862, + "blending": 23863, + "bassist": 23864, + "scorpio": 23865, + "tricky": 23866, + "cavity": 23867, + "barcode": 23868, + "mines": 23869, + "##cro": 23870, + "visa": 23871, + "repet": 23872, + "symbolizing": 23873, + "districts": 23874, + "cairns": 23875, + "silicon": 23876, + "tunisia": 23877, + "##acca": 23878, + "grades": 23879, + "snowmen": 23880, + "castlevania": 23881, + "mornings": 23882, + "opted": 23883, + "risotto": 23884, + "visionary": 23885, + "np": 23886, + "api": 23887, + "grabbing": 23888, + "pinnacle": 23889, + "maf": 23890, + "£2": 23891, + "##urer": 23892, + "marb": 23893, + "nighttime": 23894, + "firetr": 23895, + "garlands": 23896, + "sentin": 23897, + "whitet": 23898, + "##charged": 23899, + "obviously": 23900, + "gratitude": 23901, + "monsoon": 23902, + "clarinets": 23903, + "integral": 23904, + "mates": 23905, + "##kers": 23906, + "##itative": 23907, + "inlaid": 23908, + "clare": 23909, + "worse": 23910, + "thankful": 23911, + "pomee": 23912, + "behaviour": 23913, + "videogames": 23914, + "zanzibar": 23915, + "iz": 23916, + "##imi": 23917, + "neither": 23918, + "shipw": 23919, + "webpage": 23920, + "mezzanine": 23921, + "minifig": 23922, + "teardrop": 23923, + "ethernet": 23924, + "hass": 23925, + "rx": 23926, + "abaya": 23927, + "endemic": 23928, + "victims": 23929, + "kaleid": 23930, + "attributed": 23931, + "cauldron": 23932, + "niece": 23933, + "zest": 23934, + "tulsa": 23935, + "muppet": 23936, + "deaf": 23937, + "dresden": 23938, + "bans": 23939, + "brent": 23940, + "antler": 23941, + "5000": 23942, + "pickups": 23943, + "texans": 23944, + "quantity": 23945, + "territories": 23946, + "sighting": 23947, + "tok": 23948, + "trooper": 23949, + "proved": 23950, + "coup": 23951, + "shoel": 23952, + "##riers": 23953, + "developers": 23954, + "pita": 23955, + "juniors": 23956, + "ripple": 23957, + "pyongyang": 23958, + "rab": 23959, + "##cious": 23960, + "seb": 23961, + "111": 23962, + "technological": 23963, + "tubing": 23964, + "carriers": 23965, + "geometrical": 23966, + "cinemat": 23967, + "purebred": 23968, + "##heed": 23969, + "scrambled": 23970, + "1024x768": 23971, + "rookie": 23972, + "oktoberfest": 23973, + "carter": 23974, + "##enza": 23975, + "chich": 23976, + "##ifes": 23977, + "glance": 23978, + "padlocks": 23979, + "dialog": 23980, + "lonel": 23981, + "hudd": 23982, + "shattered": 23983, + "spand": 23984, + "wearable": 23985, + "schne": 23986, + "##listed": 23987, + "inverted": 23988, + "bougainvillea": 23989, + "keyboards": 23990, + "anterior": 23991, + "apache": 23992, + "thresh": 23993, + "gotta": 23994, + "tagged": 23995, + "1905": 23996, + "gabri": 23997, + "innocent": 23998, + "##etric": 23999, + "belmont": 24000, + "facilit": 24001, + "borderlands": 24002, + "contractors": 24003, + "tasmanian": 24004, + "connectors": 24005, + "turtleneck": 24006, + "hes": 24007, + "sis": 24008, + "blake": 24009, + "##tebr": 24010, + "effectively": 24011, + "fearless": 24012, + "ventura": 24013, + "conducts": 24014, + "##icultural": 24015, + "polynesian": 24016, + "guilty": 24017, + "ripples": 24018, + "blair": 24019, + "##velle": 24020, + "acrob": 24021, + "partly": 24022, + "candlest": 24023, + "conservative": 24024, + "argentine": 24025, + "invasive": 24026, + "hickory": 24027, + "ost": 24028, + "rus": 24029, + "equally": 24030, + "cameron": 24031, + "outerwear": 24032, + "veyron": 24033, + "jin": 24034, + "##itian": 24035, + "##irical": 24036, + "cheng": 24037, + "notorious": 24038, + "terre": 24039, + "albu": 24040, + "bobby": 24041, + "describing": 24042, + "initially": 24043, + "streaks": 24044, + "unloading": 24045, + "hired": 24046, + "mia": 24047, + "ville": 24048, + "everton": 24049, + "stein": 24050, + "retract": 24051, + "presentations": 24052, + "discovering": 24053, + "canoeing": 24054, + "dominant": 24055, + "bs": 24056, + "yorkie": 24057, + "dubbed": 24058, + "streamers": 24059, + "virgo": 24060, + "##ints": 24061, + "uncover": 24062, + "hert": 24063, + "##active": 24064, + "burmese": 24065, + "millennial": 24066, + "pushes": 24067, + "luau": 24068, + "pentagon": 24069, + "perforated": 24070, + "conspir": 24071, + "painful": 24072, + "salrid": 24073, + "saute": 24074, + "asht": 24075, + "honeys": 24076, + "nucleus": 24077, + "phillips": 24078, + "yarra": 24079, + "carats": 24080, + "ober": 24081, + "wallflower": 24082, + "leans": 24083, + "particle": 24084, + "##oku": 24085, + "midfielder": 24086, + "grammys": 24087, + "sophomore": 24088, + "gd": 24089, + "##yad": 24090, + "##wig": 24091, + "##amorph": 24092, + "pictorial": 24093, + "stranded": 24094, + "hamper": 24095, + "magnum": 24096, + "ordering": 24097, + "individually": 24098, + "acorna": 24099, + "strengthen": 24100, + "spilled": 24101, + "##erno": 24102, + "sandhill": 24103, + "warms": 24104, + "rigid": 24105, + "snapped": 24106, + "johns": 24107, + "lankan": 24108, + "1909": 24109, + "astros": 24110, + "knocked": 24111, + "correspond": 24112, + "hustle": 24113, + "mausoleum": 24114, + "5t": 24115, + "gathers": 24116, + "sereng": 24117, + "croquets": 24118, + "##rette": 24119, + "physique": 24120, + "1908": 24121, + "gloucestershire": 24122, + "overwhelmed": 24123, + "serengeti": 24124, + "##hed": 24125, + "floss": 24126, + "transformer": 24127, + "realized": 24128, + "deadl": 24129, + "vii": 24130, + "collages": 24131, + "dissol": 24132, + "bj": 24133, + "fs": 24134, + "tb": 24135, + "£5": 24136, + "##reck": 24137, + "shiba": 24138, + "mend": 24139, + "packaged": 24140, + "latvia": 24141, + "politician": 24142, + "bordeaux": 24143, + "##fields": 24144, + "anzac": 24145, + "sarcastic": 24146, + "glee": 24147, + "mph": 24148, + "mae": 24149, + "baub": 24150, + "##abre": 24151, + "prese": 24152, + "##quir": 24153, + "classified": 24154, + "baggage": 24155, + "removes": 24156, + "flagstone": 24157, + "lawns": 24158, + "probability": 24159, + "harmful": 24160, + "wilmington": 24161, + "exhausted": 24162, + "concentration": 24163, + "belgrade": 24164, + "a1": 24165, + "dys": 24166, + "##utional": 24167, + "synag": 24168, + "bursting": 24169, + "leftovers": 24170, + "colosseum": 24171, + "##kt": 24172, + "winston": 24173, + "finely": 24174, + "motherhood": 24175, + "intake": 24176, + "tropic": 24177, + "reasonable": 24178, + "chimph": 24179, + "##romatic": 24180, + "cascades": 24181, + "citizenship": 24182, + "advocate": 24183, + "wrinkles": 24184, + "unleash": 24185, + "toothbrushes": 24186, + "hv": 24187, + "hikes": 24188, + "maned": 24189, + "peering": 24190, + "##ereal": 24191, + "serif": 24192, + "owning": 24193, + "keyhole": 24194, + "thrive": 24195, + "##keeping": 24196, + "fleur": 24197, + "scratched": 24198, + "grizzlies": 24199, + "erected": 24200, + "appalach": 24201, + "covenant": 24202, + "v6": 24203, + "glaston": 24204, + "lapse": 24205, + "reggae": 24206, + "typeface": 24207, + "intact": 24208, + "tortillas": 24209, + "merchandising": 24210, + "prohibited": 24211, + "isaac": 24212, + "##etto": 24213, + "##ilda": 24214, + "waxed": 24215, + "##querque": 24216, + "garnet": 24217, + "dumbbells": 24218, + "nicholas": 24219, + "loudsp": 24220, + "aimed": 24221, + "##kg": 24222, + "transporting": 24223, + "volumes": 24224, + "radical": 24225, + "slimming": 24226, + "lyon": 24227, + "miser": 24228, + "wok": 24229, + "##laim": 24230, + "aroma": 24231, + "checkout": 24232, + "revamp": 24233, + "latitude": 24234, + "fedora": 24235, + "sewn": 24236, + "recreate": 24237, + "declared": 24238, + "albuquerque": 24239, + "##iton": 24240, + "##isa": 24241, + "cabb": 24242, + "sourced": 24243, + "brotherhood": 24244, + "gained": 24245, + "explosive": 24246, + "invertebr": 24247, + "solstice": 24248, + "plating": 24249, + "unable": 24250, + "helix": 24251, + "strands": 24252, + "walkways": 24253, + "##what": 24254, + "montage": 24255, + "##otypes": 24256, + "oregon": 24257, + "##irk": 24258, + "##unci": 24259, + "grips": 24260, + "boxers": 24261, + "106": 24262, + "epiph": 24263, + "enchil": 24264, + "increasingly": 24265, + "cylinders": 24266, + "cens": 24267, + "sothe": 24268, + "vents": 24269, + "glide": 24270, + "abilities": 24271, + "penh": 24272, + "annex": 24273, + "spraw": 24274, + "stagus": 24275, + "consultation": 24276, + "kernels": 24277, + "adventurous": 24278, + "nitrogen": 24279, + "counsel": 24280, + "sprig": 24281, + "weeping": 24282, + "breaker": 24283, + "taller": 24284, + "agenda": 24285, + "headgear": 24286, + "uzbekistan": 24287, + "sotheby": 24288, + "a5": 24289, + "sally": 24290, + "##ews": 24291, + "##rine": 24292, + "##ulatory": 24293, + "conve": 24294, + "algar": 24295, + "amman": 24296, + "swept": 24297, + "piglet": 24298, + "navigate": 24299, + "robinson": 24300, + "targets": 24301, + "##nemouth": 24302, + "##yers": 24303, + "transmit": 24304, + "sanit": 24305, + "hostess": 24306, + "##estrial": 24307, + "suspect": 24308, + "pretzes": 24309, + "burritos": 24310, + "kraken": 24311, + "derbyshire": 24312, + "scalloped": 24313, + "bam": 24314, + "##ignon": 24315, + "seo": 24316, + "prox": 24317, + "##afe": 24318, + "eyelashes": 24319, + "bleeding": 24320, + "criter": 24321, + "singular": 24322, + "garg": 24323, + "ethereal": 24324, + "oriented": 24325, + "mitchell": 24326, + "##note": 24327, + "promises": 24328, + "moral": 24329, + "hatching": 24330, + "handicra": 24331, + "coventry": 24332, + "tyrannosaurus": 24333, + "glastonbury": 24334, + "fx": 24335, + "kle": 24336, + "plex": 24337, + "##eers": 24338, + "##rants": 24339, + "stow": 24340, + "somewhat": 24341, + "clas": 24342, + "fronts": 24343, + "noon": 24344, + "teapots": 24345, + "filed": 24346, + "wheeler": 24347, + "saras": 24348, + "addressing": 24349, + "preseason": 24350, + "shea": 24351, + "babys": 24352, + "emmys": 24353, + "sailer": 24354, + "separation": 24355, + "plugs": 24356, + "reconna": 24357, + "chesterfield": 24358, + "hartford": 24359, + "croissant": 24360, + "grit": 24361, + "foundry": 24362, + "bakers": 24363, + "slaugh": 24364, + "cancelled": 24365, + "diablo": 24366, + "##kit": 24367, + "raisin": 24368, + "congruent": 24369, + "fb": 24370, + "pg": 24371, + "tae": 24372, + "chut": 24373, + "compl": 24374, + "slicing": 24375, + "bradley": 24376, + "filler": 24377, + "steakhouse": 24378, + "balmain": 24379, + "hinged": 24380, + "steady": 24381, + "acknow": 24382, + "caca": 24383, + "##hr": 24384, + "##wd": 24385, + "fabrication": 24386, + "##iyaki": 24387, + "hacienda": 24388, + "reconnaissance": 24389, + "5d": 24390, + "prong": 24391, + "doe": 24392, + "macy": 24393, + "imagin": 24394, + "waitress": 24395, + "refugee": 24396, + "svg": 24397, + "ganesha": 24398, + "andalus": 24399, + "capricorn": 24400, + "zo": 24401, + "fred": 24402, + "perception": 24403, + "edison": 24404, + "bloggers": 24405, + "inserts": 24406, + "opposition": 24407, + "biod": 24408, + "succulents": 24409, + "thieves": 24410, + "roar": 24411, + "embry": 24412, + "sharpie": 24413, + "affects": 24414, + "510x5": 24415, + "lilly": 24416, + "##59": 24417, + "enrich": 24418, + "uni": 24419, + "decom": 24420, + "cori": 24421, + "earthqu": 24422, + "galway": 24423, + "participant": 24424, + "economics": 24425, + "substitute": 24426, + "##unciation": 24427, + "managers": 24428, + "pliers": 24429, + "tek": 24430, + "merlin": 24431, + "proto": 24432, + "sellers": 24433, + "armadin": 24434, + "overlays": 24435, + "jimmy": 24436, + "renewal": 24437, + "frolicing": 24438, + "zambia": 24439, + "oboe": 24440, + "pf": 24441, + "anark": 24442, + "##abl": 24443, + "lockheed": 24444, + "dividing": 24445, + "succeed": 24446, + "cabbages": 24447, + "pne": 24448, + "##oud": 24449, + "staircases": 24450, + "parasol": 24451, + "karaoke": 24452, + "sue": 24453, + "##zee": 24454, + "toasting": 24455, + "coz": 24456, + "imax": 24457, + "tracing": 24458, + "trauma": 24459, + "posting": 24460, + "garages": 24461, + "mandalay": 24462, + "sparkly": 24463, + "greeted": 24464, + "whitney": 24465, + "variables": 24466, + "sideshow": 24467, + "urchins": 24468, + "sampling": 24469, + "patricks": 24470, + "sik": 24471, + "vu": 24472, + "##igans": 24473, + "##luded": 24474, + "uncon": 24475, + "overha": 24476, + "1913": 24477, + "doi": 24478, + "##plate": 24479, + "hangover": 24480, + "anglo": 24481, + "##linburg": 24482, + "ciner": 24483, + "specifications": 24484, + "chamelon": 24485, + "echidna": 24486, + "mentioned": 24487, + "prefecture": 24488, + "conspiracy": 24489, + "gable": 24490, + "##jour": 24491, + "barren": 24492, + "batting": 24493, + "embank": 24494, + "farrow": 24495, + "rooftops": 24496, + "bounty": 24497, + "reminiscent": 24498, + "fath": 24499, + "£3": 24500, + "bagh": 24501, + "##eez": 24502, + "snowstorm": 24503, + "230": 24504, + "vertically": 24505, + "conversations": 24506, + "staggering": 24507, + "anarkali": 24508, + "onboard": 24509, + "parth": 24510, + "patrons": 24511, + "cheat": 24512, + "occasional": 24513, + "marbella": 24514, + "benn": 24515, + "##orca": 24516, + "prick": 24517, + "beauties": 24518, + "##ounding": 24519, + "superv": 24520, + "geranium": 24521, + "genes": 24522, + "ergon": 24523, + "sherpa": 24524, + "collapsed": 24525, + "smurfs": 24526, + "##ewan": 24527, + "indus": 24528, + "gameplay": 24529, + "helium": 24530, + "slums": 24531, + "interviews": 24532, + "caddy": 24533, + "gloomy": 24534, + "depar": 24535, + "hyac": 24536, + "brandy": 24537, + "thermost": 24538, + "pontoon": 24539, + "fertility": 24540, + "shingle": 24541, + "exorcist": 24542, + "fuss": 24543, + "woo": 24544, + "##olor": 24545, + "##angeas": 24546, + "willows": 24547, + "squarepants": 24548, + "operators": 24549, + "planetary": 24550, + "ownership": 24551, + "hydrangeas": 24552, + "responsive": 24553, + "gables": 24554, + "jake": 24555, + "pp": 24556, + "##block": 24557, + "remas": 24558, + "scoops": 24559, + "infamous": 24560, + "richest": 24561, + "nautilus": 24562, + "counseling": 24563, + "synagogue": 24564, + "cared": 24565, + "##hal": 24566, + "##gues": 24567, + "##lux": 24568, + "plaques": 24569, + "corpus": 24570, + "##metal": 24571, + "macabre": 24572, + "virtually": 24573, + "tidal": 24574, + "morm": 24575, + "##agram": 24576, + "cutters": 24577, + "croc": 24578, + "inspects": 24579, + "inflam": 24580, + "eggn": 24581, + "samui": 24582, + "murano": 24583, + "felix": 24584, + "amphitheatre": 24585, + "vein": 24586, + "angular": 24587, + "ibm": 24588, + "tombstone": 24589, + "committed": 24590, + "remastered": 24591, + "lashes": 24592, + "chub": 24593, + "botany": 24594, + "##ahan": 24595, + "functionality": 24596, + "consequ": 24597, + "##aceae": 24598, + "hale": 24599, + "yok": 24600, + "spans": 24601, + "trafal": 24602, + "openers": 24603, + "logger": 24604, + "remained": 24605, + "executed": 24606, + "kell": 24607, + "##hma": 24608, + "##inky": 24609, + "##ophone": 24610, + "smoker": 24611, + "kurti": 24612, + "northumberland": 24613, + "v2": 24614, + "persuas": 24615, + "illustrates": 24616, + "reclining": 24617, + "accum": 24618, + "brainstorm": 24619, + "glittering": 24620, + "##iserie": 24621, + "cacti": 24622, + "interacting": 24623, + "thunderbird": 24624, + "neatly": 24625, + "lomb": 24626, + "piper": 24627, + "##x16": 24628, + "boca": 24629, + "majesty": 24630, + "pierce": 24631, + "esk": 24632, + "lentils": 24633, + "yankee": 24634, + "kombat": 24635, + "jodhpur": 24636, + "510x510": 24637, + "trafalgar": 24638, + "##coun": 24639, + "##atr": 24640, + "sta": 24641, + "swear": 24642, + "airstream": 24643, + "visor": 24644, + "gallop": 24645, + "2500": 24646, + "mallet": 24647, + "pitched": 24648, + "fulfill": 24649, + "kermit": 24650, + "monochromatic": 24651, + "crutt": 24652, + "disabilities": 24653, + "goldfinch": 24654, + "aqueduct": 24655, + "cacao": 24656, + "yuan": 24657, + "##enham": 24658, + "shank": 24659, + "blame": 24660, + "##atchewan": 24661, + "starred": 24662, + "chinoiserie": 24663, + "saskatchewan": 24664, + "busses": 24665, + "ike": 24666, + "##racan": 24667, + "holsters": 24668, + "ballot": 24669, + "potting": 24670, + "activist": 24671, + "tarzan": 24672, + "gibral": 24673, + "lago": 24674, + "##hoe": 24675, + "##alaya": 24676, + "rebirth": 24677, + "topshop": 24678, + "attracted": 24679, + "joystick": 24680, + "dialogue": 24681, + "consideration": 24682, + "significance": 24683, + "reporting": 24684, + "gloucester": 24685, + "preliminary": 24686, + "cpu": 24687, + "fis": 24688, + "rainier": 24689, + "teriyaki": 24690, + "additions": 24691, + "professionally": 24692, + "##onnaise": 24693, + "crumpled": 24694, + "speckled": 24695, + "##oten": 24696, + "manuel": 24697, + "radha": 24698, + "specializes": 24699, + "ranks": 24700, + "nicaragua": 24701, + "cuddle": 24702, + "jose": 24703, + "anub": 24704, + "##ceeds": 24705, + "bays": 24706, + "mys": 24707, + "presley": 24708, + "harps": 24709, + "capitals": 24710, + "infiniti": 24711, + "hinge": 24712, + "gag": 24713, + "heap": 24714, + "ym": 24715, + "##esc": 24716, + "waterford": 24717, + "skilled": 24718, + "festiv": 24719, + "stormtr": 24720, + "pierre": 24721, + "wyn": 24722, + "caviar": 24723, + "haunting": 24724, + "armenian": 24725, + "gta": 24726, + "##pat": 24727, + "##xide": 24728, + "clinton": 24729, + "woody": 24730, + "skylights": 24731, + "##itsu": 24732, + "shipyard": 24733, + "donations": 24734, + "illumination": 24735, + "trusted": 24736, + "iraqi": 24737, + "glitch": 24738, + "riverfront": 24739, + "necktie": 24740, + "##inois": 24741, + "melody": 24742, + "splatter": 24743, + "affir": 24744, + "315": 24745, + "beneficial": 24746, + "snapshot": 24747, + "gooey": 24748, + "ruffled": 24749, + "carnegie": 24750, + "kaleidoscope": 24751, + "chutney": 24752, + "gaga": 24753, + "jp": 24754, + "##organ": 24755, + "##icon": 24756, + "reen": 24757, + "ladybugs": 24758, + "inflated": 24759, + "hyenian": 24760, + "coordinate": 24761, + "spoken": 24762, + "heathrow": 24763, + "cus": 24764, + "lacy": 24765, + "##hips": 24766, + "chilly": 24767, + "poins": 24768, + "unm": 24769, + "recliner": 24770, + "treasury": 24771, + "overcome": 24772, + "kardashian": 24773, + "pneum": 24774, + "1926": 24775, + "boxwood": 24776, + "commuters": 24777, + "preser": 24778, + "palaces": 24779, + "facel": 24780, + "founding": 24781, + "simplified": 24782, + "yawning": 24783, + "throt": 24784, + "##cho": 24785, + "frugal": 24786, + "alham": 24787, + "moat": 24788, + "harle": 24789, + "cellos": 24790, + "melan": 24791, + "deeply": 24792, + "##rability": 24793, + "retrievers": 24794, + "75th": 24795, + "dismant": 24796, + "sublimation": 24797, + "##gov": 24798, + "##oglyph": 24799, + "baron": 24800, + "spotting": 24801, + "steamer": 24802, + "cyborg": 24803, + "embraces": 24804, + "parasite": 24805, + "flavours": 24806, + "handkerchiefs": 24807, + "waikiki": 24808, + "trafford": 24809, + "pestic": 24810, + "##inus": 24811, + "##ained": 24812, + "reich": 24813, + "chevelle": 24814, + "##shan": 24815, + "mayonnaise": 24816, + "activists": 24817, + "catalina": 24818, + "aloha": 24819, + "gibraltar": 24820, + "anubis": 24821, + "bce": 24822, + "threw": 24823, + "stet": 24824, + "greys": 24825, + "##athing": 24826, + "equator": 24827, + "deadline": 24828, + "scarborough": 24829, + "##while": 24830, + "deaths": 24831, + "impressions": 24832, + "commanding": 24833, + "mesopotam": 24834, + "paso": 24835, + "calcium": 24836, + "comprehension": 24837, + "surprisingly": 24838, + "phenomen": 24839, + "satisfying": 24840, + "##eday": 24841, + "##kees": 24842, + "buys": 24843, + "classification": 24844, + "warship": 24845, + "motorola": 24846, + "shipped": 24847, + "casing": 24848, + "specialists": 24849, + "##inox": 24850, + "nurseries": 24851, + "captivating": 24852, + "pharmacist": 24853, + "redevelopment": 24854, + "2a": 24855, + "isot": 24856, + "##ollen": 24857, + "tallinn": 24858, + "hairdry": 24859, + "tinkerbell": 24860, + "barclays": 24861, + "##xi": 24862, + "29th": 24863, + "multiplic": 24864, + "hauling": 24865, + "cuckoo": 24866, + "ks": 24867, + "doves": 24868, + "modeled": 24869, + "hoodies": 24870, + "##ippi": 24871, + "loneliness": 24872, + "##rider": 24873, + "knifes": 24874, + "longines": 24875, + "westfield": 24876, + "admiring": 24877, + "consecut": 24878, + "##ingly": 24879, + "idi": 24880, + "discard": 24881, + "sloty": 24882, + "numerals": 24883, + "tangerine": 24884, + "011": 24885, + "##bal": 24886, + "##irah": 24887, + "##onte": 24888, + "marijuana": 24889, + "barriers": 24890, + "supra": 24891, + "anytime": 24892, + "accounts": 24893, + "subl": 24894, + "langley": 24895, + "optic": 24896, + "flipping": 24897, + "poodler": 24898, + "cany": 24899, + "interlock": 24900, + "flamen": 24901, + "organizers": 24902, + "cleaners": 24903, + "futon": 24904, + "pullover": 24905, + "apostles": 24906, + "flamenco": 24907, + "citation": 24908, + "naked": 24909, + "rational": 24910, + "flares": 24911, + "urant": 24912, + "grills": 24913, + "incorporates": 24914, + "skipper": 24915, + "opponent": 24916, + "aver": 24917, + "footh": 24918, + "watchers": 24919, + "regulations": 24920, + "civilian": 24921, + "embankment": 24922, + "stakes": 24923, + "lov": 24924, + "##achal": 24925, + "##berley": 24926, + "##auk": 24927, + "studied": 24928, + "huracan": 24929, + "##certain": 24930, + "barracks": 24931, + "##fold": 24932, + "isab": 24933, + "adri": 24934, + "intensive": 24935, + "##country": 24936, + "spoonful": 24937, + "daw": 24938, + "##aps": 24939, + "##isserie": 24940, + "belved": 24941, + "approval": 24942, + "wwf": 24943, + "kameez": 24944, + "noses": 24945, + "hieroglyph": 24946, + "aria": 24947, + "alvin": 24948, + "moines": 24949, + "postp": 24950, + "barrington": 24951, + "eurasian": 24952, + "assists": 24953, + "mindset": 24954, + "toasters": 24955, + "belvedere": 24956, + "bowler": 24957, + "tis": 24958, + "headline": 24959, + "fuj": 24960, + "tao": 24961, + "fixer": 24962, + "contours": 24963, + "goldfishs": 24964, + "mutual": 24965, + "canyons": 24966, + "010": 24967, + "##81": 24968, + "desol": 24969, + "bucc": 24970, + "grange": 24971, + "noted": 24972, + "descending": 24973, + "democracy": 24974, + "beekeeping": 24975, + "platters": 24976, + "meridian": 24977, + "yahoo": 24978, + "##card": 24979, + "alger": 24980, + "alga": 24981, + "foal": 24982, + "archie": 24983, + "goodwood": 24984, + "farmall": 24985, + "stairwell": 24986, + "fewer": 24987, + "##actions": 24988, + "isla": 24989, + "cox": 24990, + "derel": 24991, + "suffer": 24992, + "mimic": 24993, + "gatlinburg": 24994, + "trilopod": 24995, + "eos": 24996, + "##vic": 24997, + "##edict": 24998, + "stopper": 24999, + "sarat": 25000, + "defeating": 25001, + "piazza": 25002, + "damp": 25003, + "##arth": 25004, + "##anza": 25005, + "roe": 25006, + "backside": 25007, + "exeter": 25008, + "##elfish": 25009, + "milling": 25010, + "##osphere": 25011, + "cascading": 25012, + "handcuffs": 25013, + "algarve": 25014, + "##govina": 25015, + "##xter": 25016, + "##frame": 25017, + "##orant": 25018, + "drapery": 25019, + "roundhouse": 25020, + "apprentice": 25021, + "throttle": 25022, + "saratoga": 25023, + "cen": 25024, + "##hun": 25025, + "induction": 25026, + "valid": 25027, + "pacifier": 25028, + "hobart": 25029, + "esot": 25030, + "astra": 25031, + "spiked": 25032, + "cellular": 25033, + "chlor": 25034, + "##adas": 25035, + "meow": 25036, + "##ologie": 25037, + "sportsman": 25038, + "##rett": 25039, + "durability": 25040, + "fatal": 25041, + "awaiting": 25042, + "alamo": 25043, + "frightening": 25044, + "suddenly": 25045, + "mojito": 25046, + "suspenders": 25047, + "fading": 25048, + "ux": 25049, + "##cepting": 25050, + "quaint": 25051, + "percy": 25052, + "runes": 25053, + "sensation": 25054, + "johnny": 25055, + "lettues": 25056, + "iguaa": 25057, + "estonia": 25058, + "amendment": 25059, + "©20": 25060, + "##how": 25061, + "##dder": 25062, + "##oia": 25063, + "marbles": 25064, + "curzon": 25065, + "cyan": 25066, + "terrific": 25067, + "booster": 25068, + "snorking": 25069, + "sunbea": 25070, + "condensed": 25071, + "freemason": 25072, + "goofy": 25073, + "esoteric": 25074, + "carls": 25075, + "leds": 25076, + "vill": 25077, + "##arney": 25078, + "##utical": 25079, + "lettered": 25080, + "hugo": 25081, + "astrological": 25082, + "clarity": 25083, + "##casting": 25084, + "##ogenic": 25085, + "bard": 25086, + "rb": 25087, + "rash": 25088, + "##cose": 25089, + "frigate": 25090, + "rovers": 25091, + "blackhawks": 25092, + "ade": 25093, + "amc": 25094, + "angelfish": 25095, + "##aji": 25096, + "spirals": 25097, + "phenomenon": 25098, + "quintess": 25099, + "acceler": 25100, + "twill": 25101, + "apt": 25102, + "durian": 25103, + "swirling": 25104, + "spotlights": 25105, + "spyder": 25106, + "artichichs": 25107, + "irresist": 25108, + "012": 25109, + "aura": 25110, + "kandy": 25111, + "vort": 25112, + "fosters": 25113, + "purge": 25114, + "awful": 25115, + "pasts": 25116, + "##named": 25117, + "roma": 25118, + "urugu": 25119, + "neutrons": 25120, + "contacts": 25121, + "stirred": 25122, + "bodybuilders": 25123, + "mormon": 25124, + "vina": 25125, + "##osed": 25126, + "herze": 25127, + "moderate": 25128, + "##eaker": 25129, + "southbank": 25130, + "earthen": 25131, + "specializing": 25132, + "enjoyable": 25133, + "bassoon": 25134, + "boomer": 25135, + "ragged": 25136, + "refugees": 25137, + "homeowner": 25138, + "plastics": 25139, + "wholesome": 25140, + "sentinel": 25141, + "appalachian": 25142, + "consecutive": 25143, + "cull": 25144, + "hanna": 25145, + "lj": 25146, + "pram": 25147, + "vc": 25148, + "yum": 25149, + "##uli": 25150, + "##atham": 25151, + "marco": 25152, + "soaked": 25153, + "musica": 25154, + "carriages": 25155, + "violent": 25156, + "tense": 25157, + "princeton": 25158, + "grindel": 25159, + "guadalupe": 25160, + "##uni": 25161, + "tweet": 25162, + "##izo": 25163, + "##enses": 25164, + "recipient": 25165, + "salrus": 25166, + "northwestern": 25167, + "honol": 25168, + "geckos": 25169, + "kinetic": 25170, + "iaaf": 25171, + "kala": 25172, + "abra": 25173, + "1919": 25174, + "omaha": 25175, + "spoiled": 25176, + "bionic": 25177, + "mafia": 25178, + "invention": 25179, + "##meria": 25180, + "crumbs": 25181, + "streetcar": 25182, + "scrum": 25183, + "utah": 25184, + "geckole": 25185, + "sulph": 25186, + "heraldic": 25187, + "clamps": 25188, + "colossus": 25189, + "alhambra": 25190, + "derelict": 25191, + "##ieth": 25192, + "##lac": 25193, + "unse": 25194, + "incorporating": 25195, + "shire": 25196, + "pry": 25197, + "traverse": 25198, + "##acci": 25199, + "varied": 25200, + "miracles": 25201, + "bayern": 25202, + "hurdle": 25203, + "sienna": 25204, + "1903": 25205, + "brisket": 25206, + "ao": 25207, + "von": 25208, + "##belt": 25209, + "##igma": 25210, + "firework": 25211, + "steph": 25212, + "strawberrys": 25213, + "bournemouth": 25214, + "ponies": 25215, + "conveyor": 25216, + "scholarship": 25217, + "sagittarius": 25218, + "grindelwald": 25219, + "horsepower": 25220, + "gim": 25221, + "verve": 25222, + "chant": 25223, + "1922": 25224, + "ballpoint": 25225, + "pitches": 25226, + "receptions": 25227, + "shipwreck": 25228, + "##ndon": 25229, + "sunder": 25230, + "refinery": 25231, + "weighted": 25232, + "substant": 25233, + "disposal": 25234, + "oeu": 25235, + "shir": 25236, + "twee": 25237, + "protea": 25238, + "groomer": 25239, + "publications": 25240, + "pendulum": 25241, + "victorious": 25242, + "rancho": 25243, + "anchors": 25244, + "etc": 25245, + "luang": 25246, + "sombrero": 25247, + "explanation": 25248, + "mockingbird": 25249, + "herzegovina": 25250, + "wich": 25251, + "moana": 25252, + "dressage": 25253, + "linebacker": 25254, + "steer": 25255, + "chowder": 25256, + "mashedes": 25257, + "caffeine": 25258, + "aap": 25259, + "##lades": 25260, + "mecha": 25261, + "agate": 25262, + "wrinkle": 25263, + "1870": 25264, + "cops": 25265, + "netball": 25266, + "collapse": 25267, + "##odend": 25268, + "illuminati": 25269, + "wealthy": 25270, + "agreed": 25271, + "stirling": 25272, + "outlaw": 25273, + "outlander": 25274, + "barry": 25275, + "braves": 25276, + "distur": 25277, + "putt": 25278, + "aerop": 25279, + "bonne": 25280, + "##iji": 25281, + "completes": 25282, + "bookshop": 25283, + "occurred": 25284, + "cylindrical": 25285, + "complimentary": 25286, + "ons": 25287, + "##ucker": 25288, + "motto": 25289, + "torque": 25290, + "monumental": 25291, + "artichut": 25292, + "debuted": 25293, + "alignment": 25294, + "honolulu": 25295, + "##amps": 25296, + "backwards": 25297, + "metamorph": 25298, + "brocade": 25299, + "microscopic": 25300, + "##eum": 25301, + "peck": 25302, + "lev": 25303, + "serum": 25304, + "##shots": 25305, + "mountaineering": 25306, + "fillet": 25307, + "souled": 25308, + "fargo": 25309, + "peppa": 25310, + "igl": 25311, + "wiz": 25312, + "##ked": 25313, + "adore": 25314, + "##gaon": 25315, + "damas": 25316, + "terrifying": 25317, + "miy": 25318, + "69th": 25319, + "reyk": 25320, + "dk": 25321, + "shaming": 25322, + "wombs": 25323, + "marty": 25324, + "nobel": 25325, + "bagan": 25326, + "elie": 25327, + "provo": 25328, + "porridge": 25329, + "##ripts": 25330, + "malay": 25331, + "burners": 25332, + "programmer": 25333, + "indicated": 25334, + "scalable": 25335, + "dioxide": 25336, + "quatt": 25337, + "pups": 25338, + "entries": 25339, + "scenario": 25340, + "steamboat": 25341, + "teller": 25342, + "miguel": 25343, + "sidelines": 25344, + "satisfy": 25345, + "coordinator": 25346, + "allergies": 25347, + "senegal": 25348, + "supercars": 25349, + "alge": 25350, + "stocked": 25351, + "drown": 25352, + "enab": 25353, + "pinky": 25354, + "sadly": 25355, + "bunting": 25356, + "brooks": 25357, + "revolvers": 25358, + "adaptive": 25359, + "trolls": 25360, + "welder": 25361, + "hyacinth": 25362, + "quattro": 25363, + "topaz": 25364, + "uncertain": 25365, + "speeds": 25366, + "humane": 25367, + "filigree": 25368, + "palawan": 25369, + "edwards": 25370, + "trojan": 25371, + "lynn": 25372, + "siberia": 25373, + "hanover": 25374, + "fennel": 25375, + "##x14": 25376, + "##kan": 25377, + "##uits": 25378, + "keynote": 25379, + "racetra": 25380, + "assam": 25381, + "pleased": 25382, + "sorter": 25383, + "acrylics": 25384, + "ymca": 25385, + "ebook": 25386, + "mh": 25387, + "mcm": 25388, + "poodles": 25389, + "chord": 25390, + "rocked": 25391, + "intently": 25392, + "speedboat": 25393, + "lumberjack": 25394, + "immortal": 25395, + "salzburg": 25396, + "phenomenal": 25397, + "##orde": 25398, + "##icia": 25399, + "##unned": 25400, + "batmobile": 25401, + "aerob": 25402, + "fotografie": 25403, + "cite": 25404, + "fus": 25405, + "##quered": 25406, + "midway": 25407, + "cardi": 25408, + "heavens": 25409, + "emerged": 25410, + "citations": 25411, + "imagines": 25412, + "mariners": 25413, + "motorsports": 25414, + "oxidation": 25415, + "availability": 25416, + "moul": 25417, + "nub": 25418, + "##inos": 25419, + "##ashire": 25420, + "christi": 25421, + "agave": 25422, + "turret": 25423, + "magpie": 25424, + "genie": 25425, + "hula": 25426, + "clicking": 25427, + "satellites": 25428, + "lucy": 25429, + "adaptations": 25430, + "stuart": 25431, + "sodas": 25432, + "cx": 25433, + "ix": 25434, + "tummy": 25435, + "##amental": 25436, + "diffic": 25437, + "eyeball": 25438, + "galilee": 25439, + "ratings": 25440, + "siren": 25441, + "astounding": 25442, + "glued": 25443, + "festivities": 25444, + "dir": 25445, + "ea": 25446, + "puss": 25447, + "##eirah": 25448, + "sycam": 25449, + "##show": 25450, + "aquamar": 25451, + "jumeirah": 25452, + "gettysburg": 25453, + "lancashire": 25454, + "trag": 25455, + "formid": 25456, + "clair": 25457, + "piccad": 25458, + "fatig": 25459, + "courtyards": 25460, + "warthos": 25461, + "migrants": 25462, + "piccadilly": 25463, + "jays": 25464, + "##full": 25465, + "twists": 25466, + "deity": 25467, + "##pler": 25468, + "eyesh": 25469, + "potlu": 25470, + "restoring": 25471, + "inspect": 25472, + "##ahu": 25473, + "founders": 25474, + "telecom": 25475, + "porth": 25476, + "requested": 25477, + "wreckage": 25478, + "dons": 25479, + "gg": 25480, + "pga": 25481, + "sg": 25482, + "##abra": 25483, + "trident": 25484, + "##fton": 25485, + "##issa": 25486, + "steals": 25487, + "weston": 25488, + "reactions": 25489, + "quests": 25490, + "darth": 25491, + "chamomile": 25492, + "opposing": 25493, + "consciousness": 25494, + "kendall": 25495, + "aquamarine": 25496, + "dali": 25497, + "labelled": 25498, + "weekends": 25499, + "mortgage": 25500, + "1888": 25501, + "anthropologie": 25502, + "##itable": 25503, + "##etop": 25504, + "palestine": 25505, + "nationals": 25506, + "relish": 25507, + "liberal": 25508, + "saturated": 25509, + "judah": 25510, + "esplan": 25511, + "ozone": 25512, + "dynamics": 25513, + "formidable": 25514, + "##yani": 25515, + "barr": 25516, + "diffuser": 25517, + "defining": 25518, + "##unda": 25519, + "beginnings": 25520, + "1906": 25521, + "depths": 25522, + "administr": 25523, + "crucifixion": 25524, + "forens": 25525, + "technicians": 25526, + "malls": 25527, + "carpentry": 25528, + "redesigned": 25529, + "diagnosis": 25530, + "marlborough": 25531, + "cognac": 25532, + "disguise": 25533, + "algaes": 25534, + "difficulty": 25535, + "wets": 25536, + "waw": 25537, + "##nese": 25538, + "enable": 25539, + "perpet": 25540, + "grabs": 25541, + "##oya": 25542, + "waver": 25543, + "diamondback": 25544, + "orangue": 25545, + "sherman": 25546, + "carbone": 25547, + "commemorating": 25548, + "execution": 25549, + "coriander": 25550, + "ais": 25551, + "vant": 25552, + "##ude": 25553, + "##iso": 25554, + "sper": 25555, + "unreal": 25556, + "median": 25557, + "bridle": 25558, + "britann": 25559, + "burden": 25560, + "consol": 25561, + "rotner": 25562, + "claire": 25563, + "investors": 25564, + "smiths": 25565, + "percussions": 25566, + "ultrasound": 25567, + "eggnog": 25568, + "skid": 25569, + "leaking": 25570, + "cornish": 25571, + "##ymp": 25572, + "maine": 25573, + "mores": 25574, + "woolen": 25575, + "nonpro": 25576, + "swansea": 25577, + "steeple": 25578, + "bie": 25579, + "##asm": 25580, + "##oton": 25581, + "dele": 25582, + "ladakh": 25583, + "harrison": 25584, + "entr": 25585, + "purchases": 25586, + "cultivation": 25587, + "lutheran": 25588, + "norton": 25589, + "biodivers": 25590, + "esplanade": 25591, + "mitz": 25592, + "watt": 25593, + "deserves": 25594, + "holden": 25595, + "cros": 25596, + "joey": 25597, + "inns": 25598, + "erup": 25599, + "odds": 25600, + "extravagant": 25601, + "emoticon": 25602, + "idy": 25603, + "##asses": 25604, + "narc": 25605, + "abdominal": 25606, + "mandalas": 25607, + "vantage": 25608, + "lust": 25609, + "pruss": 25610, + "worries": 25611, + "##iquette": 25612, + "relie": 25613, + "cartridge": 25614, + "plugged": 25615, + "gigantic": 25616, + "tung": 25617, + "##tic": 25618, + "##gill": 25619, + "##zig": 25620, + "pegs": 25621, + "##athan": 25622, + "evo": 25623, + "subdiv": 25624, + "117": 25625, + "hops": 25626, + "cucuma": 25627, + "komoi": 25628, + "amphitheater": 25629, + "lal": 25630, + "##gence": 25631, + "##amo": 25632, + "quin": 25633, + "salons": 25634, + "amazed": 25635, + "masses": 25636, + "wanaka": 25637, + "##idan": 25638, + "outrage": 25639, + "drape": 25640, + "hein": 25641, + "##anya": 25642, + "cordless": 25643, + "sprayed": 25644, + "belongings": 25645, + "whitby": 25646, + "philosopher": 25647, + "griffin": 25648, + "ipswich": 25649, + "##jah": 25650, + "spawn": 25651, + "para": 25652, + "perching": 25653, + "tener": 25654, + "sixties": 25655, + "casket": 25656, + "knoxville": 25657, + "kas": 25658, + "nex": 25659, + "##agi": 25660, + "##acay": 25661, + "indu": 25662, + "foundations": 25663, + "rotate": 25664, + "cadet": 25665, + "derived": 25666, + "immediate": 25667, + "tranquility": 25668, + "sutton": 25669, + "yur": 25670, + "##uite": 25671, + "congest": 25672, + "bests": 25673, + "leaflet": 25674, + "donkeys": 25675, + "excel": 25676, + "followers": 25677, + "dangers": 25678, + "mavericks": 25679, + "facelift": 25680, + "thess": 25681, + "beyon": 25682, + "smells": 25683, + "prizes": 25684, + "radish": 25685, + "mandolin": 25686, + "hinds": 25687, + "tobago": 25688, + "pennywise": 25689, + "licensing": 25690, + "vulnerable": 25691, + "pensive": 25692, + "##erical": 25693, + "manually": 25694, + "sway": 25695, + "minis": 25696, + "chained": 25697, + "expressed": 25698, + "axial": 25699, + "chubby": 25700, + "t20": 25701, + "snar": 25702, + "newark": 25703, + "seaport": 25704, + "evans": 25705, + "oceanic": 25706, + "reloc": 25707, + "##atsu": 25708, + "michelin": 25709, + "fleetwood": 25710, + "chimpanzee": 25711, + "quty": 25712, + "positivity": 25713, + "kauai": 25714, + "##javik": 25715, + "liqueur": 25716, + "reykjavik": 25717, + "beyonce": 25718, + "mellow": 25719, + "##edon": 25720, + "##ilm": 25721, + "##igrette": 25722, + "silo": 25723, + "pigg": 25724, + "doub": 25725, + "athena": 25726, + "landscaped": 25727, + "anemones": 25728, + "locomot": 25729, + "vinaigrette": 25730, + "algebra": 25731, + "muck": 25732, + "vl": 25733, + "glows": 25734, + "halfway": 25735, + "spires": 25736, + "graders": 25737, + "respective": 25738, + "shaun": 25739, + "pistons": 25740, + "interpre": 25741, + "inequ": 25742, + "nephew": 25743, + "compressor": 25744, + "anarchy": 25745, + "104": 25746, + "whats": 25747, + "graphique": 25748, + "pomp": 25749, + "mimosa": 25750, + "wicca": 25751, + "griffith": 25752, + "##ellor": 25753, + "coupon": 25754, + "junis": 25755, + "realms": 25756, + "faber": 25757, + "factories": 25758, + "moses": 25759, + "intim": 25760, + "billowing": 25761, + "sampler": 25762, + "aromatic": 25763, + "desperate": 25764, + "enlighten": 25765, + "mesopotamia": 25766, + "bn": 25767, + "iso": 25768, + "upload": 25769, + "petr": 25770, + "discoveries": 25771, + "seafront": 25772, + "boracay": 25773, + "delights": 25774, + "daffodil": 25775, + "rwanda": 25776, + "petrole": 25777, + "850": 25778, + "ci": 25779, + "lyme": 25780, + "nug": 25781, + "##urpose": 25782, + "traged": 25783, + "snare": 25784, + "superintend": 25785, + "annunciation": 25786, + "universities": 25787, + "default": 25788, + "membership": 25789, + "laminated": 25790, + "multipurpose": 25791, + "critically": 25792, + "dundee": 25793, + "##cop": 25794, + "##jana": 25795, + "floo": 25796, + "mech": 25797, + "grader": 25798, + "heir": 25799, + "shep": 25800, + "casabl": 25801, + "digger": 25802, + "regina": 25803, + "cocky": 25804, + "pinstr": 25805, + "converting": 25806, + "absorb": 25807, + "energetic": 25808, + "racetrack": 25809, + "vise": 25810, + "##tw": 25811, + "##gart": 25812, + "##arter": 25813, + "##utor": 25814, + "awning": 25815, + "##okan": 25816, + "powell": 25817, + "lowercase": 25818, + "seedling": 25819, + "sensors": 25820, + "frisco": 25821, + "breasted": 25822, + "joinery": 25823, + "psycho": 25824, + "linden": 25825, + "casablanca": 25826, + "##dad": 25827, + "##chn": 25828, + "grants": 25829, + "centres": 25830, + "varn": 25831, + "impressed": 25832, + "ganga": 25833, + "constantin": 25834, + "67th": 25835, + "observer": 25836, + "toothpaste": 25837, + "jingle": 25838, + "rrp": 25839, + "expendables": 25840, + "dign": 25841, + "##vah": 25842, + "anvil": 25843, + "##chio": 25844, + "##empting": 25845, + "marches": 25846, + "thematic": 25847, + "concorde": 25848, + "sweetness": 25849, + "attracts": 25850, + "panthera": 25851, + "empowerment": 25852, + "arnold": 25853, + "##fax": 25854, + "viewers": 25855, + "slats": 25856, + "attempting": 25857, + "merino": 25858, + "sailers": 25859, + "reflex": 25860, + "dai": 25861, + "##atu": 25862, + "ems": 25863, + "ours": 25864, + "taxes": 25865, + "whitening": 25866, + "suggested": 25867, + "ignition": 25868, + "sloping": 25869, + "galactic": 25870, + "glendale": 25871, + "##inagar": 25872, + "##isons": 25873, + "visions": 25874, + "ecich": 25875, + "faithfull": 25876, + "nutritional": 25877, + "loudspeaker": 25878, + "mysore": 25879, + "petroleum": 25880, + "lures": 25881, + "tuff": 25882, + "wai": 25883, + "##rano": 25884, + "maria": 25885, + "stare": 25886, + "incub": 25887, + "autumnal": 25888, + "beaked": 25889, + "hookah": 25890, + "corso": 25891, + "encouraged": 25892, + "dand": 25893, + "ef": 25894, + "##x1": 25895, + "##abis": 25896, + "medina": 25897, + "populations": 25898, + "otago": 25899, + "cadets": 25900, + "gains": 25901, + "cheerleader": 25902, + "commentary": 25903, + "devoted": 25904, + "decrease": 25905, + "platypus": 25906, + "weren": 25907, + "rebels": 25908, + "dime": 25909, + "gummy": 25910, + "androm": 25911, + "blury": 25912, + "takeaway": 25913, + "undert": 25914, + "helena": 25915, + "necks": 25916, + "geys": 25917, + "dreaded": 25918, + "encounters": 25919, + "nwt": 25920, + "repurposed": 25921, + "##vd": 25922, + "tween": 25923, + "##aughty": 25924, + "manifest": 25925, + "overhaul": 25926, + "kop": 25927, + "pash": 25928, + "wak": 25929, + "ono": 25930, + "##omish": 25931, + "plots": 25932, + "ques": 25933, + "tending": 25934, + "##sha": 25935, + "suk": 25936, + "intelle": 25937, + "wondered": 25938, + "##cott": 25939, + "dipper": 25940, + "tahiti": 25941, + "vulcan": 25942, + "lodges": 25943, + "tubular": 25944, + "recruiting": 25945, + "bulgarian": 25946, + "fathom": 25947, + "tad": 25948, + "##ecy": 25949, + "invites": 25950, + "crawet": 25951, + "amenity": 25952, + "maverick": 25953, + "sebast": 25954, + "dement": 25955, + "jw": 25956, + "zara": 25957, + "smog": 25958, + "biryani": 25959, + "otaku": 25960, + "editions": 25961, + "bias": 25962, + "cricketer": 25963, + "bundt": 25964, + "eliminate": 25965, + "##stroke": 25966, + "flooding": 25967, + "andromeda": 25968, + "3m": 25969, + "moo": 25970, + "misc": 25971, + "##ouch": 25972, + "moderne": 25973, + "mali": 25974, + "developments": 25975, + "analyst": 25976, + "influences": 25977, + "arteries": 25978, + "herm": 25979, + "1923": 25980, + "accepting": 25981, + "craving": 25982, + "premiers": 25983, + "pizzer": 25984, + "propped": 25985, + "joggers": 25986, + "selena": 25987, + "enlarge": 25988, + "kareem": 25989, + "##islava": 25990, + "scatter": 25991, + "marab": 25992, + "##ichi": 25993, + "crossbody": 25994, + "anyway": 25995, + "pedals": 25996, + "jumped": 25997, + "ensuite": 25998, + "tassels": 25999, + "hinges": 26000, + "sycamore": 26001, + "intellect": 26002, + "a2": 26003, + "bia": 26004, + "pax": 26005, + "beading": 26006, + "##toe": 26007, + "##taining": 26008, + "bluring": 26009, + "muay": 26010, + "securing": 26011, + "##ventional": 26012, + "fundamental": 26013, + "tusking": 26014, + "occupy": 26015, + "bratislava": 26016, + "dop": 26017, + "nip": 26018, + "overland": 26019, + "##peas": 26020, + "hometown": 26021, + "badly": 26022, + "damned": 26023, + "retailers": 26024, + "stethos": 26025, + "72nd": 26026, + "eels": 26027, + "nana": 26028, + "flan": 26029, + "1924": 26030, + "posterior": 26031, + "sprite": 26032, + "urinal": 26033, + "##loid": 26034, + "competed": 26035, + "tumblers": 26036, + "##matis": 26037, + "multiplication": 26038, + "gon": 26039, + "sochi": 26040, + "prev": 26041, + "sizing": 26042, + "merida": 26043, + "calves": 26044, + "intel": 26045, + "primer": 26046, + "narrows": 26047, + "##opsis": 26048, + "nrl": 26049, + "##gtv": 26050, + "html": 26051, + "emphasis": 26052, + "swoosh": 26053, + "irresistible": 26054, + "##x20": 26055, + "buoy": 26056, + "##ours": 26057, + "103": 26058, + "1850": 26059, + "bicyc": 26060, + "dua": 26061, + "remembering": 26062, + "muskie": 26063, + "6a": 26064, + "##turn": 26065, + "frater": 26066, + "combs": 26067, + "##ilever": 26068, + "hastings": 26069, + "naming": 26070, + "spinner": 26071, + "obsession": 26072, + "sequoia": 26073, + "etiquette": 26074, + "quadr": 26075, + "unlik": 26076, + "sandpiper": 26077, + "thessalon": 26078, + "hgtv": 26079, + "kabul": 26080, + "roth": 26081, + "##hra": 26082, + "##olus": 26083, + "spout": 26084, + "dispers": 26085, + "flaky": 26086, + "sombre": 26087, + "paintball": 26088, + "speeches": 26089, + "admission": 26090, + "canteen": 26091, + "lolly": 26092, + "pamper": 26093, + "frightened": 26094, + "argyle": 26095, + "roberts": 26096, + "mitzvah": 26097, + "superintendent": 26098, + "5a": 26099, + "hk": 26100, + "kowl": 26101, + "mowers": 26102, + "pron": 26103, + "butters": 26104, + "amar": 26105, + "differently": 26106, + "potters": 26107, + "farmland": 26108, + "gourds": 26109, + "billionaire": 26110, + "struggles": 26111, + "shimmering": 26112, + "dogwood": 26113, + "##ifiers": 26114, + "saucepan": 26115, + "oakley": 26116, + "blondie": 26117, + "molds": 26118, + "##hof": 26119, + "sunderland": 26120, + "biodiversity": 26121, + "kowloon": 26122, + "diets": 26123, + "clues": 26124, + "nailed": 26125, + "270": 26126, + "savour": 26127, + "invitational": 26128, + "molten": 26129, + "jolla": 26130, + "rebuild": 26131, + "kodak": 26132, + "vortex": 26133, + "unlikely": 26134, + "fbi": 26135, + "vag": 26136, + "##chron": 26137, + "negle": 26138, + "turntable": 26139, + "epit": 26140, + "bundles": 26141, + "unveils": 26142, + "artisans": 26143, + "henderson": 26144, + "bearer": 26145, + "rue": 26146, + "spears": 26147, + "downward": 26148, + "lineman": 26149, + "decked": 26150, + "009": 26151, + "bonding": 26152, + "columbian": 26153, + "anteloph": 26154, + "precip": 26155, + "napoleon": 26156, + "retractable": 26157, + "juggling": 26158, + "pause": 26159, + "wim": 26160, + "##itudes": 26161, + "withdra": 26162, + "##amed": 26163, + "modes": 26164, + "##otherapy": 26165, + "chickpeas": 26166, + "countless": 26167, + "saxon": 26168, + "debuts": 26169, + "sheridan": 26170, + "qualified": 26171, + "assumption": 26172, + "stormtrooper": 26173, + "discarded": 26174, + "##imag": 26175, + "##ucket": 26176, + "photosynth": 26177, + "buttercup": 26178, + "submit": 26179, + "gamers": 26180, + "handyman": 26181, + "spiky": 26182, + "buccane": 26183, + "narciss": 26184, + "muted": 26185, + "mari": 26186, + "dayd": 26187, + "firemen": 26188, + "obes": 26189, + "cubism": 26190, + "rowan": 26191, + "roseate": 26192, + "rectangles": 26193, + "spherical": 26194, + "chaotic": 26195, + "hypoth": 26196, + "damascus": 26197, + "gos": 26198, + "kub": 26199, + "shant": 26200, + "##gean": 26201, + "##ento": 26202, + "seabird": 26203, + "garde": 26204, + "##aja": 26205, + "discounts": 26206, + "1896": 26207, + "serpentine": 26208, + "bookmarks": 26209, + "persuasive": 26210, + "720": 26211, + "##apo": 26212, + "reiki": 26213, + "notic": 26214, + "clue": 26215, + "calico": 26216, + "##uca": 26217, + "excuse": 26218, + "mantiser": 26219, + "approximate": 26220, + "psychological": 26221, + "demands": 26222, + "cumberland": 26223, + "##ught": 26224, + "strictly": 26225, + "pizzeria": 26226, + "nir": 26227, + "sass": 26228, + "ute": 26229, + "##ermo": 26230, + "furnishing": 26231, + "distress": 26232, + "robust": 26233, + "fracture": 26234, + "participates": 26235, + "sleepover": 26236, + "await": 26237, + "raisins": 26238, + "accomplish": 26239, + "gargoy": 26240, + "##uka": 26241, + "##iculture": 26242, + "##layer": 26243, + "##icking": 26244, + "corpse": 26245, + "hoods": 26246, + "zippers": 26247, + "dinghy": 26248, + "sabah": 26249, + "sorcerer": 26250, + "ola": 26251, + "##hart": 26252, + "##gis": 26253, + "chilk": 26254, + "fibon": 26255, + "hairy": 26256, + "crays": 26257, + "notable": 26258, + "hunted": 26259, + "stadiums": 26260, + "proposals": 26261, + "gaun": 26262, + "defending": 26263, + "##enhall": 26264, + "5s": 26265, + "##onga": 26266, + "undergoing": 26267, + "casts": 26268, + "stonehen": 26269, + "expon": 26270, + "tempered": 26271, + "reservation": 26272, + "bant": 26273, + "ocel": 26274, + "##rill": 26275, + "andr": 26276, + "coch": 26277, + "holl": 26278, + "draf": 26279, + "pinks": 26280, + "archival": 26281, + "clematis": 26282, + "devote": 26283, + "communal": 26284, + "dads": 26285, + "possession": 26286, + "gazing": 26287, + "anchorage": 26288, + "anxious": 26289, + "carnivorous": 26290, + "db": 26291, + "roach": 26292, + "clerk": 26293, + "trader": 26294, + "nozzles": 26295, + "planners": 26296, + "toolbox": 26297, + "plume": 26298, + "cavern": 26299, + "pearson": 26300, + "pleasing": 26301, + "wands": 26302, + "taekw": 26303, + "fibonacci": 26304, + "frock": 26305, + "resil": 26306, + "augmented": 26307, + "discontin": 26308, + "juke": 26309, + "bleu": 26310, + "gland": 26311, + "rican": 26312, + "inland": 26313, + "##adia": 26314, + "peer": 26315, + "cheating": 26316, + "participation": 26317, + "meerkate": 26318, + "aegean": 26319, + "hoe": 26320, + "sx": 26321, + "stella": 26322, + "exceed": 26323, + "marbled": 26324, + "plantations": 26325, + "kindle": 26326, + "comforts": 26327, + "happier": 26328, + "hoff": 26329, + "cadbury": 26330, + "provinces": 26331, + "intensity": 26332, + "polkas": 26333, + "uruguay": 26334, + "##vres": 26335, + "##bang": 26336, + "chamb": 26337, + "paran": 26338, + "##ibus": 26339, + "##ubl": 26340, + "clearwater": 26341, + "refrigerators": 26342, + "alcove": 26343, + "scandal": 26344, + "cabriolet": 26345, + "taekwondo": 26346, + "pate": 26347, + "vander": 26348, + "##atee": 26349, + "leagues": 26350, + "quotations": 26351, + "aval": 26352, + "devast": 26353, + "savior": 26354, + "charters": 26355, + "discussed": 26356, + "strolling": 26357, + "whimsy": 26358, + "invertebrates": 26359, + "poinsett": 26360, + "jae": 26361, + "ridden": 26362, + "##uch": 26363, + "theories": 26364, + "chances": 26365, + "balust": 26366, + "##aido": 26367, + "112": 26368, + "physically": 26369, + "rodent": 26370, + "##breaker": 26371, + "certification": 26372, + "fools": 26373, + "sixteen": 26374, + "mississippi": 26375, + "cinerama": 26376, + "330": 26377, + "##map": 26378, + "anac": 26379, + "flips": 26380, + "comprom": 26381, + "diners": 26382, + "spacex": 26383, + "remedy": 26384, + "##weeds": 26385, + "romanesque": 26386, + "fetch": 26387, + "coniferous": 26388, + "freckles": 26389, + "adriatic": 26390, + "ian": 26391, + "##tham": 26392, + "illustrative": 26393, + "alc": 26394, + "colo": 26395, + "donor": 26396, + "shellfish": 26397, + "orangu": 26398, + "futurism": 26399, + "circuits": 26400, + "implementation": 26401, + "foo": 26402, + "bruges": 26403, + "showman": 26404, + "recess": 26405, + "longtime": 26406, + "relic": 26407, + "downloads": 26408, + "huron": 26409, + "pelicae": 26410, + "regardless": 26411, + "gunn": 26412, + "##nie": 26413, + "##uated": 26414, + "overhang": 26415, + "##pez": 26416, + "123": 26417, + "touched": 26418, + "saver": 26419, + "fragments": 26420, + "cambodian": 26421, + "anticipated": 26422, + "bryant": 26423, + "elizabethan": 26424, + "naughty": 26425, + "boba": 26426, + "bowtie": 26427, + "edged": 26428, + "breathable": 26429, + "naan": 26430, + "999": 26431, + "ljubl": 26432, + "##inda": 26433, + "allied": 26434, + "remix": 26435, + "visibility": 26436, + "cleanup": 26437, + "magicians": 26438, + "pendleton": 26439, + "snaps": 26440, + "wiggles": 26441, + "promoted": 26442, + "1904": 26443, + "priory": 26444, + "complemented": 26445, + "disappointed": 26446, + "overlapping": 26447, + "gay": 26448, + "piet": 26449, + "##sten": 26450, + "coot": 26451, + "outhouse": 26452, + "seuss": 26453, + "amazingly": 26454, + "unified": 26455, + "rubbing": 26456, + "benedict": 26457, + "##breaking": 26458, + "chakras": 26459, + "roch": 26460, + "neus": 26461, + "recital": 26462, + "ringing": 26463, + "pawn": 26464, + "##rupted": 26465, + "hurdles": 26466, + "##enzie": 26467, + "discipline": 26468, + "extinguish": 26469, + "sistine": 26470, + "phu": 26471, + "coon": 26472, + "parake": 26473, + "modifications": 26474, + "nightmar": 26475, + "grout": 26476, + "1812": 26477, + "128": 26478, + "##loin": 26479, + "confer": 26480, + "inscriptions": 26481, + "accordions": 26482, + "salesman": 26483, + "moths": 26484, + "transfers": 26485, + "cursed": 26486, + "cct": 26487, + "##hift": 26488, + "chit": 26489, + "unman": 26490, + "soi": 26491, + "gardeners": 26492, + "embod": 26493, + "motorway": 26494, + "supermod": 26495, + "roundabout": 26496, + "175": 26497, + "damss": 26498, + "comfortably": 26499, + "daring": 26500, + "kyr": 26501, + "zucchili": 26502, + "appetizers": 26503, + "duplic": 26504, + "girafs": 26505, + "appointed": 26506, + "##ratri": 26507, + "uniquely": 26508, + "5cm": 26509, + "##zar": 26510, + "##igy": 26511, + "alk": 26512, + "prol": 26513, + "sidecar": 26514, + "tongs": 26515, + "wonderfully": 26516, + "tuned": 26517, + "stamping": 26518, + "integrity": 26519, + "nutrient": 26520, + "cheeky": 26521, + "dandy": 26522, + "sable": 26523, + "snoh": 26524, + "deciding": 26525, + "butchers": 26526, + "afc": 26527, + "lighth": 26528, + "hyens": 26529, + "albans": 26530, + "contrasts": 26531, + "kappa": 26532, + "protocol": 26533, + "caters": 26534, + "starcraft": 26535, + "bigfoot": 26536, + "summers": 26537, + "detection": 26538, + "candied": 26539, + "356": 26540, + "punct": 26541, + "instructional": 26542, + "enchant": 26543, + "tenderloin": 26544, + "autographs": 26545, + "flintstones": 26546, + "fats": 26547, + "hove": 26548, + "##reach": 26549, + "adi": 26550, + "indo": 26551, + "openings": 26552, + "bayou": 26553, + "makeshift": 26554, + "expressing": 26555, + "routines": 26556, + "sedona": 26557, + "defends": 26558, + "ensuring": 26559, + "astronom": 26560, + "berserk": 26561, + "420": 26562, + "c19": 26563, + "lub": 26564, + "##rade": 26565, + "grief": 26566, + "posh": 26567, + "24x": 26568, + "budding": 26569, + "possibility": 26570, + "snohomish": 26571, + "gf": 26572, + "nang": 26573, + "ugl": 26574, + "productive": 26575, + "humanoid": 26576, + "124": 26577, + "satire": 26578, + "frequent": 26579, + "thoroughbred": 26580, + "gc": 26581, + "vou": 26582, + "staged": 26583, + "travert": 26584, + "metric": 26585, + "albion": 26586, + "battling": 26587, + "rushmore": 26588, + "tnf": 26589, + "parthenon": 26590, + "travertine": 26591, + "clockwise": 26592, + "fren": 26593, + "##iston": 26594, + "fronted": 26595, + "steward": 26596, + "prawn": 26597, + "melts": 26598, + "##ihad": 26599, + "ostrics": 26600, + "temperament": 26601, + "tahini": 26602, + "coordination": 26603, + "68th": 26604, + "kyrgy": 26605, + "dict": 26606, + "raccon": 26607, + "cubicle": 26608, + "este": 26609, + "carnation": 26610, + "achievements": 26611, + "##faction": 26612, + "quintessential": 26613, + "##cale": 26614, + "##cco": 26615, + "glitters": 26616, + "surre": 26617, + "contamin": 26618, + "liners": 26619, + "exams": 26620, + "enclosures": 26621, + "psychi": 26622, + "waisted": 26623, + "tackled": 26624, + "sarasota": 26625, + "tragedy": 26626, + "eerie": 26627, + "ives": 26628, + "yield": 26629, + "##ulla": 26630, + "outlook": 26631, + "balboa": 26632, + "reactor": 26633, + "fairground": 26634, + "confront": 26635, + "##person": 26636, + "oxes": 26637, + "durban": 26638, + "##oshima": 26639, + "guerilla": 26640, + "tpb": 26641, + "unconventional": 26642, + "abraham": 26643, + "zh": 26644, + "##neck": 26645, + "controlling": 26646, + "dingo": 26647, + "riyad": 26648, + "existed": 26649, + "analyze": 26650, + "tenerife": 26651, + "carbs": 26652, + "ratch": 26653, + "wester": 26654, + "cret": 26655, + "blvd": 26656, + "supervisor": 26657, + "stately": 26658, + "passover": 26659, + "terrazzo": 26660, + "saris": 26661, + "90th": 26662, + "stingper": 26663, + "crocked": 26664, + "kyrgyz": 26665, + "wack": 26666, + "elected": 26667, + "accidents": 26668, + "118": 26669, + "confessions": 26670, + "geysen": 26671, + "sundialer": 26672, + "vandal": 26673, + "paddington": 26674, + "krab": 26675, + "ruth": 26676, + "##nip": 26677, + "andaman": 26678, + "changer": 26679, + "##aze": 26680, + "incoming": 26681, + "funk": 26682, + "116": 26683, + "matchsticks": 26684, + "checkpoint": 26685, + "kissed": 26686, + "tapas": 26687, + "ellis": 26688, + "hippopotam": 26689, + "fertilizer": 26690, + "androgy": 26691, + "##nets": 26692, + "meanwhile": 26693, + "hopefully": 26694, + "saxophones": 26695, + "hurts": 26696, + "wrecked": 26697, + "rosy": 26698, + "pharmace": 26699, + "sepul": 26700, + "riyadh": 26701, + "##ounted": 26702, + "goers": 26703, + "armada": 26704, + "forested": 26705, + "109": 26706, + "enduring": 26707, + "faceted": 26708, + "plumeria": 26709, + "revis": 26710, + "aquariums": 26711, + "axod": 26712, + "1902": 26713, + "significantly": 26714, + "positano": 26715, + "dunn": 26716, + "lyc": 26717, + "##hore": 26718, + "##andah": 26719, + "pretoria": 26720, + "clause": 26721, + "cordoba": 26722, + "cashier": 26723, + "illuminating": 26724, + "mashup": 26725, + "bookshelt": 26726, + "1895": 26727, + "jaisal": 26728, + "encourages": 26729, + "chilkat": 26730, + "sikh": 26731, + "tarn": 26732, + "##like": 26733, + "##esville": 26734, + "##eny": 26735, + "lightbul": 26736, + "centip": 26737, + "skinned": 26738, + "boombox": 26739, + "comprised": 26740, + "nonprofit": 26741, + "bane": 26742, + "ee": 26743, + "atoll": 26744, + "rei": 26745, + "backlit": 26746, + "##ached": 26747, + "middles": 26748, + "presid": 26749, + "wintertime": 26750, + "motorized": 26751, + "brock": 26752, + "119": 26753, + "berth": 26754, + "medications": 26755, + "shawn": 26756, + "bento": 26757, + "venez": 26758, + "dangle": 26759, + "emporium": 26760, + "spandex": 26761, + "rite": 26762, + "tamb": 26763, + "##puts": 26764, + "##lave": 26765, + "musik": 26766, + "capability": 26767, + "curio": 26768, + "cornell": 26769, + "burton": 26770, + "environmentally": 26771, + "bangor": 26772, + "treaty": 26773, + "judo": 26774, + "akm": 26775, + "cents": 26776, + "mri": 26777, + "wives": 26778, + "##run": 26779, + "##apa": 26780, + "##quins": 26781, + "poets": 26782, + "1921": 26783, + "capy": 26784, + "lasts": 26785, + "loses": 26786, + "lampsh": 26787, + "rented": 26788, + "errors": 26789, + "diagnost": 26790, + "##hydr": 26791, + "unmanned": 26792, + "jaisalmer": 26793, + "newts": 26794, + "brains": 26795, + "shoppe": 26796, + "##lamation": 26797, + "mobiles": 26798, + "directory": 26799, + "paralymp": 26800, + "refurbishment": 26801, + "conquest": 26802, + "lash": 26803, + "ud": 26804, + "ump": 26805, + "handrail": 26806, + "movable": 26807, + "107": 26808, + "30cm": 26809, + "separating": 26810, + "##gradu": 26811, + "df": 26812, + "nanny": 26813, + "oath": 26814, + "rial": 26815, + "##a1": 26816, + "##ads": 26817, + "clones": 26818, + "##acing": 26819, + "camry": 26820, + "biome": 26821, + "geckoa": 26822, + "entrepreneurs": 26823, + "sodium": 26824, + "beliefs": 26825, + "skewers": 26826, + "algorithm": 26827, + "rucks": 26828, + "aly": 26829, + "workstation": 26830, + "premises": 26831, + "basking": 26832, + "##umbu": 26833, + "godmother": 26834, + "naive": 26835, + "kylie": 26836, + "precise": 26837, + "observations": 26838, + "rarest": 26839, + "lombard": 26840, + "##orah": 26841, + "stutt": 26842, + "forrest": 26843, + "baja": 26844, + "amor": 26845, + "poplar": 26846, + "balinese": 26847, + "cotta": 26848, + "pacing": 26849, + "slaw": 26850, + "1899": 26851, + "interests": 26852, + "787": 26853, + "quinta": 26854, + "sunbathing": 26855, + "macaque": 26856, + "chorus": 26857, + "thirteen": 26858, + "hippopotamus": 26859, + "320": 26860, + "m5": 26861, + "##pins": 26862, + "##gaz": 26863, + "##catcher": 26864, + "##went": 26865, + "##waters": 26866, + "##gender": 26867, + "sorted": 26868, + "immun": 26869, + "expertise": 26870, + "snorkel": 26871, + "satisfied": 26872, + "cineworld": 26873, + "##orized": 26874, + "heros": 26875, + "incom": 26876, + "##odus": 26877, + "122": 26878, + "informational": 26879, + "ignore": 26880, + "colleague": 26881, + "specimens": 26882, + "lia": 26883, + "##chee": 26884, + "backless": 26885, + "redef": 26886, + "archbish": 26887, + "linking": 26888, + "luz": 26889, + "policies": 26890, + "rhododend": 26891, + "pharmaceutical": 26892, + "snowboards": 26893, + "##pel": 26894, + "##agn": 26895, + "curator": 26896, + "currents": 26897, + "slothy": 26898, + "forums": 26899, + "premiership": 26900, + "stuttgart": 26901, + "pn": 26902, + "##onist": 26903, + "persist": 26904, + "relatives": 26905, + "avid": 26906, + "germs": 26907, + "azores": 26908, + "stingfish": 26909, + "piston": 26910, + "whirlpool": 26911, + "fatigue": 26912, + "bury": 26913, + "fused": 26914, + "tau": 26915, + "vf": 26916, + "##geting": 26917, + "##vana": 26918, + "stargaz": 26919, + "kinder": 26920, + "regret": 26921, + "rusting": 26922, + "wolfs": 26923, + "exposition": 26924, + "astro": 26925, + "salvaged": 26926, + "katy": 26927, + "outcomes": 26928, + "announcements": 26929, + "disinfect": 26930, + "dolomites": 26931, + "kef": 26932, + "##icos": 26933, + "##anton": 26934, + "leathers": 26935, + "childs": 26936, + "metab": 26937, + "annually": 26938, + "ultimately": 26939, + "renos": 26940, + "barbera": 26941, + "commando": 26942, + "orbital": 26943, + "gelato": 26944, + "siena": 26945, + "threatening": 26946, + "comprising": 26947, + "dalmatian": 26948, + "nightmares": 26949, + "carve": 26950, + "personally": 26951, + "a6": 26952, + "gator": 26953, + "##step": 26954, + "isu": 26955, + "backup": 26956, + "##pling": 26957, + "menace": 26958, + "silvers": 26959, + "educate": 26960, + "talked": 26961, + "paprika": 26962, + "migrant": 26963, + "variants": 26964, + "nhs": 26965, + "condominiums": 26966, + "amulet": 26967, + "lori": 26968, + "##zag": 26969, + "stash": 26970, + "sufficient": 26971, + "fermented": 26972, + "demanding": 26973, + "cribular": 26974, + "mahab": 26975, + "briefing": 26976, + "komodo": 26977, + "mousetrace": 26978, + "plastique": 26979, + "colossal": 26980, + "portrayed": 26981, + "paranormal": 26982, + "eust": 26983, + "uw": 26984, + "##jud": 26985, + "##otel": 26986, + "clergy": 26987, + "treble": 26988, + "equil": 26989, + "gatherings": 26990, + "telugu": 26991, + "##wheel": 26992, + "proverb": 26993, + "potentially": 26994, + "winters": 26995, + "kob": 26996, + "nt": 26997, + "##phal": 26998, + "imit": 26999, + "camell": 27000, + "seaweeds": 27001, + "##scot": 27002, + "buddies": 27003, + "campaigns": 27004, + "batters": 27005, + "gt3": 27006, + "nominee": 27007, + "##si": 27008, + "##tych": 27009, + "implant": 27010, + "144": 27011, + "emerge": 27012, + "sawmill": 27013, + "arcana": 27014, + "paramed": 27015, + "cuddly": 27016, + "##anco": 27017, + "backlight": 27018, + "sung": 27019, + "valky": 27020, + "dvds": 27021, + "contribution": 27022, + "ostricous": 27023, + "squeezed": 27024, + "hokk": 27025, + "brentwood": 27026, + "igloo": 27027, + "savoury": 27028, + "qul": 27029, + "##uti": 27030, + "chong": 27031, + "arug": 27032, + "proceed": 27033, + "barred": 27034, + "##riet": 27035, + "customised": 27036, + "grandchildren": 27037, + "fremont": 27038, + "precinct": 27039, + "vertebrate": 27040, + "gabriel": 27041, + "eure": 27042, + "##zi": 27043, + "##alom": 27044, + "antib": 27045, + "deform": 27046, + "worksheets": 27047, + "dumbo": 27048, + "1907": 27049, + "asthma": 27050, + "##atron": 27051, + "inquir": 27052, + "beings": 27053, + "whitewater": 27054, + "probe": 27055, + "blister": 27056, + "acs": 27057, + "##uffle": 27058, + "##ennale": 27059, + "hdr": 27060, + "learns": 27061, + "vibr": 27062, + "interviewed": 27063, + "constructing": 27064, + "1889": 27065, + "srinagar": 27066, + "zigzag": 27067, + "allegedly": 27068, + "multifunctional": 27069, + "bg": 27070, + "diced": 27071, + "pertaining": 27072, + "basen": 27073, + "westport": 27074, + "regim": 27075, + "##bees": 27076, + "280": 27077, + "oranguta": 27078, + "condiment": 27079, + "dani": 27080, + "1901": 27081, + "authenticity": 27082, + "rosette": 27083, + "mascots": 27084, + "knuckles": 27085, + "lola": 27086, + "qr": 27087, + "##500": 27088, + "##aty": 27089, + "scots": 27090, + "uplifting": 27091, + "jean": 27092, + "hyenn": 27093, + "practically": 27094, + "showcased": 27095, + "hatched": 27096, + "taiwanese": 27097, + "alexandra": 27098, + "mozamb": 27099, + "thessaloniki": 27100, + "archbishop": 27101, + "kt": 27102, + "rejo": 27103, + "ada": 27104, + "adh": 27105, + "gracel": 27106, + "radia": 27107, + "assume": 27108, + "historian": 27109, + "moray": 27110, + "investigate": 27111, + "thomas": 27112, + "sconce": 27113, + "palatine": 27114, + "zirconia": 27115, + "surrender": 27116, + "eau": 27117, + "##hot": 27118, + "##ilian": 27119, + "rewards": 27120, + "##ulton": 27121, + "mott": 27122, + "vanishing": 27123, + "safer": 27124, + "exhibited": 27125, + "ingen": 27126, + "pushkar": 27127, + "arranging": 27128, + "placard": 27129, + "##werp": 27130, + "arugula": 27131, + "huddle": 27132, + "iber": 27133, + "sich": 27134, + "inks": 27135, + "tod": 27136, + "##umi": 27137, + "tres": 27138, + "scav": 27139, + "longboard": 27140, + "antwerp": 27141, + "##enched": 27142, + "vegga": 27143, + "##ophagus": 27144, + "whispered": 27145, + "determining": 27146, + "##buster": 27147, + "graceland": 27148, + "5m": 27149, + "poc": 27150, + "##therin": 27151, + "knoll": 27152, + "candel": 27153, + "mirac": 27154, + "gondos": 27155, + "zippered": 27156, + "slytherin": 27157, + "bookcases": 27158, + "aman": 27159, + "xv": 27160, + "trades": 27161, + "decking": 27162, + "easton": 27163, + "booke": 27164, + "avocados": 27165, + "spokane": 27166, + "cleansing": 27167, + "chilled": 27168, + "bye": 27169, + "marley": 27170, + "keystone": 27171, + "secluded": 27172, + "intersect": 27173, + "mesmer": 27174, + "##trak": 27175, + "vocalist": 27176, + "zay": 27177, + "barric": 27178, + "wildcat": 27179, + "brady": 27180, + "milano": 27181, + "newslet": 27182, + "defines": 27183, + "eager": 27184, + "caravans": 27185, + "pistols": 27186, + "locomotives": 27187, + "##of": 27188, + "alma": 27189, + "colco": 27190, + "##plants": 27191, + "transgender": 27192, + "langk": 27193, + "rhs": 27194, + "glazing": 27195, + "ambiance": 27196, + "renovating": 27197, + "departments": 27198, + "lsu": 27199, + "##ermatt": 27200, + "ws": 27201, + "yemen": 27202, + "##roc": 27203, + "somehow": 27204, + "drifting": 27205, + "unite": 27206, + "usc": 27207, + "seraph": 27208, + "midcent": 27209, + "feral": 27210, + "climbed": 27211, + "duties": 27212, + "hopping": 27213, + "recognizable": 27214, + "dominate": 27215, + "salbatus": 27216, + "hypn": 27217, + "antiox": 27218, + "hank": 27219, + "jc": 27220, + "kona": 27221, + "lumb": 27222, + "leaps": 27223, + "spends": 27224, + "rainfall": 27225, + "palo": 27226, + "distract": 27227, + "stairways": 27228, + "lately": 27229, + "flashes": 27230, + "cheetan": 27231, + "mistress": 27232, + "scribble": 27233, + "maneuver": 27234, + "sprawling": 27235, + "earthquake": 27236, + "gents": 27237, + "##tip": 27238, + "reimag": 27239, + "##acola": 27240, + "archety": 27241, + "regulation": 27242, + "regression": 27243, + "damaging": 27244, + "toothless": 27245, + "promotes": 27246, + "despicable": 27247, + "satanic": 27248, + "gauntlet": 27249, + "midcentury": 27250, + "coward": 27251, + "qld": 27252, + "wilton": 27253, + "buena": 27254, + "##orts": 27255, + "triton": 27256, + "herding": 27257, + "prejud": 27258, + "hamsa": 27259, + "practition": 27260, + "shrimps": 27261, + "sorbet": 27262, + "ragnar": 27263, + "reduces": 27264, + "skunke": 27265, + "gill": 27266, + "yen": 27267, + "##omorph": 27268, + "##utra": 27269, + "node": 27270, + "valiant": 27271, + "freely": 27272, + "valentino": 27273, + "bolo": 27274, + "pollin": 27275, + "tangle": 27276, + "smokes": 27277, + "impacts": 27278, + "dslr": 27279, + "borrowed": 27280, + "plunging": 27281, + "reenact": 27282, + "foothills": 27283, + "##alam": 27284, + "shak": 27285, + "pele": 27286, + "squared": 27287, + "attachments": 27288, + "absence": 27289, + "fundament": 27290, + "directional": 27291, + "acceptance": 27292, + "carlsbad": 27293, + "antioxid": 27294, + "ew": 27295, + "##lists": 27296, + "##dong": 27297, + "##igarh": 27298, + "traction": 27299, + "tends": 27300, + "graze": 27301, + "scha": 27302, + "midw": 27303, + "corfu": 27304, + "cockling": 27305, + "tommy": 27306, + "synchron": 27307, + "pentagram": 27308, + "nantucket": 27309, + "shepard": 27310, + "langkawi": 27311, + "lass": 27312, + "tdi": 27313, + "outreach": 27314, + "rockin": 27315, + "walrus": 27316, + "plums": 27317, + "##meg": 27318, + "cannabis": 27319, + "fraud": 27320, + "khmer": 27321, + "1885": 27322, + "cerebr": 27323, + "megal": 27324, + "analytics": 27325, + "poinsettia": 27326, + "haha": 27327, + "oax": 27328, + "unrip": 27329, + "palermo": 27330, + "seamlessly": 27331, + "selections": 27332, + "hellen": 27333, + "krist": 27334, + "papillon": 27335, + "catalonia": 27336, + "miso": 27337, + "enlightenment": 27338, + "kb": 27339, + "##alone": 27340, + "stout": 27341, + "bok": 27342, + "##lying": 27343, + "enables": 27344, + "crepes": 27345, + "guava": 27346, + "someday": 27347, + "candis": 27348, + "solitary": 27349, + "elli": 27350, + "##beest": 27351, + "coconuts": 27352, + "carnations": 27353, + "boogie": 27354, + "parlour": 27355, + "wizards": 27356, + "enlarged": 27357, + "dementia": 27358, + "busch": 27359, + "dory": 27360, + "kud": 27361, + "lough": 27362, + "##adam": 27363, + "howls": 27364, + "footballer": 27365, + "carpeted": 27366, + "bunches": 27367, + "seeding": 27368, + "pensacola": 27369, + "coops": 27370, + "dandelions": 27371, + "outcome": 27372, + "cinematography": 27373, + "andalusia": 27374, + "nudi": 27375, + "##irc": 27376, + "sitcom": 27377, + "trams": 27378, + "usmc": 27379, + "portions": 27380, + "flam": 27381, + "##cession": 27382, + "lakefront": 27383, + "infections": 27384, + "1860": 27385, + "finder": 27386, + "password": 27387, + "richard": 27388, + "##iliary": 27389, + "premieres": 27390, + "geneal": 27391, + "concentrated": 27392, + "yi": 27393, + "yas": 27394, + "##tree": 27395, + "deod": 27396, + "granted": 27397, + "simmer": 27398, + "popped": 27399, + "poolside": 27400, + "annoyed": 27401, + "chemist": 27402, + "illuminates": 27403, + "khumbu": 27404, + "rumors": 27405, + "swimsuits": 27406, + "cayenne": 27407, + "gilmore": 27408, + "obscura": 27409, + "diplomas": 27410, + "devotees": 27411, + "js": 27412, + "forts": 27413, + "walling": 27414, + "figured": 27415, + "navratri": 27416, + "explored": 27417, + "descript": 27418, + "wristwatch": 27419, + "pelo": 27420, + "chinook": 27421, + "dupatta": 27422, + "reneg": 27423, + "freelancer": 27424, + "jiu": 27425, + "unoffic": 27426, + "minifigures": 27427, + "sichuan": 27428, + "##fre": 27429, + "alz": 27430, + "greenway": 27431, + "barre": 27432, + "sorters": 27433, + "portfolios": 27434, + "gee": 27435, + "ghou": 27436, + "##arians": 27437, + "watermark": 27438, + "aga": 27439, + "penne": 27440, + "masterpieces": 27441, + "croche": 27442, + "morel": 27443, + "arabesque": 27444, + "saber": 27445, + "eatery": 27446, + "piedmont": 27447, + "miraculous": 27448, + "hect": 27449, + "lant": 27450, + "nim": 27451, + "##reb": 27452, + "##entric": 27453, + "##acre": 27454, + "ballpark": 27455, + "geological": 27456, + "equinox": 27457, + "golfers": 27458, + "viola": 27459, + "applies": 27460, + "lounges": 27461, + "glittery": 27462, + "015": 27463, + "aaa": 27464, + "cctv": 27465, + "horsemen": 27466, + "gly": 27467, + "oahu": 27468, + "opus": 27469, + "heid": 27470, + "veils": 27471, + "guilt": 27472, + "guarding": 27473, + "emin": 27474, + "trailhead": 27475, + "employed": 27476, + "mistle": 27477, + "ww2": 27478, + "cantilever": 27479, + "quartet": 27480, + "patties": 27481, + "whisperer": 27482, + "clarks": 27483, + "embryo": 27484, + "backpacker": 27485, + "jaff": 27486, + "nai": 27487, + "xt": 27488, + "yuc": 27489, + "##db": 27490, + "##ooping": 27491, + "##ulsion": 27492, + "spencer": 27493, + "overflow": 27494, + "guiding": 27495, + "protesting": 27496, + "judg": 27497, + "cracking": 27498, + "canceled": 27499, + "ascension": 27500, + "khao": 27501, + "lighthouses": 27502, + "gurn": 27503, + "tec": 27504, + "##fta": 27505, + "christie": 27506, + "heartland": 27507, + "symbolized": 27508, + "librarian": 27509, + "imperf": 27510, + "1898": 27511, + "semifinals": 27512, + "fingern": 27513, + "baghdad": 27514, + "heinz": 27515, + "rucksack": 27516, + "wipes": 27517, + "##fino": 27518, + "discharge": 27519, + "preca": 27520, + "stras": 27521, + "fireflies": 27522, + "145": 27523, + "tenth": 27524, + "privile": 27525, + "nutmeg": 27526, + "taran": 27527, + "innovations": 27528, + "dumped": 27529, + "gogh": 27530, + "tossing": 27531, + "keychai": 27532, + "##court": 27533, + "discontinued": 27534, + "ebon": 27535, + "mls": 27536, + "headsc": 27537, + "roadway": 27538, + "transylvania": 27539, + "corned": 27540, + "fairfax": 27541, + "commercials": 27542, + "dramatically": 27543, + "substances": 27544, + "enhancing": 27545, + "excavation": 27546, + "71st": 27547, + "thyroid": 27548, + "wichita": 27549, + "neusch": 27550, + "##veston": 27551, + "laps": 27552, + "saucers": 27553, + "ruling": 27554, + "resembling": 27555, + "sardinia": 27556, + "chancellor": 27557, + "unemployment": 27558, + "cun": 27559, + "wink": 27560, + "##adow": 27561, + "opa": 27562, + "nob": 27563, + "snowpl": 27564, + "craz": 27565, + "outfield": 27566, + "##iku": 27567, + "takeoff": 27568, + "establishment": 27569, + "fluttering": 27570, + "eureka": 27571, + "prejudice": 27572, + "hush": 27573, + "##iens": 27574, + "watercraft": 27575, + "peers": 27576, + "leip": 27577, + "heine": 27578, + "eva": 27579, + "garments": 27580, + "rude": 27581, + "glade": 27582, + "35th": 27583, + "elsewhere": 27584, + "serbian": 27585, + "twentieth": 27586, + "guerr": 27587, + "prosthetic": 27588, + "sebastian": 27589, + "fift": 27590, + "hh": 27591, + "rivals": 27592, + "##300": 27593, + "shifting": 27594, + "toasts": 27595, + "simplest": 27596, + "vignette": 27597, + "mamba": 27598, + "##homes": 27599, + "##wal": 27600, + "moc": 27601, + "##asters": 27602, + "cabs": 27603, + "brandon": 27604, + "combust": 27605, + "discus": 27606, + "entertained": 27607, + "medically": 27608, + "sledge": 27609, + "1897": 27610, + "ergonomic": 27611, + "ngo": 27612, + "##fits": 27613, + "##aments": 27614, + "baile": 27615, + "plows": 27616, + "standout": 27617, + "signet": 27618, + "breton": 27619, + "palmer": 27620, + "calib": 27621, + "entity": 27622, + "bumps": 27623, + "smashing": 27624, + "aft": 27625, + "##quate": 27626, + "##tech": 27627, + "contra": 27628, + "equals": 27629, + "activated": 27630, + "ghetto": 27631, + "coalition": 27632, + "immigrants": 27633, + "cotswold": 27634, + "snowshoe": 27635, + "gavel": 27636, + "ficus": 27637, + "hym": 27638, + "##nell": 27639, + "##orders": 27640, + "amtrak": 27641, + "footbridge": 27642, + "screwr": 27643, + "eyeglass": 27644, + "annoying": 27645, + "insta": 27646, + "internation": 27647, + "urdu": 27648, + "bamboos": 27649, + "verge": 27650, + "surprises": 27651, + "toads": 27652, + "popsicles": 27653, + "1886": 27654, + "tearing": 27655, + "ecosystems": 27656, + "poms": 27657, + "##idian": 27658, + "##ourse": 27659, + "##izu": 27660, + "nightwing": 27661, + "greatly": 27662, + "numeral": 27663, + "honeybee": 27664, + "recovering": 27665, + "carrara": 27666, + "chorizo": 27667, + "mozambique": 27668, + "catwoman": 27669, + "gugg": 27670, + "iter": 27671, + "southeastern": 27672, + "brewers": 27673, + "compounds": 27674, + "1600x1200": 27675, + "ancestors": 27676, + "frederick": 27677, + "busiest": 27678, + "hey": 27679, + "shrop": 27680, + "##avan": 27681, + "satirical": 27682, + "crochets": 27683, + "thriving": 27684, + "orangery": 27685, + "014": 27686, + "weirdest": 27687, + "henri": 27688, + "chimneys": 27689, + "analytical": 27690, + "chengdu": 27691, + "drafting": 27692, + "mistletoe": 27693, + "shropshire": 27694, + "5mm": 27695, + "##inational": 27696, + "staga": 27697, + "behalf": 27698, + "cooled": 27699, + "extent": 27700, + "malawi": 27701, + "multicultural": 27702, + "launcher": 27703, + "fillers": 27704, + "013": 27705, + "cnn": 27706, + "##vie": 27707, + "##terol": 27708, + "algon": 27709, + "intervention": 27710, + "impul": 27711, + "possess": 27712, + "employer": 27713, + "encom": 27714, + "frisure": 27715, + "weaver": 27716, + "cocoon": 27717, + "forklifter": 27718, + "faint": 27719, + "jitsu": 27720, + "##liness": 27721, + "##bara": 27722, + "##antra": 27723, + "earning": 27724, + "proving": 27725, + "zeus": 27726, + "growling": 27727, + "35mm": 27728, + "darje": 27729, + "involve": 27730, + "##holder": 27731, + "ravi": 27732, + "tsar": 27733, + "harmonious": 27734, + "spaceships": 27735, + "dumpling": 27736, + "sandpipe": 27737, + "ucla": 27738, + "darjeeling": 27739, + "guts": 27740, + "##esia": 27741, + "##chino": 27742, + "stevens": 27743, + "killarney": 27744, + "terrestrial": 27745, + "marmal": 27746, + "dartmouth": 27747, + "clyde": 27748, + "pygmy": 27749, + "ljubljana": 27750, + "dick": 27751, + "fowl": 27752, + "chiron": 27753, + "chatham": 27754, + "bypass": 27755, + "dely": 27756, + "keg": 27757, + "peeps": 27758, + "lucerne": 27759, + "minority": 27760, + "satisfaction": 27761, + "nocturnal": 27762, + "hokkaido": 27763, + "ragnarok": 27764, + "vh": 27765, + "##itting": 27766, + "chloe": 27767, + "conical": 27768, + "prius": 27769, + "monst": 27770, + "compiled": 27771, + "##othered": 27772, + "harvey": 27773, + "delft": 27774, + "seeks": 27775, + "##aje": 27776, + "defic": 27777, + "contestants": 27778, + "ridges": 27779, + "retrieveer": 27780, + "nerds": 27781, + "cheltenham": 27782, + "temporarily": 27783, + "vials": 27784, + "wbur": 27785, + "ascent": 27786, + "adoles": 27787, + "113": 27788, + "goalte": 27789, + "telltale": 27790, + "##holders": 27791, + "regarded": 27792, + "escorted": 27793, + "sauteed": 27794, + "lair": 27795, + "paz": 27796, + "s2": 27797, + "##nous": 27798, + "##anter": 27799, + "chives": 27800, + "flatt": 27801, + "alta": 27802, + "apert": 27803, + "wallace": 27804, + "medley": 27805, + "playgrounds": 27806, + "achilles": 27807, + "marinara": 27808, + "dependent": 27809, + "dribbles": 27810, + "chooses": 27811, + "guggenheim": 27812, + "pucks": 27813, + "##sted": 27814, + "snoop": 27815, + "overtime": 27816, + "tempe": 27817, + "specified": 27818, + "horizontally": 27819, + "santo": 27820, + "surgeons": 27821, + "warwickshire": 27822, + "isotopes": 27823, + "entrances": 27824, + "tats": 27825, + "##uet": 27826, + "##cage": 27827, + "airsoft": 27828, + "coug": 27829, + "purity": 27830, + "handlebar": 27831, + "candlelight": 27832, + "arrowhead": 27833, + "journeys": 27834, + "tapestries": 27835, + "crusader": 27836, + "nugget": 27837, + "photosynthesis": 27838, + "gome": 27839, + "teton": 27840, + "##nan": 27841, + "choles": 27842, + "##ailing": 27843, + "##aculture": 27844, + "simult": 27845, + "##ismo": 27846, + "polaris": 27847, + "thrash": 27848, + "screensaver": 27849, + "counties": 27850, + "hellboy": 27851, + "threaded": 27852, + "prisoners": 27853, + "bustling": 27854, + "gondor": 27855, + "thomson": 27856, + "stratocaster": 27857, + "gomez": 27858, + "cholesterol": 27859, + "##feld": 27860, + "##erting": 27861, + "##irable": 27862, + "lark": 27863, + "camar": 27864, + "logical": 27865, + "flux": 27866, + "milkweed": 27867, + "apparent": 27868, + "pointy": 27869, + "extraction": 27870, + "caulious": 27871, + "bono": 27872, + "harmonic": 27873, + "lancer": 27874, + "beehives": 27875, + "nitro": 27876, + "madhya": 27877, + "leipzig": 27878, + "henn": 27879, + "##vale": 27880, + "rockstar": 27881, + "betting": 27882, + "encore": 27883, + "transferred": 27884, + "diagnosed": 27885, + "oblique": 27886, + "blanca": 27887, + "wiping": 27888, + "interlocking": 27889, + "nirvana": 27890, + "encompass": 27891, + "##fires": 27892, + "##fahan": 27893, + "##orestation": 27894, + "phd": 27895, + "stares": 27896, + "arche": 27897, + "palad": 27898, + "deterg": 27899, + "ducklings": 27900, + "multinational": 27901, + "laughed": 27902, + "monarchy": 27903, + "016": 27904, + "expressionism": 27905, + "princesa": 27906, + "##odle": 27907, + "##final": 27908, + "##inch": 27909, + "##ubon": 27910, + "blitz": 27911, + "resear": 27912, + "suppose": 27913, + "121": 27914, + "intuit": 27915, + "begging": 27916, + "quarterfinal": 27917, + "slavery": 27918, + "bleached": 27919, + "outcrop": 27920, + "wildebeest": 27921, + "enchilada": 27922, + "nexus": 27923, + "##graduate": 27924, + "ebony": 27925, + "##itone": 27926, + "##atum": 27927, + "tosc": 27928, + "thatch": 27929, + "nodes": 27930, + "fireball": 27931, + "autob": 27932, + "harem": 27933, + "elong": 27934, + "dreamer": 27935, + "darede": 27936, + "pulitzer": 27937, + "weighed": 27938, + "raglan": 27939, + "echidon": 27940, + "phenomena": 27941, + "tolerant": 27942, + "alzheim": 27943, + "corm": 27944, + "tame": 27945, + "##send": 27946, + "##aber": 27947, + "houseboat": 27948, + "camino": 27949, + "postbox": 27950, + "northeastern": 27951, + "inclusion": 27952, + "registry": 27953, + "equity": 27954, + "embarr": 27955, + "coronado": 27956, + "018": 27957, + "mileage": 27958, + "crimea": 27959, + "imaginative": 27960, + "forensic": 27961, + "simultaneous": 27962, + "birdie": 27963, + "tiers": 27964, + "##ongs": 27965, + "clef": 27966, + "castor": 27967, + "servers": 27968, + "donned": 27969, + "sketchy": 27970, + "dulux": 27971, + "drywall": 27972, + "sensual": 27973, + "kimchi": 27974, + "sockets": 27975, + "zoological": 27976, + "unofficial": 27977, + "embarrass": 27978, + "ez": 27979, + "rall": 27980, + "##nake": 27981, + "##rays": 27982, + "butte": 27983, + "##coe": 27984, + "optimal": 27985, + "advisory": 27986, + "subsid": 27987, + "mere": 27988, + "reth": 27989, + "raging": 27990, + "##wave": 27991, + "ples": 27992, + "birding": 27993, + "flowery": 27994, + "headshot": 27995, + "seams": 27996, + "corruption": 27997, + "cabaret": 27998, + "presby": 27999, + "severn": 28000, + "basalt": 28001, + "solitude": 28002, + "rhomb": 28003, + "orangut": 28004, + "rubicon": 28005, + "kaiser": 28006, + "spatial": 28007, + "coordinates": 28008, + "orioles": 28009, + "substantial": 28010, + "##due": 28011, + "abode": 28012, + "lightroom": 28013, + "bravo": 28014, + "puja": 28015, + "narr": 28016, + "complexity": 28017, + "80th": 28018, + "peterborough": 28019, + "perpetual": 28020, + "ugliest": 28021, + "camellia": 28022, + "nud": 28023, + "##iate": 28024, + "##etter": 28025, + "##lude": 28026, + "##app": 28027, + "rockwell": 28028, + "centaur": 28029, + "bathed": 28030, + "petron": 28031, + "graphical": 28032, + "duel": 28033, + "relaxes": 28034, + "biennale": 28035, + "succession": 28036, + "ambient": 28037, + "kenyan": 28038, + "briefs": 28039, + "aldi": 28040, + "lodging": 28041, + "mourning": 28042, + "idyl": 28043, + "noticeable": 28044, + "nave": 28045, + "stard": 28046, + "##estown": 28047, + "enlisted": 28048, + "skim": 28049, + "vectorial": 28050, + "abat": 28051, + "blot": 28052, + "puree": 28053, + "matisse": 28054, + "thicker": 28055, + "inventor": 28056, + "omelette": 28057, + "inverness": 28058, + "britannia": 28059, + "1a": 28060, + "c10": 28061, + "##alo": 28062, + "grun": 28063, + "trink": 28064, + "scape": 28065, + "pei": 28066, + "photographie": 28067, + "helper": 28068, + "panam": 28069, + "coolers": 28070, + "motorcyc": 28071, + "raceway": 28072, + "constantine": 28073, + "mantisor": 28074, + "kitsch": 28075, + "jogger": 28076, + "enterprises": 28077, + "browsing": 28078, + "fearsome": 28079, + "descriptive": 28080, + "yolk": 28081, + "##jing": 28082, + "phy": 28083, + "telecaster": 28084, + "undis": 28085, + "rapha": 28086, + "nasal": 28087, + "##iii": 28088, + "adjusted": 28089, + "marshal": 28090, + "wentworth": 28091, + "grinning": 28092, + "polynesia": 28093, + "prickly": 28094, + "mell": 28095, + "wang": 28096, + "stony": 28097, + "manas": 28098, + "pric": 28099, + "schoolboy": 28100, + "##oded": 28101, + "##odrome": 28102, + "brickwork": 28103, + "burgl": 28104, + "moraine": 28105, + "bloodhound": 28106, + "directing": 28107, + "adjustment": 28108, + "bodh": 28109, + "pooja": 28110, + "##wanstein": 28111, + "cameroon": 28112, + "gudo": 28113, + "neuschwanstein": 28114, + "backpackers": 28115, + "flem": 28116, + "s10": 28117, + "##atur": 28118, + "##oupe": 28119, + "smear": 28120, + "admit": 28121, + "adoration": 28122, + "blink": 28123, + "acquisition": 28124, + "boarded": 28125, + "badlands": 28126, + "warmers": 28127, + "remembered": 28128, + "##giate": 28129, + "needlepoint": 28130, + "mannequins": 28131, + "unwanted": 28132, + "yucatan": 28133, + "4x": 28134, + "8x10": 28135, + "bism": 28136, + "s9": 28137, + "##fron": 28138, + "##unc": 28139, + "bladder": 28140, + "dexter": 28141, + "disorders": 28142, + "extras": 28143, + "##bowl": 28144, + "velcro": 28145, + "maa": 28146, + "fedex": 28147, + "investments": 28148, + "1893": 28149, + "uphill": 28150, + "aisles": 28151, + "gills": 28152, + "roder": 28153, + "##acial": 28154, + "delorean": 28155, + "pyj": 28156, + "preparations": 28157, + "bulldo": 28158, + "wrestlers": 28159, + "accompanying": 28160, + "mvp": 28161, + "kany": 28162, + "lore": 28163, + "woke": 28164, + "zak": 28165, + "anolis": 28166, + "mountainside": 28167, + "characterized": 28168, + "batu": 28169, + "dreamcatcher": 28170, + "prays": 28171, + "compliments": 28172, + "nineteenth": 28173, + "tweezers": 28174, + "friction": 28175, + "arid": 28176, + "whitewash": 28177, + "imdb": 28178, + "##indh": 28179, + "printmaking": 28180, + "annapolis": 28181, + "##cotch": 28182, + "biologist": 28183, + "##bao": 28184, + "snakeskin": 28185, + "gesturing": 28186, + "anthology": 28187, + "seemingly": 28188, + "fragrances": 28189, + "cockroach": 28190, + "butterscotch": 28191, + "##poon": 28192, + "spindle": 28193, + "undergo": 28194, + "gerber": 28195, + "126": 28196, + "novos": 28197, + "touchscreen": 28198, + "corded": 28199, + "fifteen": 28200, + "weakness": 28201, + "lakme": 28202, + "octagonal": 28203, + "hertfordshire": 28204, + "affirm": 28205, + "bonneville": 28206, + "ek": 28207, + "viva": 28208, + "##pla": 28209, + "##free": 28210, + "##amination": 28211, + "trest": 28212, + "##oured": 28213, + "mince": 28214, + "jackal": 28215, + "varieg": 28216, + "galveston": 28217, + "essen": 28218, + "kochi": 28219, + "essentially": 28220, + "shimano": 28221, + "shaman": 28222, + "elimin": 28223, + "horticulture": 28224, + "vascular": 28225, + "napoleonic": 28226, + "wc": 28227, + "##math": 28228, + "##enk": 28229, + "gru": 28230, + "implements": 28231, + "prosc": 28232, + "##ibir": 28233, + "tonga": 28234, + "praw": 28235, + "audubon": 28236, + "shrink": 28237, + "juicer": 28238, + "tanning": 28239, + "kris": 28240, + "requirement": 28241, + "yaing": 28242, + "unlocked": 28243, + "640": 28244, + "anfield": 28245, + "frill": 28246, + "artery": 28247, + "##antas": 28248, + "##ansas": 28249, + "acids": 28250, + "preced": 28251, + "partridges": 28252, + "hummings": 28253, + "filmmaker": 28254, + "tombs": 28255, + "eaters": 28256, + "pathfinder": 28257, + "501": 28258, + "finalist": 28259, + "lunchbox": 28260, + "##uracy": 28261, + "captains": 28262, + "assembling": 28263, + "saxophonist": 28264, + "tightly": 28265, + "orbiting": 28266, + "findings": 28267, + "belonged": 28268, + "firearms": 28269, + "endeavour": 28270, + "grenadines": 28271, + "sanitizer": 28272, + "subdivision": 28273, + "4d": 28274, + "jerk": 28275, + "nig": 28276, + "tester": 28277, + "£10": 28278, + "##oids": 28279, + "##rud": 28280, + "thon": 28281, + "2022": 28282, + "tract": 28283, + "scam": 28284, + "incline": 28285, + "sukh": 28286, + "zoos": 28287, + "revised": 28288, + "entertainer": 28289, + "airliner": 28290, + "analyzing": 28291, + "whiskers": 28292, + "restricted": 28293, + "dilap": 28294, + "circumst": 28295, + "auxiliary": 28296, + "unseen": 28297, + "##ifty": 28298, + "olde": 28299, + "##quatch": 28300, + "burg": 28301, + "baden": 28302, + "fastening": 28303, + "automob": 28304, + "ethics": 28305, + "eras": 28306, + "mallorca": 28307, + "integers": 28308, + "mollus": 28309, + "feminism": 28310, + "overwhelming": 28311, + "sedimentary": 28312, + "chirop": 28313, + "bacterial": 28314, + "novosibir": 28315, + "novosibirsk": 28316, + "mab": 28317, + "##ev": 28318, + "shiv": 28319, + "carson": 28320, + "pett": 28321, + "##arna": 28322, + "anemonic": 28323, + "grapevine": 28324, + "paramedic": 28325, + "76th": 28326, + "cheeked": 28327, + "insights": 28328, + "paneer": 28329, + "fw": 28330, + "##bana": 28331, + "ston": 28332, + "whipp": 28333, + "roset": 28334, + "##phi": 28335, + "paratr": 28336, + "decon": 28337, + "striker": 28338, + "warned": 28339, + "coffees": 28340, + "ladle": 28341, + "repos": 28342, + "shiplap": 28343, + "steppe": 28344, + "225": 28345, + "jams": 28346, + "gurgaon": 28347, + "sasquatch": 28348, + "judgment": 28349, + "snee": 28350, + "starship": 28351, + "broker": 28352, + "calab": 28353, + "masterplan": 28354, + "caulie": 28355, + "reporters": 28356, + "fulf": 28357, + "sprinkler": 28358, + "scramble": 28359, + "preserving": 28360, + "harlequin": 28361, + "oaxaca": 28362, + "dana": 28363, + "##wah": 28364, + "##stad": 28365, + "lei": 28366, + "##ensen": 28367, + "ballistic": 28368, + "warp": 28369, + "aftermath": 28370, + "relate": 28371, + "himachal": 28372, + "##affe": 28373, + "appearances": 28374, + "lido": 28375, + "zucchi": 28376, + "lincolnshire": 28377, + "dormitory": 28378, + "coordinated": 28379, + "sheepskin": 28380, + "arguing": 28381, + "lightbulb": 28382, + "dilapidated": 28383, + "nab": 28384, + "tk": 28385, + "vets": 28386, + "token": 28387, + "##intosh": 28388, + "lend": 28389, + "unchart": 28390, + "periw": 28391, + "patty": 28392, + "slid": 28393, + "salary": 28394, + "bankrupt": 28395, + "lacquered": 28396, + "juliet": 28397, + "cyclocross": 28398, + "##bowski": 28399, + "corbis": 28400, + "porcupine": 28401, + "uncharted": 28402, + "boos": 28403, + "##eous": 28404, + "##nap": 28405, + "##eading": 28406, + "##umel": 28407, + "sevens": 28408, + "adder": 28409, + "decanter": 28410, + "empires": 28411, + "breakthrough": 28412, + "accuracy": 28413, + "velo": 28414, + "bibli": 28415, + "gongs": 28416, + "injections": 28417, + "birdhouses": 28418, + "px": 28419, + "qantas": 28420, + "wain": 28421, + "threats": 28422, + "##anese": 28423, + "##urf": 28424, + "##sequ": 28425, + "##ulf": 28426, + "##avia": 28427, + "reviewing": 28428, + "radioactive": 28429, + "explorers": 28430, + "forbes": 28431, + "dismiss": 28432, + "iq": 28433, + "rak": 28434, + "##anthus": 28435, + "##elo": 28436, + "buster": 28437, + "lebowski": 28438, + "##othec": 28439, + "apothec": 28440, + "margin": 28441, + "butch": 28442, + "amusing": 28443, + "infect": 28444, + "beetroot": 28445, + "gyms": 28446, + "talon": 28447, + "sorrent": 28448, + "cushioned": 28449, + "psalms": 28450, + "lusc": 28451, + "zam": 28452, + "##9th": 28453, + "##entine": 28454, + "proves": 28455, + "##usted": 28456, + "##dend": 28457, + "sunr": 28458, + "landfill": 28459, + "hatter": 28460, + "palis": 28461, + "malam": 28462, + "sidewalks": 28463, + "finalists": 28464, + "collectable": 28465, + "gazer": 28466, + "armadil": 28467, + "cooperative": 28468, + "gnomes": 28469, + "mckin": 28470, + "oeuvres": 28471, + "newsletter": 28472, + "chiropract": 28473, + "eman": 28474, + "sow": 28475, + "##lide": 28476, + "##enton": 28477, + "trailing": 28478, + "##ickers": 28479, + "skaters": 28480, + "classmates": 28481, + "##ycar": 28482, + "origa": 28483, + "banjos": 28484, + "halle": 28485, + "proteins": 28486, + "occupational": 28487, + "shimla": 28488, + "prophetic": 28489, + "icebergs": 28490, + "hurricanes": 28491, + "vallarta": 28492, + "threshold": 28493, + "kays": 28494, + "lact": 28495, + "smarter": 28496, + "stylus": 28497, + "hebr": 28498, + "fuller": 28499, + "genoa": 28500, + "gibbons": 28501, + "hampi": 28502, + "phillies": 28503, + "cozumel": 28504, + "pronoun": 28505, + "androgynous": 28506, + "petronas": 28507, + "cid": 28508, + "fists": 28509, + "golds": 28510, + "zermatt": 28511, + "bose": 28512, + "boasting": 28513, + "clears": 28514, + "amaze": 28515, + "##onaut": 28516, + "wondrous": 28517, + "troubles": 28518, + "daim": 28519, + "gaia": 28520, + "carlisle": 28521, + "adventurer": 28522, + "dixie": 28523, + "algeria": 28524, + "cerebral": 28525, + "luscious": 28526, + "eug": 28527, + "##cla": 28528, + "blazing": 28529, + "seph": 28530, + "##inders": 28531, + "strut": 28532, + "milit": 28533, + "sported": 28534, + "##occhi": 28535, + "burial": 28536, + "discrim": 28537, + "folly": 28538, + "quilter": 28539, + "cribs": 28540, + "documentaries": 28541, + "hof": 28542, + "promotions": 28543, + "waltz": 28544, + "targeted": 28545, + "census": 28546, + "idyllic": 28547, + "##alee": 28548, + "congregation": 28549, + "quilling": 28550, + "abas": 28551, + "sermon": 28552, + "##bally": 28553, + "willis": 28554, + "lakewood": 28555, + "cinque": 28556, + "millionaire": 28557, + "omn": 28558, + "occupation": 28559, + "utilize": 28560, + "comprises": 28561, + "soprano": 28562, + "aperture": 28563, + "flemish": 28564, + "periwinkle": 28565, + "apothecary": 28566, + "lbs": 28567, + "##plus": 28568, + "##x17": 28569, + "##cock": 28570, + "shibu": 28571, + "co2": 28572, + "trot": 28573, + "lobe": 28574, + "beachside": 28575, + "armrest": 28576, + "elegantly": 28577, + "utter": 28578, + "cheetang": 28579, + "confirmation": 28580, + "eccentric": 28581, + "fundamentals": 28582, + "fifties": 28583, + "##uu": 28584, + "impe": 28585, + "fibr": 28586, + "marcel": 28587, + "marinated": 28588, + "usable": 28589, + "##ftan": 28590, + "serena": 28591, + "flowerpot": 28592, + "burbank": 28593, + "tailoring": 28594, + "polishing": 28595, + "iguad": 28596, + "baldwin": 28597, + "trilob": 28598, + "maxima": 28599, + "instructors": 28600, + "bilbao": 28601, + "kilometres": 28602, + "olaf": 28603, + "sacks": 28604, + "##qing": 28605, + "##ipers": 28606, + "##quite": 28607, + "##ontas": 28608, + "dressy": 28609, + "mins": 28610, + "jackfruit": 28611, + "infer": 28612, + "captive": 28613, + "malvern": 28614, + "thinker": 28615, + "gotten": 28616, + "morton": 28617, + "driller": 28618, + "017": 28619, + "armadillo": 28620, + "etihad": 28621, + "tuner": 28622, + "dingle": 28623, + "chilean": 28624, + "delayed": 28625, + "lx": 28626, + "##fiction": 28627, + "spades": 28628, + "animations": 28629, + "swoon": 28630, + "anglers": 28631, + "bandic": 28632, + "odessa": 28633, + "maraud": 28634, + "aligned": 28635, + "bologna": 28636, + "avalanche": 28637, + "algonquin": 28638, + "##gly": 28639, + "exodus": 28640, + "proceeds": 28641, + "##boot": 28642, + "moma": 28643, + "audit": 28644, + "microf": 28645, + "cheetars": 28646, + "directs": 28647, + "adjusts": 28648, + "hubs": 28649, + "molasses": 28650, + "ensures": 28651, + "chalice": 28652, + "navigator": 28653, + "slaughter": 28654, + "ppt": 28655, + "##inting": 28656, + "##sta": 28657, + "perce": 28658, + "valence": 28659, + "caicos": 28660, + "bullion": 28661, + "pixiv": 28662, + "donegal": 28663, + "lulu": 28664, + "accidentally": 28665, + "acceleration": 28666, + "consequences": 28667, + "braw": 28668, + "writ": 28669, + "corro": 28670, + "ego": 28671, + "freshness": 28672, + "1700": 28673, + "uranus": 28674, + "sulf": 28675, + "tumble": 28676, + "chandigarh": 28677, + "genealogy": 28678, + "opaque": 28679, + "carab": 28680, + "carac": 28681, + "bf": 28682, + "foul": 28683, + "##geable": 28684, + "##ifted": 28685, + "##istles": 28686, + "conting": 28687, + "rockef": 28688, + "highways": 28689, + "turks": 28690, + "valance": 28691, + "##ahontas": 28692, + "creamer": 28693, + "techno": 28694, + "ramps": 28695, + "rumble": 28696, + "##wyn": 28697, + "marinade": 28698, + "euph": 28699, + "renewed": 28700, + "responded": 28701, + "redwoods": 28702, + "ascending": 28703, + "pocahontas": 28704, + "sagu": 28705, + "x5": 28706, + "##cr": 28707, + "tha": 28708, + "outward": 28709, + "beaum": 28710, + "##ondale": 28711, + "sumo": 28712, + "motorists": 28713, + "informative": 28714, + "watershed": 28715, + "fenway": 28716, + "marmos": 28717, + "speake": 28718, + "recruits": 28719, + "bulldozer": 28720, + "bandicoot": 28721, + "##x18": 28722, + "##bones": 28723, + "smaug": 28724, + "##into": 28725, + "lok": 28726, + "reson": 28727, + "craze": 28728, + "starlight": 28729, + "workouts": 28730, + "##dynam": 28731, + "undercover": 28732, + "subtra": 28733, + "develops": 28734, + "bolton": 28735, + "glacial": 28736, + "richards": 28737, + "figuris": 28738, + "shorthaired": 28739, + "paragl": 28740, + "zagreb": 28741, + "tempest": 28742, + "kyrgyzstan": 28743, + "valkyrie": 28744, + "automobiles": 28745, + "munn": 28746, + "withstand": 28747, + "chard": 28748, + "amino": 28749, + "frem": 28750, + "psychologist": 28751, + "cereals": 28752, + "skipping": 28753, + "meander": 28754, + "philippe": 28755, + "dales": 28756, + "inmates": 28757, + "manger": 28758, + "##obab": 28759, + "wrappers": 28760, + "lego®": 28761, + "alternating": 28762, + "converts": 28763, + "fringed": 28764, + "hagia": 28765, + "reddit": 28766, + "allegory": 28767, + "tolerance": 28768, + "inflammation": 28769, + "constantinople": 28770, + "rockefeller": 28771, + "020": 28772, + "h2": 28773, + "ting": 28774, + "##amine": 28775, + "enroll": 28776, + "interval": 28777, + "##umburg": 28778, + "tenor": 28779, + "ostrict": 28780, + "019": 28781, + "cheetahs": 28782, + "wheaton": 28783, + "odi": 28784, + "licence": 28785, + "certificates": 28786, + "dazzle": 28787, + "handicraft": 28788, + "cope": 28789, + "kak": 28790, + "baobab": 28791, + "desirable": 28792, + "alright": 28793, + "strewn": 28794, + "armors": 28795, + "breakup": 28796, + "hyenna": 28797, + "libert": 28798, + "embark": 28799, + "gearbox": 28800, + "melanch": 28801, + "publisher": 28802, + "concerning": 28803, + "seventies": 28804, + "cultivated": 28805, + "asymmetric": 28806, + "combustion": 28807, + "noun": 28808, + "##cor": 28809, + "##bacca": 28810, + "graces": 28811, + "barney": 28812, + "ella": 28813, + "halt": 28814, + "kitts": 28815, + "138": 28816, + "motivate": 28817, + "hubei": 28818, + "sewage": 28819, + "1892": 28820, + "chinch": 28821, + "##combe": 28822, + "respectively": 28823, + "bookends": 28824, + "das": 28825, + "dort": 28826, + "##sl": 28827, + "##gus": 28828, + "##mor": 28829, + "##mins": 28830, + "##utic": 28831, + "canisters": 28832, + "usd": 28833, + "##ansk": 28834, + "heist": 28835, + "menorah": 28836, + "emission": 28837, + "impea": 28838, + "careg": 28839, + "cinco": 28840, + "ambience": 28841, + "motivated": 28842, + "458": 28843, + "yankees": 28844, + "sorrento": 28845, + "lager": 28846, + "sall": 28847, + "##fork": 28848, + "##aby": 28849, + "loon": 28850, + "centr": 28851, + "needing": 28852, + "workings": 28853, + "guyana": 28854, + "socialist": 28855, + "changi": 28856, + "weathering": 28857, + "snapper": 28858, + "armadi": 28859, + "wwi": 28860, + "gnocchi": 28861, + "ancestry": 28862, + "subtract": 28863, + "internationally": 28864, + "leth": 28865, + "##yle": 28866, + "dressers": 28867, + "blenheim": 28868, + "slated": 28869, + "maybach": 28870, + "remot": 28871, + "seasonings": 28872, + "explode": 28873, + "barbican": 28874, + "tarara": 28875, + "funnies": 28876, + "warthoger": 28877, + "cisco": 28878, + "peregr": 28879, + "sepulchre": 28880, + "fanny": 28881, + "jedd": 28882, + "lut": 28883, + "##crest": 28884, + "shored": 28885, + "##amom": 28886, + "##imity": 28887, + "phonics": 28888, + "spins": 28889, + "soar": 28890, + "artistry": 28891, + "townhomes": 28892, + "knitwear": 28893, + "males": 28894, + "inkjet": 28895, + "nutty": 28896, + "sadd": 28897, + "saddles": 28898, + "sorties": 28899, + "martens": 28900, + "executives": 28901, + "gypsum": 28902, + "definitions": 28903, + "barton": 28904, + "accountant": 28905, + "inuy": 28906, + "evacuation": 28907, + "remnants": 28908, + "intriguing": 28909, + "goaltender": 28910, + "jeddah": 28911, + "huss": 28912, + "##ombe": 28913, + "##advant": 28914, + "##acan": 28915, + "##uckle": 28916, + "sald": 28917, + "riverwalk": 28918, + "masra": 28919, + "pointers": 28920, + "paella": 28921, + "numerology": 28922, + "fitzroy": 28923, + "rialto": 28924, + "gis": 28925, + "hain": 28926, + "ios": 28927, + "ibe": 28928, + "utop": 28929, + "##erpt": 28930, + "##etical": 28931, + "##acht": 28932, + "afb": 28933, + "evenings": 28934, + "rotting": 28935, + "groovy": 28936, + "chewbacca": 28937, + "ephem": 28938, + "baileys": 28939, + "presbyter": 28940, + "lur": 28941, + "##icion": 28942, + "chey": 28943, + "spats": 28944, + "grom": 28945, + "disadvant": 28946, + "allies": 28947, + "##plane": 28948, + "expose": 28949, + "aquaman": 28950, + "excerpt": 28951, + "fasten": 28952, + "divides": 28953, + "condor": 28954, + "jammu": 28955, + "consistency": 28956, + "gambia": 28957, + "hotdogs": 28958, + "meriden": 28959, + "speakeasy": 28960, + "##san": 28961, + "##chem": 28962, + "arp": 28963, + "acer": 28964, + "doorstep": 28965, + "relating": 28966, + "impair": 28967, + "macadam": 28968, + "totes": 28969, + "primes": 28970, + "##azzle": 28971, + "descriptions": 28972, + "lilo": 28973, + "carroll": 28974, + "evolve": 28975, + "settled": 28976, + "legislative": 28977, + "congestion": 28978, + "hoffman": 28979, + "cowardly": 28980, + "##rh": 28981, + "inquisition": 28982, + "heuer": 28983, + "slipp": 28984, + "infants": 28985, + "complements": 28986, + "shelt": 28987, + "samoa": 28988, + "scarab": 28989, + "##wright": 28990, + "payments": 28991, + "investor": 28992, + "temperate": 28993, + "crouching": 28994, + "investigating": 28995, + "ashtray": 28996, + "pinstripe": 28997, + "rgb": 28998, + "tiling": 28999, + "##pto": 29000, + "##iated": 29001, + "##beth": 29002, + "baik": 29003, + "criss": 29004, + "southport": 29005, + "walkie": 29006, + "dukes": 29007, + "sinister": 29008, + "batsman": 29009, + "obscured": 29010, + "grayscale": 29011, + "jalap": 29012, + "proximity": 29013, + "malamute": 29014, + "shoreditch": 29015, + "inuyasha": 29016, + "presbyterian": 29017, + "##3rd": 29018, + "##mies": 29019, + "##omac": 29020, + "sprigs": 29021, + "merit": 29022, + "lowrider": 29023, + "pyeong": 29024, + "sixty": 29025, + "necessarily": 29026, + "municipality": 29027, + "opinions": 29028, + "gutter": 29029, + "orphanage": 29030, + "galloping": 29031, + "fraternity": 29032, + "vaux": 29033, + "##entive": 29034, + "remb": 29035, + "colob": 29036, + "retiring": 29037, + "annot": 29038, + "##ocia": 29039, + "thrust": 29040, + "##cheon": 29041, + "loafs": 29042, + "divergent": 29043, + "seventeen": 29044, + "outrageous": 29045, + "rift": 29046, + "vang": 29047, + "##pen": 29048, + "##oula": 29049, + "phosph": 29050, + "##osla": 29051, + "surplus": 29052, + "##awak": 29053, + "yearly": 29054, + "ama": 29055, + "supporter": 29056, + "114": 29057, + "torre": 29058, + "lounging": 29059, + "functioning": 29060, + "matterhorn": 29061, + "accommodations": 29062, + "dorchester": 29063, + "couscous": 29064, + "evaluate": 29065, + "xylophone": 29066, + "clementine": 29067, + "pledge": 29068, + "##fle": 29069, + "amelia": 29070, + "potomac": 29071, + "##letta": 29072, + "##venor": 29073, + "irving": 29074, + "sledding": 29075, + "kneel": 29076, + "lakeland": 29077, + "grosvenor": 29078, + "fasting": 29079, + "kota": 29080, + "##harm": 29081, + "##tails": 29082, + "manns": 29083, + "craig": 29084, + "refin": 29085, + "reli": 29086, + "distances": 29087, + "homme": 29088, + "pumped": 29089, + "##warding": 29090, + "mahi": 29091, + "determination": 29092, + "ovation": 29093, + "krabi": 29094, + "1⁄": 29095, + "csa": 29096, + "lps": 29097, + "tard": 29098, + "itc": 29099, + "##ums": 29100, + "seize": 29101, + "centur": 29102, + "juno": 29103, + "acclaimed": 29104, + "profiles": 29105, + "aeros": 29106, + "60l": 29107, + "65th": 29108, + "salvage": 29109, + "vulgar": 29110, + "dangling": 29111, + "lenox": 29112, + "fotografia": 29113, + "acknowled": 29114, + "rish": 29115, + "phon": 29116, + "##quire": 29117, + "creators": 29118, + "workflow": 29119, + "tyrol": 29120, + "aviary": 29121, + "racism": 29122, + "cultured": 29123, + "spaced": 29124, + "professions": 29125, + "kingdoms": 29126, + "cortex": 29127, + "subtraction": 29128, + "nac": 29129, + "##pany": 29130, + "##abar": 29131, + "mania": 29132, + "decept": 29133, + "topographic": 29134, + "everg": 29135, + "rely": 29136, + "anybody": 29137, + "somalia": 29138, + "confusion": 29139, + "##omba": 29140, + "positioning": 29141, + "commandments": 29142, + "tenants": 29143, + "fulham": 29144, + "thermos": 29145, + "gelatin": 29146, + "preventing": 29147, + "concentric": 29148, + "bratz": 29149, + "housewives": 29150, + "4s": 29151, + "nak": 29152, + "##dal": 29153, + "flipped": 29154, + "woodblock": 29155, + "handprint": 29156, + "cardiac": 29157, + "casp": 29158, + "wellbe": 29159, + "idiot": 29160, + "deforestation": 29161, + "hua": 29162, + "orchards": 29163, + "journalists": 29164, + "uppercase": 29165, + "dugout": 29166, + "slap": 29167, + "tricolor": 29168, + "domesticated": 29169, + "bellagio": 29170, + "sconces": 29171, + "potluck": 29172, + "schaumburg": 29173, + "40s": 29174, + "f2": 29175, + "idle": 29176, + "miff": 29177, + "fors": 29178, + "prow": 29179, + "##anka": 29180, + "typh": 29181, + "birthplace": 29182, + "volum": 29183, + "##retta": 29184, + "audiences": 29185, + "drumsticks": 29186, + "crawler": 29187, + "rotunda": 29188, + "docking": 29189, + "fuels": 29190, + "##imples": 29191, + "binds": 29192, + "hilux": 29193, + "piers": 29194, + "drunk": 29195, + "megan": 29196, + "ballon": 29197, + "stickman": 29198, + "forestry": 29199, + "irides": 29200, + "sparklers": 29201, + "juda": 29202, + "leek": 29203, + "66th": 29204, + "ozarks": 29205, + "##rates": 29206, + "frontage": 29207, + "grounded": 29208, + "larva": 29209, + "decl": 29210, + "overfl": 29211, + "spew": 29212, + "lapland": 29213, + "fairway": 29214, + "urine": 29215, + "spaghett": 29216, + "ambulances": 29217, + "gaining": 29218, + "mattresses": 29219, + "bouldering": 29220, + "bisque": 29221, + "comedians": 29222, + "wellbeing": 29223, + "vida": 29224, + "sco": 29225, + "snowdon": 29226, + "workforce": 29227, + "##ierville": 29228, + "lowland": 29229, + "cubist": 29230, + "muscat": 29231, + "maxx": 29232, + "acadia": 29233, + "therape": 29234, + "stampede": 29235, + "accompany": 29236, + "marmota": 29237, + "catalogs": 29238, + "vigil": 29239, + "boudoir": 29240, + "dortmund": 29241, + "hc": 29242, + "klim": 29243, + "xp": 29244, + "##zone": 29245, + "shu": 29246, + "blackened": 29247, + "crocs": 29248, + "armory": 29249, + "obey": 29250, + "porl": 29251, + "specification": 29252, + "alphab": 29253, + "verandah": 29254, + "broadcasting": 29255, + "nonfiction": 29256, + "junkyard": 29257, + "pembrok": 29258, + "fy": 29259, + "gup": 29260, + "teep": 29261, + "wats": 29262, + "##x24": 29263, + "##bars": 29264, + "isfahan": 29265, + "periph": 29266, + "hears": 29267, + "awake": 29268, + "100m": 29269, + "attaches": 29270, + "tutor": 29271, + "deepest": 29272, + "expressway": 29273, + "tanj": 29274, + "woolly": 29275, + "ivan": 29276, + "##ielding": 29277, + "##juice": 29278, + "aloft": 29279, + "pampered": 29280, + "trestle": 29281, + "1⁄2": 29282, + "pamp": 29283, + "##iem": 29284, + "asb": 29285, + "breaching": 29286, + "collegiate": 29287, + "##ilingual": 29288, + "lawyers": 29289, + "answered": 29290, + "volunteering": 29291, + "georges": 29292, + "pansies": 29293, + "lazio": 29294, + "gaps": 29295, + "culver": 29296, + "baguet": 29297, + "dunks": 29298, + "surveys": 29299, + "imposed": 29300, + "kombu": 29301, + "elys": 29302, + "beaufort": 29303, + "coupons": 29304, + "macadamia": 29305, + "310": 29306, + "gts": 29307, + "waker": 29308, + "##onica": 29309, + "##arie": 29310, + "loaves": 29311, + "leica": 29312, + "downing": 29313, + "roost": 29314, + "geelong": 29315, + "brightness": 29316, + "spanner": 29317, + "shutdown": 29318, + "ramada": 29319, + "strapped": 29320, + "puffs": 29321, + "bibs": 29322, + "herbiv": 29323, + "scrappy": 29324, + "kiliman": 29325, + "watercolours": 29326, + "teepee": 29327, + "nog": 29328, + "##uate": 29329, + "##atars": 29330, + "toeed": 29331, + "##emaster": 29332, + "charred": 29333, + "memo": 29334, + "okra": 29335, + "ashton": 29336, + "nickn": 29337, + "intricately": 29338, + "conjunction": 29339, + "arguably": 29340, + "chenille": 29341, + "daydream": 29342, + "cretaceous": 29343, + "cures": 29344, + "ionic": 29345, + "##enberg": 29346, + "reset": 29347, + "cappad": 29348, + "readily": 29349, + "1876": 29350, + "##lusive": 29351, + "institutions": 29352, + "droplet": 29353, + "perspect": 29354, + "highlander": 29355, + "kelp": 29356, + "conjuring": 29357, + "ardu": 29358, + "##unes": 29359, + "nep": 29360, + "aloud": 29361, + "gry": 29362, + "marse": 29363, + "resulted": 29364, + "hangzhou": 29365, + "carpath": 29366, + "institutional": 29367, + "hazards": 29368, + "croatian": 29369, + "ryokan": 29370, + "inequality": 29371, + "kanch": 29372, + "kilda": 29373, + "wv": 29374, + "zaha": 29375, + "##sing": 29376, + "##base": 29377, + "reboot": 29378, + "disrupt": 29379, + "acura": 29380, + "baritone": 29381, + "cardamom": 29382, + "inferno": 29383, + "avion": 29384, + "accused": 29385, + "powerhouse": 29386, + "pickers": 29387, + "irises": 29388, + "conversions": 29389, + "armadie": 29390, + "estimates": 29391, + "communicating": 29392, + "checkerboard": 29393, + "##x1600": 29394, + "##hp": 29395, + "##7th": 29396, + "##bler": 29397, + "sts": 29398, + "##ooga": 29399, + "salford": 29400, + "165": 29401, + "##burgr": 29402, + "theatres": 29403, + "personalities": 29404, + "chessboard": 29405, + "sprouting": 29406, + "strangers": 29407, + "articulated": 29408, + "remotely": 29409, + "rembrand": 29410, + "nurt": 29411, + "sank": 29412, + "yves": 29413, + "##runner": 29414, + "##cence": 29415, + "##antly": 29416, + "unice": 29417, + "walton": 29418, + "##othermal": 29419, + "corral": 29420, + "fullerton": 29421, + "##woods": 29422, + "massey": 29423, + "subsequ": 29424, + "slumber": 29425, + "erases": 29426, + "papy": 29427, + "kilts": 29428, + "axolo": 29429, + "##cadero": 29430, + "aeroplane": 29431, + "affirmations": 29432, + "fives": 29433, + "noosa": 29434, + "r2": 29435, + "##opia": 29436, + "206": 29437, + "traced": 29438, + "abn": 29439, + "reserves": 29440, + "agile": 29441, + "americans": 29442, + "digimon": 29443, + "insan": 29444, + "emerges": 29445, + "ramstein": 29446, + "creeping": 29447, + "overlaying": 29448, + "randomly": 29449, + "scallops": 29450, + "##avera": 29451, + "lansing": 29452, + "epitome": 29453, + "soiree": 29454, + "guerrilla": 29455, + "whits": 29456, + "chau": 29457, + "tron": 29458, + "vega": 29459, + "galla": 29460, + "tarp": 29461, + "primrose": 29462, + "switching": 29463, + "slaves": 29464, + "flourish": 29465, + "needlework": 29466, + "hemi": 29467, + "kiln": 29468, + "venomous": 29469, + "henley": 29470, + "dodger": 29471, + "fjords": 29472, + "gandhi": 29473, + "aes": 29474, + "jess": 29475, + "tinder": 29476, + "##hat": 29477, + "thug": 29478, + "##arity": 29479, + "exiting": 29480, + "##ustice": 29481, + "hep": 29482, + "winder": 29483, + "southw": 29484, + "envision": 29485, + "applic": 29486, + "sarawak": 29487, + "rvs": 29488, + "echide": 29489, + "##seye": 29490, + "wavel": 29491, + "candlestick": 29492, + "wawona": 29493, + "microfiber": 29494, + "pembrokeshire": 29495, + "gop": 29496, + "##jaro": 29497, + "##ente": 29498, + "electr": 29499, + "remover": 29500, + "valletta": 29501, + "fours": 29502, + "sanitary": 29503, + "samara": 29504, + "editors": 29505, + "brookfield": 29506, + "demonstrations": 29507, + "strollers": 29508, + "utilizing": 29509, + "##journal": 29510, + "slippery": 29511, + "kilimanjaro": 29512, + "wob": 29513, + "##yte": 29514, + "fluted": 29515, + "outpost": 29516, + "largemouth": 29517, + "bartlet": 29518, + "guern": 29519, + "evident": 29520, + "salmat": 29521, + "himalaya": 29522, + "blogging": 29523, + "hugh": 29524, + "imagined": 29525, + "destroying": 29526, + "truths": 29527, + "maintains": 29528, + "albanian": 29529, + "rejected": 29530, + "snarling": 29531, + "alzheimer": 29532, + "rembrandt": 29533, + "nal": 29534, + "frit": 29535, + "frang": 29536, + "resign": 29537, + "moonlit": 29538, + "ankles": 29539, + "suspicious": 29540, + "requests": 29541, + "circumference": 29542, + "buccaneers": 29543, + "bly": 29544, + "fd": 29545, + "nue": 29546, + "##mina": 29547, + "##will": 29548, + "##vol": 29549, + "##owa": 29550, + "anise": 29551, + "flirty": 29552, + "perks": 29553, + "##irect": 29554, + "juneau": 29555, + "expense": 29556, + "moonstone": 29557, + "protru": 29558, + "eldor": 29559, + "commands": 29560, + "saxony": 29561, + "durango": 29562, + "burrs": 29563, + "moistur": 29564, + "squeezing": 29565, + "criticism": 29566, + "intellectual": 29567, + "frangip": 29568, + "lds": 29569, + "vila": 29570, + "##eroy": 29571, + "##amer": 29572, + "traders": 29573, + "photojournal": 29574, + "intertw": 29575, + "penns": 29576, + "egrea": 29577, + "humanitarian": 29578, + "angelina": 29579, + "dually": 29580, + "drool": 29581, + "##iosis": 29582, + "honorary": 29583, + "mosquitoes": 29584, + "partnered": 29585, + "lorry": 29586, + "avoiding": 29587, + "memoirs": 29588, + "counselor": 29589, + "perspectives": 29590, + "kund": 29591, + "vid": 29592, + "vail": 29593, + "##uv": 29594, + "##uke": 29595, + "charmed": 29596, + "sumatra": 29597, + "retention": 29598, + "comma": 29599, + "toms": 29600, + "elemental": 29601, + "ambitious": 29602, + "caulium": 29603, + "kimberley": 29604, + "radiology": 29605, + "##steps": 29606, + "nazi": 29607, + "chromosomes": 29608, + "compliance": 29609, + "varnish": 29610, + "detergent": 29611, + "##fighter": 29612, + "scanned": 29613, + "rainwater": 29614, + "panas": 29615, + "digs": 29616, + "breakfasts": 29617, + "assisting": 29618, + "venison": 29619, + "sema": 29620, + "boutiques": 29621, + "munsters": 29622, + "indicators": 29623, + "dormer": 29624, + "stilts": 29625, + "criteria": 29626, + "caracas": 29627, + "##kel": 29628, + "##stool": 29629, + "outgoing": 29630, + "als": 29631, + "motley": 29632, + "expor": 29633, + "gorgon": 29634, + "leaded": 29635, + "neighboring": 29636, + "sometime": 29637, + "sympos": 29638, + "cucumn": 29639, + "humidity": 29640, + "shibuya": 29641, + "caspian": 29642, + "birdb": 29643, + "gior": 29644, + "mala": 29645, + "nl": 29646, + "yunn": 29647, + "chil": 29648, + "mocc": 29649, + "peels": 29650, + "acute": 29651, + "maga": 29652, + "12v": 29653, + "##occhio": 29654, + "venge": 29655, + "crochen": 29656, + "fastback": 29657, + "babois": 29658, + "merchants": 29659, + "gamma": 29660, + "voyages": 29661, + "montego": 29662, + "mackin": 29663, + "jacksons": 29664, + "accomplished": 29665, + "raffle": 29666, + "aq": 29667, + "kett": 29668, + "tlc": 29669, + "##lean": 29670, + "##string": 29671, + "scub": 29672, + "redback": 29673, + "##plete": 29674, + "opul": 29675, + "noida": 29676, + "warn": 29677, + "cellars": 29678, + "##ocus": 29679, + "pinocchio": 29680, + "answering": 29681, + "luft": 29682, + "appetite": 29683, + "dominion": 29684, + "knocking": 29685, + "macaq": 29686, + "remembers": 29687, + "coordinating": 29688, + "capuch": 29689, + "drowning": 29690, + "wacky": 29691, + "kombucha": 29692, + "wavelength": 29693, + "bartlett": 29694, + "yunnan": 29695, + "340": 29696, + "bp": 29697, + "nix": 29698, + "wade": 29699, + "xd": 29700, + "##tons": 29701, + "salini": 29702, + "maggie": 29703, + "runaway": 29704, + "beetlejuice": 29705, + "palma": 29706, + "divisions": 29707, + "coinc": 29708, + "enclave": 29709, + "caribou": 29710, + "sensible": 29711, + "squatting": 29712, + "ajax": 29713, + "chongqing": 29714, + "iridescent": 29715, + "460": 29716, + "kag": 29717, + "vader": 29718, + "##pread": 29719, + "stated": 29720, + "toon": 29721, + "larch": 29722, + "maxwell": 29723, + "narwh": 29724, + "tripodic": 29725, + "gilg": 29726, + "iwo": 29727, + "ihsa": 29728, + "halftime": 29729, + "departing": 29730, + "wallingford": 29731, + "ravioli": 29732, + "prawns": 29733, + "dab": 29734, + "nb": 29735, + "vat": 29736, + "stov": 29737, + "brio": 29738, + "##antan": 29739, + "meyer": 29740, + "amps": 29741, + "croft": 29742, + "quotable": 29743, + "motos": 29744, + "curt": 29745, + "chico": 29746, + "dashing": 29747, + "hazy": 29748, + "virginia": 29749, + "voters": 29750, + "pursue": 29751, + "pamph": 29752, + "whitetail": 29753, + "administrator": 29754, + "shawnee": 29755, + "symposium": 29756, + "cura": 29757, + "kalk": 29758, + "x3": 29759, + "##lion": 29760, + "##isman": 29761, + "roster": 29762, + "mariner": 29763, + "botox": 29764, + "amy": 29765, + "preserves": 29766, + "sizz": 29767, + "mastering": 29768, + "learners": 29769, + "bordered": 29770, + "##agonist": 29771, + "saraje": 29772, + "orbits": 29773, + "##lias": 29774, + "f150": 29775, + "dorado": 29776, + "commodore": 29777, + "bluebells": 29778, + "umpire": 29779, + "sarajevo": 29780, + "benchmark": 29781, + "riley": 29782, + "##grad": 29783, + "scones": 29784, + "abdu": 29785, + "agnes": 29786, + "antal": 29787, + "titus": 29788, + "duckling": 29789, + "irvine": 29790, + "practiced": 29791, + "labeling": 29792, + "ketches": 29793, + "injustice": 29794, + "predomin": 29795, + "luis": 29796, + "##x800": 29797, + "##x900": 29798, + "baubles": 29799, + "brainstorming": 29800, + "24x36": 29801, + "cappadocia": 29802, + "fron": 29803, + "convin": 29804, + "horde": 29805, + "elig": 29806, + "combing": 29807, + "adopting": 29808, + "maul": 29809, + "semic": 29810, + "kickbox": 29811, + "macker": 29812, + "phillip": 29813, + "globally": 29814, + "uncommon": 29815, + "nevada": 29816, + "evangelion": 29817, + "lofoten": 29818, + "miserables": 29819, + "busan": 29820, + "##cote": 29821, + "##enium": 29822, + "stour": 29823, + "##ayt": 29824, + "ally": 29825, + "marys": 29826, + "barrett": 29827, + "spaceman": 29828, + "keypad": 29829, + "matcha": 29830, + "heroic": 29831, + "speedy": 29832, + "saturdays": 29833, + "collagen": 29834, + "##garh": 29835, + "bobble": 29836, + "spectacles": 29837, + "dynamite": 29838, + "qualitative": 29839, + "ellipt": 29840, + "fertile": 29841, + "piag": 29842, + "widespread": 29843, + "dismantled": 29844, + "wimpy": 29845, + "zayed": 29846, + "vauxhall": 29847, + "unicef": 29848, + "dt": 29849, + "pikes": 29850, + "zhang": 29851, + "chattan": 29852, + "motive": 29853, + "##obil": 29854, + "refused": 29855, + "keyring": 29856, + "elsa": 29857, + "picn": 29858, + "cymb": 29859, + "kenworth": 29860, + "overlooked": 29861, + "updating": 29862, + "bombard": 29863, + "chimps": 29864, + "toughest": 29865, + "convoy": 29866, + "outsiders": 29867, + "madhub": 29868, + "daredevil": 29869, + "lest": 29870, + "nus": 29871, + "£4": 29872, + "##assee": 29873, + "grail": 29874, + "##atted": 29875, + "compens": 29876, + "minsk": 29877, + "pastoral": 29878, + "invo": 29879, + "protester": 29880, + "aspire": 29881, + "scandinavia": 29882, + "surrealist": 29883, + "boosting": 29884, + "bouncing": 29885, + "cognitive": 29886, + "kotor": 29887, + "sval": 29888, + "##nius": 29889, + "##mad": 29890, + "##bot": 29891, + "##urus": 29892, + "reck": 29893, + "oldies": 29894, + "redneck": 29895, + "##ibal": 29896, + "childcare": 29897, + "restor": 29898, + "philharm": 29899, + "robins": 29900, + "folio": 29901, + "recorders": 29902, + "contracts": 29903, + "wombat": 29904, + "searches": 29905, + "hoodedie": 29906, + "alder": 29907, + "dahlias": 29908, + "antalya": 29909, + "chattanooga": 29910, + "eis": 29911, + "jima": 29912, + "kosh": 29913, + "##2nd": 29914, + "##urd": 29915, + "handball": 29916, + "##bose": 29917, + "carpeting": 29918, + "cornerback": 29919, + "instrumental": 29920, + "napping": 29921, + "conservancy": 29922, + "peeks": 29923, + "danbury": 29924, + "mtb": 29925, + "dispense": 29926, + "consuming": 29927, + "glucose": 29928, + "resolutions": 29929, + "confidential": 29930, + "crumbled": 29931, + "rendition": 29932, + "sublime": 29933, + "strasbourg": 29934, + "cess": 29935, + "tern": 29936, + "tho": 29937, + "covet": 29938, + "loki": 29939, + "teasp": 29940, + "groundbreaking": 29941, + "saluting": 29942, + "##parts": 29943, + "clueless": 29944, + "clothesline": 29945, + "charlton": 29946, + "revolving": 29947, + "marathi": 29948, + "1881": 29949, + "herbalife": 29950, + "vernacular": 29951, + "escapes": 29952, + "scribe": 29953, + "robbery": 29954, + "halftone": 29955, + "##urally": 29956, + "##istor": 29957, + "loin": 29958, + "leela": 29959, + "abort": 29960, + "##uden": 29961, + "crocheted": 29962, + "kayakers": 29963, + "reproductive": 29964, + "rhinos": 29965, + "pelicaos": 29966, + "lawsuit": 29967, + "outsider": 29968, + "hammersmith": 29969, + "##burgring": 29970, + "ist": 29971, + "jenny": 29972, + "##rods": 29973, + "grafton": 29974, + "centenary": 29975, + "screams": 29976, + "hamlet": 29977, + "bubbly": 29978, + "savvy": 29979, + "scubaing": 29980, + "kyiv": 29981, + "ridg": 29982, + "misfits": 29983, + "reinvent": 29984, + "versatility": 29985, + "hazelnuts": 29986, + "sansk": 29987, + "convex": 29988, + "obesity": 29989, + "ratchet": 29990, + "inquiry": 29991, + "cited": 29992, + "eric": 29993, + "##maline": 29994, + "monoc": 29995, + "underrated": 29996, + "summar": 29997, + "reprint": 29998, + "genetics": 29999 + } + } +} \ No newline at end of file diff --git a/fourm/vq/__init__.py b/fourm/vq/__init__.py new file mode 100755 index 0000000000000000000000000000000000000000..a77845b18c7f6e3f99483387c1d6d466a212a790 --- /dev/null +++ b/fourm/vq/__init__.py @@ -0,0 +1,79 @@ +import sys, os +import torch + +from .vqvae import VQ, VQVAE, DiVAE, VQControlNet +from .scheduling import * + + +def get_image_tokenizer(tokenizer_id: str, + tokenizers_root: str = './tokenizer_ckpts', + encoder_only: bool = False, + device: str = 'cuda', + verbose: bool = True, + return_None_on_fail: bool = False,): + """ + Load a pretrained image tokenizer from a checkpoint. + + Args: + tokenizer_id (str): ID of the tokenizer to load (name of the checkpoint file without ".pth"). + tokenizers_root (str): Path to the directory containing the tokenizer checkpoints. + encoder_only (bool): Set to True to load only the encoder part of the tokenizer. + device (str): Device to load the tokenizer on. + verbose (bool): Set to True to print load_state_dict warning/success messages + return_None_on_fail (bool): Set to True to return None if the tokenizer fails to load (e.g. doesn't exist) + + Returns: + model (nn.Module): The loaded tokenizer. + """ + if return_None_on_fail and not os.path.exists(os.path.join(tokenizers_root, f'{tokenizer_id}.pth')): + return None + + if verbose: + print(f'Loading tokenizer {tokenizer_id} ... ', end='') + + ckpt = torch.load(os.path.join(tokenizers_root, f'{tokenizer_id}.pth'), map_location='cpu') + + # Handle renamed arguments + if 'CLIP' in ckpt['args'].domain or 'DINO' in ckpt['args'].domain or 'ImageBind' in ckpt['args'].domain: + ckpt['args'].patch_proj = False + elif 'sam' in ckpt['args'].domain: + ckpt['args'].input_size_min = ckpt['args'].mask_size + ckpt['args'].input_size_max = ckpt['args'].mask_size + ckpt['args'].input_size = ckpt['args'].mask_size + + ckpt['args'].quant_type = getattr(ckpt['args'], 'quantizer_type', None) + ckpt['args'].enc_type = getattr(ckpt['args'], 'encoder_type', None) + ckpt['args'].dec_type = getattr(ckpt['args'], 'decoder_type', None) + ckpt['args'].image_size = getattr(ckpt['args'], 'input_size', None) or getattr(ckpt['args'], 'input_size_max', None) + ckpt['args'].image_size_enc = getattr(ckpt['args'], 'input_size_enc', None) + ckpt['args'].image_size_dec = getattr(ckpt['args'], 'input_size_dec', None) + ckpt['args'].image_size_sd = getattr(ckpt['args'], 'input_size_sd', None) + ckpt['args'].ema_decay = getattr(ckpt['args'], 'quantizer_ema_decay', None) + ckpt['args'].enable_xformer = getattr(ckpt['args'], 'use_xformer', None) + if 'cls_emb.weight' in ckpt['model']: + ckpt['args'].n_labels, ckpt['args'].n_channels = n_labels, n_channels = ckpt['model']['cls_emb.weight'].shape + elif 'encoder.linear_in.weight' in ckpt['model']: + ckpt['args'].n_channels = ckpt['model']['encoder.linear_in.weight'].shape[1] + else: + ckpt['args'].n_channels = ckpt['model']['encoder.proj.weight'].shape[1] + ckpt['args'].sync_codebook = False + + if encoder_only: + model_type = VQ + ckpt['model'] = {k: v for k, v in ckpt['model'].items() if 'decoder' not in k and 'post_quant_proj' not in k} + else: + # TODO: Add the model type to the checkpoint when training so we can avoid this hackery + if any(['controlnet' in k for k in ckpt['model'].keys()]): + ckpt['args'].model_type = 'VQControlNet' + elif hasattr(ckpt['args'], 'beta_schedule'): + ckpt['args'].model_type = 'DiVAE' + else: + ckpt['args'].model_type = 'VQVAE' + model_type = getattr(sys.modules[__name__], ckpt['args'].model_type) + model = model_type(**vars(ckpt['args'])) + + msg = model.load_state_dict(ckpt['model'], strict=False) + if verbose: + print(msg) + + return model.to(device).eval(), ckpt['args'] diff --git a/fourm/vq/models/__init__.py b/fourm/vq/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/fourm/vq/models/controlnet.py b/fourm/vq/models/controlnet.py new file mode 100644 index 0000000000000000000000000000000000000000..c4a3789f07c7b2398b3018f36ed437c66258ba31 --- /dev/null +++ b/fourm/vq/models/controlnet.py @@ -0,0 +1,287 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Union, Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from diffusers.models import ControlNetModel +from diffusers.configuration_utils import ConfigMixin +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.controlnet import zero_module + +from fourm.utils import to_2tuple +from .lm_models import create_model + + +class ControlNetAdapterEmbedding(nn.Module): + + def __init__( + self, + conditioning_embedding_channels, + adapter, + conditioning_channels=3, + ): + super().__init__() + + self.adapter_model = create_model( + in_channels=conditioning_channels, + output_type="stats", + ) + self._load_adapter(adapter) + + self.conv_out = zero_module( + nn.Conv2d(8, conditioning_embedding_channels, kernel_size=3, padding=1) + ) + + def forward(self, conditioning): + embedding = self.adapter_model(quant=conditioning) + + embedding = self.conv_out(embedding) + + return embedding + + def _load_adapter(self, path): + ckpt = torch.load(path)['model'] + for key in list(ckpt.keys()): + if 'vq_model' in key or 'vae' in key: + del ckpt[key] + self.adapter_model.load_state_dict(ckpt) + print("Loaded the adapter model") + + +class ControlNetConditioningEmbedding(nn.Module): + + def __init__( + self, + conditioning_embedding_channels, + conditioning_channels = 3, + block_out_channels = (16, 32, 96, 256), + ): + super().__init__() + + self.conv_in = nn.Conv2d(conditioning_channels, block_out_channels[0], kernel_size=3, padding=1) + + self.blocks = nn.ModuleList([]) + + for i in range(len(block_out_channels) - 1): + channel_in = block_out_channels[i] + channel_out = block_out_channels[i + 1] + self.blocks.append(nn.Conv2d(channel_in, channel_in, kernel_size=3, padding=1)) + self.blocks.append(nn.Conv2d(channel_in, channel_out, kernel_size=3, padding=1)) + + self.conv_out = zero_module( + nn.Conv2d(block_out_channels[-1], conditioning_embedding_channels, kernel_size=3, padding=1) + ) + + def forward(self, conditioning): + embedding = self.conv_in(conditioning) + embedding = F.silu(embedding) + + for block in self.blocks: + embedding = block(embedding) + embedding = F.silu(embedding) + + embedding = self.conv_out(embedding) + + return embedding + + +class ControlnetCond(ModelMixin, ConfigMixin): + def __init__(self, + in_channels, + cond_channels, + sd_pipeline, + image_size, + freeze_params=True, + block_out_channels = (320, 640, 1280, 1280), + conditioning_embedding_out_channels = (32, 32, 96, 256), + pretrained_cn=False, + enable_xformer=False, + adapter=None, + *args, + **kwargs + ): + super().__init__() + self.in_channels = in_channels + self.cond_channels = cond_channels + + self.sd_pipeline = sd_pipeline + self.unet = sd_pipeline.unet + self.text_encoder = sd_pipeline.text_encoder + self.tokenizer = sd_pipeline.tokenizer + + if pretrained_cn: + self.controlnet = ControlNetModel.from_unet(self.unet, conditioning_embedding_out_channels=conditioning_embedding_out_channels) + self.controlnet.conditioning_channels = cond_channels + self.controlnet.config.conditioning_channels = cond_channels + else: + self.controlnet = ControlNetModel( + in_channels=in_channels, + conditioning_channels=cond_channels, + block_out_channels=block_out_channels, + conditioning_embedding_out_channels=conditioning_embedding_out_channels, + *args, + **kwargs, + ) + + self.use_adapter = adapter is not None + if adapter is not None: + self.controlnet.controlnet_cond_embedding = ControlNetAdapterEmbedding( + conditioning_embedding_channels=self.controlnet.config.block_out_channels[0], + adapter=adapter, + conditioning_channels=cond_channels, + ) + else: + self.controlnet.controlnet_cond_embedding = ControlNetConditioningEmbedding( + conditioning_embedding_channels=self.controlnet.config.block_out_channels[0], + block_out_channels=self.controlnet.config.conditioning_embedding_out_channels, + conditioning_channels=cond_channels, + ) + + if enable_xformer: + print('xFormer enabled') + self.unet.enable_xformers_memory_efficient_attention() + self.controlnet.enable_xformers_memory_efficient_attention() + + self.empty_str_encoding = nn.Parameter(self._encode_prompt(""), requires_grad=False) + if freeze_params: + self.freeze_params() + + self.sample_size = image_size // sd_pipeline.vae_scale_factor + self.H, self.W = to_2tuple(self.sample_size) + + def forward(self, + sample: torch.FloatTensor, # Shape (B, C, H, W), + timestep: Union[torch.Tensor, float, int], + encoder_hidden_states: torch.Tensor = None, # Shape (B, D_C, H_C, W_C) + cond_mask: Optional[torch.BoolTensor] = None, # Boolen tensor of shape (B, H_C, W_C). True for masked out pixels, + prompt = None, + unconditional = False, + cond_scale = 1.0, + **kwargs): + + # Optionally mask out conditioning + if cond_mask is not None: + encoder_hidden_states = torch.where(cond_mask[:,None,:,:], 0.0, encoder_hidden_states) + + if not self.use_adapter: + controlnet_cond = F.interpolate(encoder_hidden_states, (self.H, self.W), mode="nearest") + else: + controlnet_cond = F.interpolate(encoder_hidden_states, (self.H // 2, self.W // 2), mode="nearest") + + # encoder_hidden_states is the propmp embedding in the controlnet model, for now it's set to zeros. + if prompt is None or unconditional: + encoder_hidden_states = torch.cat([self.empty_str_encoding] * sample.shape[0]) + else: + encoder_hidden_states = self._encode_prompt(prompt) + + down_block_res_samples, mid_block_res_sample = self.controlnet( + sample, + timestep, + encoder_hidden_states=encoder_hidden_states, + controlnet_cond=controlnet_cond, + conditioning_scale=cond_scale, + return_dict=False, + ) + + # TODO not the most efficient way + if unconditional: + down_block_res_samples = [torch.zeros_like(s) for s in down_block_res_samples] + controlnet_cond = torch.zeros_like(controlnet_cond) + + noise_pred = self.unet( + sample, + timestep, + encoder_hidden_states=encoder_hidden_states, + down_block_additional_residuals=down_block_res_samples, + mid_block_additional_residual=mid_block_res_sample, + return_dict=False, + )[0] + + return noise_pred + + def freeze_params(self): + for param in self.unet.parameters(): + param.requires_grad = False + for param in self.text_encoder.parameters(): + param.requires_grad = False + + def unfreeze_params(self): + for param in self.unet.parameters(): + param.requires_grad = True + for param in self.text_encoder.parameters(): + param.requires_grad = True + + @torch.no_grad() + def _encode_prompt(self, prompt): + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(self.device) + else: + attention_mask = None + + prompt_embeds = self.text_encoder( + text_input_ids.to(self.device), + attention_mask=attention_mask, + )[0] + + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=self.device) + + return prompt_embeds + + +def controlnet(*args, **kwargs): + return ControlnetCond( + flip_sin_to_cos=True, + freq_shift=0, + down_block_types= + ['CrossAttnDownBlock2D', + 'CrossAttnDownBlock2D', + 'CrossAttnDownBlock2D', + 'DownBlock2D'], + only_cross_attention=False, + block_out_channels=[320, 640, 1280, 1280], + layers_per_block=2, + downsample_padding=1, + mid_block_scale_factor=1, + act_fn='silu', + norm_num_groups=32, + norm_eps=1e-05, + cross_attention_dim=768, + attention_head_dim=8, + num_attention_heads=None, + use_linear_projection=False, + class_embed_type=None, + num_class_embeds=None, + upcast_attention=False, + resnet_time_scale_shift='default', + projection_class_embeddings_input_dim=None, + controlnet_conditioning_channel_order='rgb', + conditioning_embedding_out_channels=[kwargs['cond_channels'], 32, 96, 256], + global_pool_conditions=False, + freeze_params=True, + *args, + **kwargs, + ) \ No newline at end of file diff --git a/fourm/vq/models/lm_models.py b/fourm/vq/models/lm_models.py new file mode 100644 index 0000000000000000000000000000000000000000..c0aed5a7713edb01a1f477f514e4ac0405f76c29 --- /dev/null +++ b/fourm/vq/models/lm_models.py @@ -0,0 +1,178 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + lm: latent mapping +""" + +from typing import Optional + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from diffusers.models.unet_2d_blocks import UNetMidBlock2D, get_up_block + + +FREEZE_MODULES = ['encoder', 'quant_proj', 'quantize', 'cls_emb'] + +class Token2VAE(nn.Module): + def __init__( + self, + in_channels=32, + output_type="stats", + up_block_types=("UpDecoderBlock2D", "UpDecoderBlock2D",), + block_out_channels=(256, 512), + layers_per_block=2, + norm_num_groups=32, + act_fn="silu", + vq_model=None, + vae=None, + ): + super().__init__() + + assert output_type in ["stats", "sample"], "`output_type` can be either of 'stats' or 'sample'" + self.output_type = output_type + out_channels = 4 if output_type == "sample" else 8 + + self.layers_per_block = layers_per_block + + self.conv_in = nn.Conv2d( + in_channels, + block_out_channels[-1], + kernel_size=3, + stride=1, + padding=1, + ) + + self.mid_block = None + self.up_blocks = nn.ModuleList([]) + + # mid + self.mid_block = UNetMidBlock2D( + in_channels=block_out_channels[-1], + resnet_eps=1e-6, + resnet_act_fn=act_fn, + output_scale_factor=1, + resnet_time_scale_shift="default", + attention_head_dim=block_out_channels[-1], + resnet_groups=norm_num_groups, + temb_channels=None, + ) + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + + is_final_block = i == len(block_out_channels) - 1 + + up_block = get_up_block( + up_block_type, + num_layers=self.layers_per_block + 1, + in_channels=prev_output_channel, + out_channels=output_channel, + prev_output_channel=None, + add_upsample=not is_final_block, + resnet_eps=1e-6, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + attention_head_dim=output_channel, + temb_channels=None, + resnet_time_scale_shift="group", + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + self.conv_norm_out = nn.GroupNorm(num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6) + self.conv_act = nn.SiLU() + self.conv_out = nn.Conv2d(block_out_channels[0], out_channels, 3, padding=1) + + self.vq_model = vq_model + self.vae = vae + + @torch.no_grad() + def vae_encode(self, x): + assert self.vae is not None, "VAE is not initialized" + z = self.vae.encode(x).latent_dist + if self.output_type == "sample": + z = z.sample() + else: + z = torch.cat((z.mean, z.std), dim=1) + z = z * self.vae.config.scaling_factor + return z + + @torch.no_grad() + def vae_decode(self, x, clip=True): + assert self.vae is not None, "VAE is not initialized" + x = self.sample(x) + x = self.vae.decode(x / self.vae.config.scaling_factor).sample + if clip: + x = torch.clip(x, min=-1, max=1) + return x + + def sample(self, x): + if x.shape[1] == 4: + return x + mean, std = x.chunk(2, dim=1) + x = mean + std * torch.randn_like(std) + return x + + def forward(self, quant=None, image=None): + + if quant is None: + assert image is not None, "Neither of `quant` or `image` are provided" + assert self.vq_model is not None, "VQ encoder is not initialized" + with torch.no_grad(): + quant, _, _ = self.vq_model.encode(image) + + x = self.conv_in(quant) + + upscale_dtype = next(iter(self.up_blocks.parameters())).dtype + + # middle + x = self.mid_block(x) + x = x.to(upscale_dtype) + + # up + for up_block in self.up_blocks: + x = up_block(x) + + # post-process + x = self.conv_norm_out(x) + x = self.conv_act(x) + x = self.conv_out(x) + + return x + +def create_model( + in_channels=32, + output_type="stats", + vq_model=None, + vae=None, +): + return Token2VAE( + in_channels=in_channels, + output_type=output_type, + up_block_types=("UpDecoderBlock2D", "UpDecoderBlock2D",), + block_out_channels=(256, 512), + layers_per_block=2, + norm_num_groups=32, + act_fn="silu", + vq_model=vq_model, + vae=vae, + ) \ No newline at end of file diff --git a/fourm/vq/models/mlp_models.py b/fourm/vq/models/mlp_models.py new file mode 100644 index 0000000000000000000000000000000000000000..57158e49f5b1ac43da9f484cfdbbba4894bcb1be --- /dev/null +++ b/fourm/vq/models/mlp_models.py @@ -0,0 +1,165 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional +from torch import nn +from einops import rearrange + + +class BottleneckBlock(nn.Module): + def __init__(self, thin, wide): + super(BottleneckBlock, self).__init__() + + self.block = nn.Sequential( + nn.Linear(thin, wide), + nn.GELU(), + nn.Linear(wide, thin) + ) + + def forward(self, x): + out = self.block(x) + + return out + + +class StandardMLP(nn.Module): + def __init__(self, dim_in, dim_out, widths): + super(StandardMLP, self).__init__() + self.dim_in = dim_in + self.dim_out = dim_out + self.widths = widths + self.linear_in = nn.Linear(self.dim_in, self.widths[0]) + self.linear_out = nn.Linear(self.widths[-1], self.dim_out) + self.layers = [] + self.layer_norms = [] + for i in range(len(self.widths) - 1): + self.layers.append(nn.Linear(self.widths[i], self.widths[i + 1])) + self.layer_norms.append(nn.LayerNorm(widths[i + 1])) + + self.layers = nn.ModuleList(self.layers) + self.layernorms = nn.ModuleList(self.layer_norms) + + def forward(self, x): + # If x is an image, apply MLP point-wise to each token/pixel + if x.ndim == 4: + _, _, H, W = x.shape + x = rearrange(x, 'b d h w -> b (h w) d') + x_is_image = True + else: + x_is_image = False + + z = self.linear_in(x) + for layer, norm in zip(self.layers, self.layer_norms): + z = norm(z) + z = layer(z) + + out = self.linear_out(z) + + # If x was an image, rearrange back to image + if x_is_image: + out = rearrange(out, 'b (h w) d -> b d h w', h=H, w=W) + + return out + + +class BottleneckMLP(nn.Module): + def __init__(self, dim_in, dim_out, block_dims): + super(BottleneckMLP, self).__init__() + self.dim_in = dim_in + self.dim_out = dim_out + self.block_dims = block_dims + + self.linear_in = nn.Linear(self.dim_in, self.block_dims[0][1]) + self.linear_out = nn.Linear(self.block_dims[-1][1], self.dim_out) + blocks = [] + layernorms = [] + + for block_dim in self.block_dims: + wide, thin = block_dim + blocks.append(BottleneckBlock(thin=thin, wide=wide)) + layernorms.append(nn.LayerNorm(thin)) + + self.blocks = nn.ModuleList(blocks) + self.layernorms = nn.ModuleList(layernorms) + + def forward(self, x): + # If x is an image, apply MLP point-wise to each token/pixel + if x.ndim == 4: + _, _, H, W = x.shape + x = rearrange(x, 'b d h w -> b (h w) d') + x_is_image = True + else: + x_is_image = False + + x = self.linear_in(x) + + for block, norm in zip(self.blocks, self.layernorms): + x = x + block(norm(x)) + + out = self.linear_out(x) + + # If x was an image, rearrange back to image + if x_is_image: + out = rearrange(out, 'b (h w) d -> b d h w', h=H, w=W) + + return out + + +def build_mlp(model_id: str = "BottleneckMLP/B_6-Wi_1024", + dim_in: Optional[int] = None, + dim_out: Optional[int] = None, + **kwargs) -> nn.Module: + """Constructs an MLP model from a model ID string, see + "Scaling MLPs: A Tale of Inductive Bias" (https://arxiv.org/abs/2306.13575). + + Args: + model_id: Model ID string. E.g. "BottleneckMLP/B_6-Wi_1024". + See https://arxiv.org/abs/2306.13575 for options and details. + dim_in: Input dimensionality. If None, defaults to MLP dimension. + dim_out: Output dimensionality. If None, defaults to MLP dimension. + + Returns: + MLP model. + """ + model, architecture = model_id.split("/") + assert model in ["BottleneckMLP", "MLP"], f"Model {model} not supported." + + sep = architecture.split("-") + num_blocks = int(sep[0].split("_")[1]) + thin = int(sep[1].split("_")[1]) + + # If dim_in and dim_out are not specified, use MLP dim + dim_in = dim_in or thin + dim_out = dim_out or thin + + if len(sep) == 3: + expansion_factor = int(sep[2].split("_")[1]) + else: + expansion_factor = 4 + + if model == "BottleneckMLP": + blocks = [[expansion_factor * thin, thin] for _ in range(num_blocks)] + + return BottleneckMLP( + dim_in=dim_in, + dim_out=dim_out, + block_dims=blocks, + ) + elif model == "MLP": + blocks = [thin for _ in range(num_blocks)] + + return StandardMLP( + dim_in=dim_in, + dim_out=dim_out, + widths=blocks, + ) \ No newline at end of file diff --git a/fourm/vq/models/unet/fp16_util.py b/fourm/vq/models/unet/fp16_util.py new file mode 100755 index 0000000000000000000000000000000000000000..e1c68b964a76c72b4cfa42574c7a6cb7d357b674 --- /dev/null +++ b/fourm/vq/models/unet/fp16_util.py @@ -0,0 +1,250 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Helpers to train with 16-bit precision. +""" + +import numpy as np +import torch as th +import torch.nn as nn +from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors + +INITIAL_LOG_LOSS_SCALE = 20.0 + + +def convert_module_to_f16(l): + """ + Convert primitive modules to float16. + """ + if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Conv3d)): + l.weight.data = l.weight.data.half() + if l.bias is not None: + l.bias.data = l.bias.data.half() + + +def convert_module_to_f32(l): + """ + Convert primitive modules to float32, undoing convert_module_to_f16(). + """ + if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Conv3d)): + l.weight.data = l.weight.data.float() + if l.bias is not None: + l.bias.data = l.bias.data.float() + +def convert_module_to_bf16(l): + """ + Convert primitive modules to bfloat16, undoing convert_module_to_f16(). + """ + if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Conv3d)): + l.weight.data = l.weight.data.bfloat16() + if l.bias is not None: + l.bias.data = l.bias.data.bfloat16() + + +def make_master_params(param_groups_and_shapes): + """ + Copy model parameters into a (differently-shaped) list of full-precision + parameters. + """ + master_params = [] + for param_group, shape in param_groups_and_shapes: + master_param = nn.Parameter( + _flatten_dense_tensors( + [param.detach().float() for (_, param) in param_group] + ).view(shape) + ) + master_param.requires_grad = True + master_params.append(master_param) + return master_params + + +def model_grads_to_master_grads(param_groups_and_shapes, master_params): + """ + Copy the gradients from the model parameters into the master parameters + from make_master_params(). + """ + for master_param, (param_group, shape) in zip( + master_params, param_groups_and_shapes + ): + master_param.grad = _flatten_dense_tensors( + [param_grad_or_zeros(param) for (_, param) in param_group] + ).view(shape) + + +def master_params_to_model_params(param_groups_and_shapes, master_params): + """ + Copy the master parameter data back into the model parameters. + """ + # Without copying to a list, if a generator is passed, this will + # silently not copy any parameters. + for master_param, (param_group, _) in zip(master_params, param_groups_and_shapes): + for (_, param), unflat_master_param in zip( + param_group, unflatten_master_params(param_group, master_param.view(-1)) + ): + param.detach().copy_(unflat_master_param) + + +def unflatten_master_params(param_group, master_param): + return _unflatten_dense_tensors(master_param, [param for (_, param) in param_group]) + + +def get_param_groups_and_shapes(named_model_params): + named_model_params = list(named_model_params) + scalar_vector_named_params = ( + [(n, p) for (n, p) in named_model_params if p.ndim <= 1], + (-1), + ) + matrix_named_params = ( + [(n, p) for (n, p) in named_model_params if p.ndim > 1], + (1, -1), + ) + return [scalar_vector_named_params, matrix_named_params] + + +def master_params_to_state_dict( + model, param_groups_and_shapes, master_params, use_fp16 +): + if use_fp16: + state_dict = model.state_dict() + for master_param, (param_group, _) in zip( + master_params, param_groups_and_shapes + ): + for (name, _), unflat_master_param in zip( + param_group, unflatten_master_params(param_group, master_param.view(-1)) + ): + assert name in state_dict + state_dict[name] = unflat_master_param + else: + state_dict = model.state_dict() + for i, (name, _value) in enumerate(model.named_parameters()): + assert name in state_dict + state_dict[name] = master_params[i] + return state_dict + + +def state_dict_to_master_params(model, state_dict, use_fp16): + if use_fp16: + named_model_params = [ + (name, state_dict[name]) for name, _ in model.named_parameters() + ] + param_groups_and_shapes = get_param_groups_and_shapes(named_model_params) + master_params = make_master_params(param_groups_and_shapes) + else: + master_params = [state_dict[name] for name, _ in model.named_parameters()] + return master_params + + +def zero_master_grads(master_params): + for param in master_params: + param.grad = None + + +def zero_grad(model_params): + for param in model_params: + # Taken from https://pytorch.org/docs/stable/_modules/torch/optim/optimizer.html#Optimizer.add_param_group + if param.grad is not None: + param.grad.detach_() + param.grad.zero_() + + +def param_grad_or_zeros(param): + if param.grad is not None: + return param.grad.data.detach() + else: + return th.zeros_like(param) + + +class MixedPrecisionTrainer: + def __init__( + self, + *, + model, + use_fp16=False, + fp16_scale_growth=1e-3, + initial_lg_loss_scale=INITIAL_LOG_LOSS_SCALE, + ): + self.model = model + self.use_fp16 = use_fp16 + self.fp16_scale_growth = fp16_scale_growth + + self.model_params = list(self.model.parameters()) + self.master_params = self.model_params + self.param_groups_and_shapes = None + self.lg_loss_scale = initial_lg_loss_scale + + if self.use_fp16: + self.param_groups_and_shapes = get_param_groups_and_shapes( + self.model.named_parameters() + ) + self.master_params = make_master_params(self.param_groups_and_shapes) + self.model.convert_to_fp16() + + def zero_grad(self): + zero_grad(self.model_params) + + def backward(self, loss: th.Tensor): + if self.use_fp16: + loss_scale = 2 ** self.lg_loss_scale + (loss * loss_scale).backward() + else: + loss.backward() + + def optimize(self, opt: th.optim.Optimizer): + if self.use_fp16: + return self._optimize_fp16(opt) + else: + return self._optimize_normal(opt) + + def _optimize_fp16(self, opt: th.optim.Optimizer): + model_grads_to_master_grads(self.param_groups_and_shapes, self.master_params) + grad_norm, param_norm = self._compute_norms(grad_scale=2 ** self.lg_loss_scale) + if check_overflow(grad_norm): + self.lg_loss_scale -= 1 + zero_master_grads(self.master_params) + return False + + for p in self.master_params: + p.grad.mul_(1.0 / (2 ** self.lg_loss_scale)) + opt.step() + zero_master_grads(self.master_params) + master_params_to_model_params(self.param_groups_and_shapes, self.master_params) + self.lg_loss_scale += self.fp16_scale_growth + return True + + def _optimize_normal(self, opt: th.optim.Optimizer): + grad_norm, param_norm = self._compute_norms() + opt.step() + return True + + def _compute_norms(self, grad_scale=1.0): + grad_norm = 0.0 + param_norm = 0.0 + for p in self.master_params: + with th.no_grad(): + param_norm += th.norm(p, p=2, dtype=th.float32).item() ** 2 + if p.grad is not None: + grad_norm += th.norm(p.grad, p=2, dtype=th.float32).item() ** 2 + return np.sqrt(grad_norm) / grad_scale, np.sqrt(param_norm) + + def master_params_to_state_dict(self, master_params): + return master_params_to_state_dict( + self.model, self.param_groups_and_shapes, master_params, self.use_fp16 + ) + + def state_dict_to_master_params(self, state_dict): + return state_dict_to_master_params(self.model, state_dict, self.use_fp16) + + +def check_overflow(value): + return (value == float("inf")) or (value == -float("inf")) or (value != value) \ No newline at end of file diff --git a/fourm/vq/models/unet/nn.py b/fourm/vq/models/unet/nn.py new file mode 100755 index 0000000000000000000000000000000000000000..a87937407d41dbdf000f15961b161e31e7b3c90a --- /dev/null +++ b/fourm/vq/models/unet/nn.py @@ -0,0 +1,179 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Various utilities for neural networks. +""" + +import math + +import torch as th +import torch.nn as nn + + +# PyTorch 1.7 has SiLU, but we support PyTorch 1.5. +class SiLU(nn.Module): + def forward(self, x): + return x * th.sigmoid(x) + + +class GroupNorm32(nn.GroupNorm): + def forward(self, x): + return super().forward(x.float()).type(x.dtype) + + +def conv_nd(dims, *args, **kwargs): + """ + Create a 1D, 2D, or 3D convolution module. + """ + if dims == 1: + return nn.Conv1d(*args, **kwargs) + elif dims == 2: + return nn.Conv2d(*args, **kwargs) + elif dims == 3: + return nn.Conv3d(*args, **kwargs) + raise ValueError(f"unsupported dimensions: {dims}") + + +def linear(*args, **kwargs): + """ + Create a linear module. + """ + return nn.Linear(*args, **kwargs) + + +def avg_pool_nd(dims, *args, **kwargs): + """ + Create a 1D, 2D, or 3D average pooling module. + """ + if dims == 1: + return nn.AvgPool1d(*args, **kwargs) + elif dims == 2: + return nn.AvgPool2d(*args, **kwargs) + elif dims == 3: + return nn.AvgPool3d(*args, **kwargs) + raise ValueError(f"unsupported dimensions: {dims}") + + +def update_ema(target_params, source_params, rate=0.99): + """ + Update target parameters to be closer to those of source parameters using + an exponential moving average. + :param target_params: the target parameter sequence. + :param source_params: the source parameter sequence. + :param rate: the EMA rate (closer to 1 means slower). + """ + for targ, src in zip(target_params, source_params): + targ.detach().mul_(rate).add_(src, alpha=1 - rate) + + +def zero_module(module): + """ + Zero out the parameters of a module and return it. + """ + for p in module.parameters(): + p.detach().zero_() + return module + + +def scale_module(module, scale): + """ + Scale the parameters of a module and return it. + """ + for p in module.parameters(): + p.detach().mul_(scale) + return module + + +def mean_flat(tensor): + """ + Take the mean over all non-batch dimensions. + """ + return tensor.mean(dim=list(range(1, len(tensor.shape)))) + + +def normalization(channels): + """ + Make a standard normalization layer. + :param channels: number of input channels. + :return: an nn.Module for normalization. + """ + return GroupNorm32(32, channels) + + +def timestep_embedding(timesteps, dim, max_period=10000): + """ + Create sinusoidal timestep embeddings. + :param timesteps: a 1-D Tensor of N indices, one per batch element. + These may be fractional. + :param dim: the dimension of the output. + :param max_period: controls the minimum frequency of the embeddings. + :return: an [N x dim] Tensor of positional embeddings. + """ + half = dim // 2 + freqs = th.exp( + -math.log(max_period) * th.arange(start=0, end=half, dtype=th.float32) / half + ).to(device=timesteps.device) + args = timesteps[:, None].float() * freqs[None] + embedding = th.cat([th.cos(args), th.sin(args)], dim=-1) + if dim % 2: + embedding = th.cat([embedding, th.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + +def checkpoint(func, inputs, params, flag): + """ + Evaluate a function without caching intermediate activations, allowing for + reduced memory at the expense of extra compute in the backward pass. + :param func: the function to evaluate. + :param inputs: the argument sequence to pass to `func`. + :param params: a sequence of parameters `func` depends on but does not + explicitly take as arguments. + :param flag: if False, disable gradient checkpointing. + """ + if flag: + args = tuple(inputs) + tuple(params) + return CheckpointFunction.apply(func, len(inputs), *args) + else: + return func(*inputs) + + +class CheckpointFunction(th.autograd.Function): + @staticmethod + def forward(ctx, run_function, length, *args): + ctx.run_function = run_function + ctx.input_tensors = list(args[:length]) + ctx.input_params = list(args[length:]) + with th.no_grad(): + output_tensors = ctx.run_function(*ctx.input_tensors) + return output_tensors + + @staticmethod + def backward(ctx, *output_grads): + ctx.input_tensors = [x.detach().requires_grad_(True) for x in ctx.input_tensors] + with th.enable_grad(): + # Fixes a bug where the first op in run_function modifies the + # Tensor storage in place, which is not allowed for detach()'d + # Tensors. + shallow_copies = [x.view_as(x) for x in ctx.input_tensors] + output_tensors = ctx.run_function(*shallow_copies) + input_grads = th.autograd.grad( + output_tensors, + ctx.input_tensors + ctx.input_params, + output_grads, + allow_unused=True, + ) + del ctx.input_tensors + del ctx.input_params + del output_tensors + return (None, None) + input_grads \ No newline at end of file diff --git a/fourm/vq/models/unet/unet.py b/fourm/vq/models/unet/unet.py new file mode 100755 index 0000000000000000000000000000000000000000..cbd2e83828f19d5d40a1d5d4b2f6f205ca8407b9 --- /dev/null +++ b/fourm/vq/models/unet/unet.py @@ -0,0 +1,755 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from abc import abstractmethod +from typing import Dict, Union, Optional + +import math + +import numpy as np +import torch +import torch as th +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange + +from diffusers.configuration_utils import ConfigMixin +from diffusers.models.modeling_utils import ModelMixin + +from .fp16_util import convert_module_to_f16, convert_module_to_f32, convert_module_to_bf16 +from .nn import ( + checkpoint, + conv_nd, + linear, + avg_pool_nd, + zero_module, + normalization, + timestep_embedding, +) + + +def pair(t): + return t if isinstance(t, tuple) else (t, t) + +class AttentionPool2d(nn.Module): + """ + Adapted from CLIP: https://github.com/openai/CLIP/blob/main/clip/model.py + """ + + def __init__( + self, + spacial_dim: int, + embed_dim: int, + num_heads_channels: int, + output_dim: int = None, + ): + super().__init__() + self.positional_embedding = nn.Parameter( + th.randn(embed_dim, spacial_dim ** 2 + 1) / embed_dim ** 0.5 + ) + self.qkv_proj = conv_nd(1, embed_dim, 3 * embed_dim, 1) + self.c_proj = conv_nd(1, embed_dim, output_dim or embed_dim, 1) + self.num_heads = embed_dim // num_heads_channels + self.attention = QKVAttention(self.num_heads) + + def forward(self, x): + b, c, *_spatial = x.shape + x = x.reshape(b, c, -1) # NC(HW) + x = th.cat([x.mean(dim=-1, keepdim=True), x], dim=-1) # NC(HW+1) + x = x + self.positional_embedding[None, :, :].to(x.dtype) # NC(HW+1) + x = self.qkv_proj(x) + x = self.attention(x) + x = self.c_proj(x) + return x[:, :, 0] + + +class TimestepBlock(nn.Module): + """ + Any module where forward() takes timestep embeddings as a second argument. + """ + + @abstractmethod + def forward(self, x, emb): + """ + Apply the module to `x` given `emb` timestep embeddings. + """ + + +class TimestepEmbedSequential(nn.Sequential, TimestepBlock): + """ + A sequential module that passes timestep embeddings to the children that + support it as an extra input. + """ + + def forward(self, x, emb): + for layer in self: + if isinstance(layer, TimestepBlock): + x = layer(x, emb) + else: + x = layer(x) + return x + + +class Upsample(nn.Module): + """ + An upsampling layer with an optional convolution. + :param channels: channels in the inputs and outputs. + :param use_conv: a bool determining if a convolution is applied. + :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then + upsampling occurs in the inner-two dimensions. + """ + + def __init__(self, channels, use_conv, dims=2, out_channels=None): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.dims = dims + if use_conv: + self.conv = conv_nd(dims, self.channels, self.out_channels, 3, padding=1) + + def forward(self, x): + assert x.shape[1] == self.channels + if self.dims == 3: + x = F.interpolate( + x, (x.shape[2], x.shape[3] * 2, x.shape[4] * 2), mode="nearest" + ) + else: + x = F.interpolate(x, scale_factor=2, mode="nearest") + if self.use_conv: + x = self.conv(x) + return x + + +class Downsample(nn.Module): + """ + A downsampling layer with an optional convolution. + :param channels: channels in the inputs and outputs. + :param use_conv: a bool determining if a convolution is applied. + :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then + downsampling occurs in the inner-two dimensions. + """ + + def __init__(self, channels, use_conv, dims=2, out_channels=None): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.dims = dims + stride = 2 if dims != 3 else (1, 2, 2) + if use_conv: + self.op = conv_nd( + dims, self.channels, self.out_channels, 3, stride=stride, padding=1 + ) + else: + assert self.channels == self.out_channels + self.op = avg_pool_nd(dims, kernel_size=stride, stride=stride) + + def forward(self, x): + assert x.shape[1] == self.channels + return self.op(x) + + +class ResBlock(TimestepBlock): + """ + A residual block that can optionally change the number of channels. + :param channels: the number of input channels. + :param emb_channels: the number of timestep embedding channels. + :param dropout: the rate of dropout. + :param out_channels: if specified, the number of out channels. + :param use_conv: if True and out_channels is specified, use a spatial + convolution instead of a smaller 1x1 convolution to change the + channels in the skip connection. + :param dims: determines if the signal is 1D, 2D, or 3D. + :param use_checkpoint: if True, use gradient checkpointing on this module. + :param up: if True, use this block for upsampling. + :param down: if True, use this block for downsampling. + """ + + def __init__( + self, + channels, + emb_channels, + dropout, + out_channels=None, + use_conv=False, + use_scale_shift_norm=False, + dims=2, + use_checkpoint=False, + up=False, + down=False, + ): + super().__init__() + self.channels = channels + self.emb_channels = emb_channels + self.dropout = dropout + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.use_checkpoint = use_checkpoint + self.use_scale_shift_norm = use_scale_shift_norm + + self.in_layers = nn.Sequential( + normalization(channels), + nn.SiLU(), + conv_nd(dims, channels, self.out_channels, 3, padding=1), + ) + + self.updown = up or down + + if up: + self.h_upd = Upsample(channels, False, dims) + self.x_upd = Upsample(channels, False, dims) + elif down: + self.h_upd = Downsample(channels, False, dims) + self.x_upd = Downsample(channels, False, dims) + else: + self.h_upd = self.x_upd = nn.Identity() + + self.emb_layers = nn.Sequential( + nn.SiLU(), + linear( + emb_channels, + 2 * self.out_channels if use_scale_shift_norm else self.out_channels, + ), + ) + self.out_layers = nn.Sequential( + normalization(self.out_channels), + nn.SiLU(), + nn.Dropout(p=dropout), + zero_module( + conv_nd(dims, self.out_channels, self.out_channels, 3, padding=1) + ), + ) + + if self.out_channels == channels: + self.skip_connection = nn.Identity() + elif use_conv: + self.skip_connection = conv_nd( + dims, channels, self.out_channels, 3, padding=1 + ) + else: + self.skip_connection = conv_nd(dims, channels, self.out_channels, 1) + + def forward(self, x, emb): + """ + Apply the block to a Tensor, conditioned on a timestep embedding. + :param x: an [N x C x ...] Tensor of features. + :param emb: an [N x emb_channels] Tensor of timestep embeddings. + :return: an [N x C x ...] Tensor of outputs. + """ + return checkpoint( + self._forward, (x, emb), self.parameters(), self.use_checkpoint + ) + + def _forward(self, x, emb): + if self.updown: + in_rest, in_conv = self.in_layers[:-1], self.in_layers[-1] + h = in_rest(x) + h = self.h_upd(h) + x = self.x_upd(x) + h = in_conv(h) + else: + h = self.in_layers(x) + emb_out = self.emb_layers(emb).type(h.dtype) + while len(emb_out.shape) < len(h.shape): + emb_out = emb_out[..., None] + if self.use_scale_shift_norm: + out_norm, out_rest = self.out_layers[0], self.out_layers[1:] + scale, shift = th.chunk(emb_out, 2, dim=1) + h = out_norm(h) * (1 + scale) + shift + h = out_rest(h) + else: + h = h + emb_out + h = self.out_layers(h) + return self.skip_connection(x) + h + + +class AttentionBlock(nn.Module): + """ + An attention block that allows spatial positions to attend to each other. + Originally ported from here, but adapted to the N-d case. + https://github.com/hojonathanho/diffusion/blob/1e0dceb3b3495bbe19116a5e1b3596cd0706c543/diffusion_tf/models/unet.py#L66. + """ + + def __init__( + self, + channels, + num_heads=1, + num_head_channels=-1, + use_checkpoint=False, + use_new_attention_order=False, + ): + super().__init__() + self.channels = channels + if num_head_channels == -1: + self.num_heads = num_heads + else: + assert ( + channels % num_head_channels == 0 + ), f"q,k,v channels {channels} is not divisible by num_head_channels {num_head_channels}" + self.num_heads = channels // num_head_channels + self.use_checkpoint = use_checkpoint + self.norm = normalization(channels) + self.qkv = conv_nd(1, channels, channels * 3, 1) + if use_new_attention_order: + # split qkv before split heads + self.attention = QKVAttention(self.num_heads) + else: + # split heads before split qkv + self.attention = QKVAttentionLegacy(self.num_heads) + + self.proj_out = zero_module(conv_nd(1, channels, channels, 1)) + + def forward(self, x): + return checkpoint(self._forward, (x,), self.parameters(), self.use_checkpoint) + + def _forward(self, x): + b, c, *spatial = x.shape + x = x.reshape(b, c, -1) + qkv = self.qkv(self.norm(x)) + h = self.attention(qkv) + h = self.proj_out(h) + return (x + h).reshape(b, c, *spatial) + + +def count_flops_attn(model, _x, y): + """ + A counter for the `thop` package to count the operations in an + attention operation. + Meant to be used like: + macs, params = thop.profile( + model, + inputs=(inputs, timestamps), + custom_ops={QKVAttention: QKVAttention.count_flops}, + ) + """ + b, c, *spatial = y[0].shape + num_spatial = int(np.prod(spatial)) + # We perform two matmuls with the same number of ops. + # The first computes the weight matrix, the second computes + # the combination of the value vectors. + matmul_ops = 2 * b * (num_spatial ** 2) * c + model.total_ops += th.DoubleTensor([matmul_ops]) + + +class QKVAttentionLegacy(nn.Module): + """ + A module which performs QKV attention. Matches legacy QKVAttention + input/ouput heads shaping + """ + + def __init__(self, n_heads): + super().__init__() + self.n_heads = n_heads + + def forward(self, qkv): + """ + Apply QKV attention. + :param qkv: an [N x (H * 3 * C) x T] tensor of Qs, Ks, and Vs. + :return: an [N x (H * C) x T] tensor after attention. + """ + bs, width, length = qkv.shape + assert width % (3 * self.n_heads) == 0 + ch = width // (3 * self.n_heads) + q, k, v = qkv.reshape(bs * self.n_heads, ch * 3, length).split(ch, dim=1) + scale = 1 / math.sqrt(math.sqrt(ch)) + weight = th.einsum( + "bct,bcs->bts", q * scale, k * scale + ) # More stable with f16 than dividing afterwards + weight = th.softmax(weight.float(), dim=-1).type(weight.dtype) + a = th.einsum("bts,bcs->bct", weight, v) + return a.reshape(bs, -1, length) + + @staticmethod + def count_flops(model, _x, y): + return count_flops_attn(model, _x, y) + + +class QKVAttention(nn.Module): + """ + A module which performs QKV attention and splits in a different order. + """ + + def __init__(self, n_heads): + super().__init__() + self.n_heads = n_heads + + def forward(self, qkv): + """ + Apply QKV attention. + :param qkv: an [N x (3 * H * C) x T] tensor of Qs, Ks, and Vs. + :return: an [N x (H * C) x T] tensor after attention. + """ + bs, width, length = qkv.shape + assert width % (3 * self.n_heads) == 0 + ch = width // (3 * self.n_heads) + q, k, v = qkv.chunk(3, dim=1) + scale = 1 / math.sqrt(math.sqrt(ch)) + weight = th.einsum( + "bct,bcs->bts", + (q * scale).view(bs * self.n_heads, ch, length), + (k * scale).view(bs * self.n_heads, ch, length), + ) # More stable with f16 than dividing afterwards + weight = th.softmax(weight.float(), dim=-1).type(weight.dtype) + a = th.einsum("bts,bcs->bct", weight, v.reshape(bs * self.n_heads, ch, length)) + return a.reshape(bs, -1, length) + + @staticmethod + def count_flops(model, _x, y): + return count_flops_attn(model, _x, y) + + +class UNetModel(ModelMixin, ConfigMixin): + """ + The full UNet model with attention and timestep embedding. + :param in_channels: channels in the input Tensor. + :param model_channels: base channel count for the model. + :param out_channels: channels in the output Tensor. + :param num_res_blocks: number of residual blocks per downsample. + :param attention_resolutions: a collection of downsample rates at which + attention will take place. May be a set, list, or tuple. + For example, if this contains 4, then at 4x downsampling, attention + will be used. + :param dropout: the dropout probability. + :param channel_mult: channel multiplier for each level of the UNet. + :param conv_resample: if True, use learned convolutions for upsampling and + downsampling. + :param dims: determines if the signal is 1D, 2D, or 3D. + :param num_classes: if specified (as an int), then this model will be + class-conditional with `num_classes` classes. + :param use_checkpoint: use gradient checkpointing to reduce memory usage. + :param num_heads: the number of attention heads in each attention layer. + :param num_heads_channels: if specified, ignore num_heads and instead use + a fixed channel width per attention head. + :param num_heads_upsample: works with num_heads to set a different number + of heads for upsampling. Deprecated. + :param use_scale_shift_norm: use a FiLM-like conditioning mechanism. + :param resblock_updown: use residual blocks for up/downsampling. + :param use_new_attention_order: use a different attention pattern for potentially + increased efficiency. + """ + + def __init__( + self, + image_size=224, + in_channels=3, + model_channels=256, + out_channels=3, + num_res_blocks=3, + attention_resolutions=[8,16], + dropout=0, + channel_mult=(1, 2, 4, 8), + conv_resample=True, + dims=2, + num_classes=None, + use_checkpoint=False, + # use_fp16=False, + num_heads=1, + num_head_channels=-1, + num_heads_upsample=-1, + use_scale_shift_norm=False, + resblock_updown=False, + use_new_attention_order=False, + ): + super().__init__() + + if num_heads_upsample == -1: + num_heads_upsample = num_heads + + self.image_size = image_size + self.sample_size = image_size # For compatibility + self.in_channels = in_channels + self.model_channels = model_channels + self.out_channels = out_channels + self.num_res_blocks = num_res_blocks + self.attention_resolutions = attention_resolutions + self.dropout = dropout + self.channel_mult = channel_mult + self.conv_resample = conv_resample + self.num_classes = num_classes + self.use_checkpoint = use_checkpoint + # self.dtype = th.float16 if use_fp16 else th.float32 + self.num_heads = num_heads + self.num_head_channels = num_head_channels + self.num_heads_upsample = num_heads_upsample + + time_embed_dim = model_channels * 4 + self.time_embed = nn.Sequential( + linear(model_channels, time_embed_dim), + nn.SiLU(), + linear(time_embed_dim, time_embed_dim), + ) + + if self.num_classes is not None: + self.label_emb = nn.Embedding(num_classes, time_embed_dim) + + ch = input_ch = int(channel_mult[0] * model_channels) + self.input_blocks = nn.ModuleList( + [TimestepEmbedSequential(conv_nd(dims, in_channels, ch, 3, padding=1))] + ) + self._feature_size = ch + input_block_chans = [ch] + ds = 1 + for level, mult in enumerate(channel_mult): + for _ in range(num_res_blocks): + layers = [ + ResBlock( + ch, + time_embed_dim, + dropout, + out_channels=int(mult * model_channels), + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ) + ] + ch = int(mult * model_channels) + if ds in attention_resolutions: + layers.append( + AttentionBlock( + ch, + use_checkpoint=use_checkpoint, + num_heads=num_heads, + num_head_channels=num_head_channels, + use_new_attention_order=use_new_attention_order, + ) + ) + self.input_blocks.append(TimestepEmbedSequential(*layers)) + self._feature_size += ch + input_block_chans.append(ch) + if level != len(channel_mult) - 1: + out_ch = ch + self.input_blocks.append( + TimestepEmbedSequential( + ResBlock( + ch, + time_embed_dim, + dropout, + out_channels=out_ch, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + down=True, + ) + if resblock_updown + else Downsample( + ch, conv_resample, dims=dims, out_channels=out_ch + ) + ) + ) + ch = out_ch + input_block_chans.append(ch) + ds *= 2 + self._feature_size += ch + + self.middle_block = TimestepEmbedSequential( + ResBlock( + ch, + time_embed_dim, + dropout, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ), + AttentionBlock( + ch, + use_checkpoint=use_checkpoint, + num_heads=num_heads, + num_head_channels=num_head_channels, + use_new_attention_order=use_new_attention_order, + ), + ResBlock( + ch, + time_embed_dim, + dropout, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ), + ) + self._feature_size += ch + + self.output_blocks = nn.ModuleList([]) + for level, mult in list(enumerate(channel_mult))[::-1]: + for i in range(num_res_blocks + 1): + ich = input_block_chans.pop() + layers = [ + ResBlock( + ch + ich, + time_embed_dim, + dropout, + out_channels=int(model_channels * mult), + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ) + ] + ch = int(model_channels * mult) + if ds in attention_resolutions: + layers.append( + AttentionBlock( + ch, + use_checkpoint=use_checkpoint, + num_heads=num_heads_upsample, + num_head_channels=num_head_channels, + use_new_attention_order=use_new_attention_order, + ) + ) + if level and i == num_res_blocks: + out_ch = ch + layers.append( + ResBlock( + ch, + time_embed_dim, + dropout, + out_channels=out_ch, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + up=True, + ) + if resblock_updown + else Upsample(ch, conv_resample, dims=dims, out_channels=out_ch) + ) + ds //= 2 + self.output_blocks.append(TimestepEmbedSequential(*layers)) + self._feature_size += ch + + self.out = nn.Sequential( + normalization(ch), + nn.SiLU(), + zero_module(conv_nd(dims, input_ch, out_channels, 3, padding=1)), + ) + + def convert_to_fp16(self): + """ + Convert the torso of the model to float16. + """ + self.input_blocks.apply(convert_module_to_f16) + self.middle_block.apply(convert_module_to_f16) + self.output_blocks.apply(convert_module_to_f16) + + def convert_to_fp32(self): + """ + Convert the torso of the model to float32. + """ + self.input_blocks.apply(convert_module_to_f32) + self.middle_block.apply(convert_module_to_f32) + self.output_blocks.apply(convert_module_to_f32) + + def convert_to_bf16(self): + """ + Convert the torso of the model to float32. + """ + self.input_blocks.apply(convert_module_to_bf16) + self.middle_block.apply(convert_module_to_bf16) + self.output_blocks.apply(convert_module_to_bf16) + + def forward(self, x, timesteps, y=None, **kwargs): + """ + Apply the model to an input batch. + :param x: an [N x C x ...] Tensor of inputs. + :param timesteps: a 1-D batch of timesteps. + :param y: an [N] Tensor of labels, if class-conditional. + :return: an [N x C x ...] Tensor of outputs. + """ + assert (y is not None) == ( + self.num_classes is not None + ), "must specify y if and only if the model is class-conditional" + + if not th.is_tensor(timesteps): + timesteps = th.tensor([timesteps], dtype=th.long, device=x.device) + elif th.is_tensor(timesteps) and len(timesteps.shape) == 0: + timesteps = timesteps[None].to(x.device) + emb = self.time_embed(timestep_embedding(timesteps, self.model_channels)) + + hs = [] + + if self.num_classes is not None: + assert y.shape == (x.shape[0],) + emb = emb + self.label_emb(y) + + h = x #.type(self.dtype) + for module in self.input_blocks: + h = module(h, emb) + hs.append(h) + h = self.middle_block(h, emb) + for module in self.output_blocks: + h = th.cat([h, hs.pop()], dim=1) + h = module(h, emb) + # h = h.type(x.dtype) + return self.out(h) + + +class PatchedUNetCondCat(UNetModel): + """Patched UNet with conditioning upsampled and concatenated to input. + For more details, see https://arxiv.org/abs/2207.04316 + + Args: + in_channels: Number of input channels + out_channels: Number of output channels + cond_channels: Number of conditioning channels + patch_size: Size of the patch projection before and after the UNet + """ + def __init__(self, in_channels: int, out_channels: int, cond_channels: int, patch_size: int, *args, **kwargs): + in_channels_p = in_channels * patch_size * patch_size + cond_channels + out_channels_p = out_channels * patch_size * patch_size + super().__init__(in_channels=in_channels_p, out_channels=out_channels_p, *args, **kwargs) + self.P_H, self.P_W = pair(patch_size) + self.in_channels = in_channels + self.out_channels = out_channels + + def forward(self, + sample: th.FloatTensor, # Shape (B, C, H, W) + timestep: Union[th.Tensor, float, int], + encoder_hidden_states: th.Tensor = None, # Shape (B, D_C, H_C, W_C) + cond_mask: Optional[th.BoolTensor] = None, # Boolen tensor of shape (B, H_C, W_C). True for masked out pixels + **kwargs): + B, C, H, W = sample.shape + assert (H % self.P_H == 0) and (W % self.P_W == 0), f'Image sizes {H}x{W} must be divisible by patch sizes {self.P_H}x{self.P_W}' + N_H, N_W = H // self.P_H, W // self.P_W # Number of patches in height and width + + # Patchify input from B C H W -> B (C * P_H * P_W) N_H N_W + x = rearrange( + sample, 'b c (nh ph) (nw pw) -> b (c ph pw) nh nw', + ph=self.P_H, pw=self.P_W, nh=N_H, nw=N_W + ) + + # Optionally mask out conditioning + if cond_mask is not None: + encoder_hidden_states = torch.where(cond_mask[:,None,:,:], 0.0, encoder_hidden_states) + + # Concat input with upsampled conditioning + cond_upsampled = F.interpolate(encoder_hidden_states, (N_H, N_W), mode="nearest") + x = th.cat([x, cond_upsampled], dim=1) + + # UNet forward pass in subspace + x = super().forward(x, timestep, **kwargs) + + # Depatchify output from B (C * P_H * P_W) N_H N_W -> B C H W + x = rearrange( + x, 'b (c ph pw) nh nw -> b c (nh ph) (nw pw)', + ph=self.P_H, pw=self.P_W, nh=N_H, nw=N_W + ) + + return x + + +def unet_patched(**kwargs): + return PatchedUNetCondCat( + patch_size=4, + model_channels=256, + num_res_blocks=3, + attention_resolutions=[4,8], + channel_mult=(1,2,2,2), + **kwargs + ) \ No newline at end of file diff --git a/fourm/vq/models/uvit.py b/fourm/vq/models/uvit.py new file mode 100644 index 0000000000000000000000000000000000000000..d52a1be4eb4080304129c612952fd107b6b78fd7 --- /dev/null +++ b/fourm/vq/models/uvit.py @@ -0,0 +1,1104 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional, Tuple, Union +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint +from einops import rearrange, repeat + +from diffusers.models.embeddings import ( + GaussianFourierProjection, + TimestepEmbedding, + Timesteps, +) +from diffusers.models.unet_2d_blocks import ( + DownBlock2D, + UpBlock2D, +) +from diffusers.models.resnet import Downsample2D, Upsample2D +from diffusers.configuration_utils import ConfigMixin +from diffusers.models.modeling_utils import ModelMixin + +# xFormers imports +try: + from xformers.ops import memory_efficient_attention, unbind + XFORMERS_AVAILABLE = True +except ImportError: + print("xFormers not available") + XFORMERS_AVAILABLE = False + + +def modulate(x, shift, scale): + return x * (1.0 + scale.unsqueeze(1)) + shift.unsqueeze(1) + +def pair(t): + return t if isinstance(t, tuple) else (t, t) + +def build_2d_sincos_posemb(h, w, embed_dim=1024, temperature=10000.): + """Sine-cosine positional embeddings as used in MoCo-v3 + + Returns positional embedding of shape [B, H, W, D] + """ + grid_w = torch.arange(w, dtype=torch.float32) + grid_h = torch.arange(h, dtype=torch.float32) + grid_w, grid_h = torch.meshgrid(grid_w, grid_h, indexing='ij') + assert embed_dim % 4 == 0, 'Embed dimension must be divisible by 4 for 2D sin-cos position embedding' + pos_dim = embed_dim // 4 + omega = torch.arange(pos_dim, dtype=torch.float32) / pos_dim + omega = 1. / (temperature ** omega) + out_w = torch.einsum('m,d->md', [grid_w.flatten(), omega]) + out_h = torch.einsum('m,d->md', [grid_h.flatten(), omega]) + pos_emb = torch.cat([torch.sin(out_w), torch.cos(out_w), torch.sin(out_h), torch.cos(out_h)], dim=1)[None, :, :] + pos_emb = rearrange(pos_emb, 'b (h w) d -> b d h w', h=h, w=w) + return pos_emb + + +def drop_path(x, drop_prob: float = 0., training: bool = False): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + This is the same as the DropConnect impl I created for EfficientNet, etc networks, however, + the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper... + See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for + changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use + 'survival rate' as the argument. + """ + if drop_prob == 0. or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = keep_prob + torch.rand(shape, dtype=x.dtype, device=x.device) + random_tensor.floor_() # binarize + output = x.div(keep_prob) * random_tensor + return output + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + def extra_repr(self) -> str: + return 'p={}'.format(self.drop_prob) + + +class Mlp(nn.Module): + def __init__(self, in_features, temb_dim=None, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): + super().__init__() + out_features = out_features or in_features + self.hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, self.hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(self.hidden_features, out_features) + self.drop = nn.Dropout(drop) + if temb_dim is not None: + self.adaLN_modulation = nn.Linear(temb_dim, 2 * self.hidden_features) + + def forward(self, x, temb=None): + x = self.fc1(x) + x = self.act(x) + + # Shift and scale using time emb (see https://arxiv.org/abs/2301.11093) + if hasattr(self, 'adaLN_modulation'): + shift, scale = self.adaLN_modulation(F.silu(temb)).chunk(2, dim=-1) + x = modulate(x, shift, scale) + + x = self.fc2(x) + x = self.drop(x) + return x + + +class Attention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0.): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim ** -0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, mask=None): + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads) + + if XFORMERS_AVAILABLE: + q, k, v = unbind(qkv, 2) + + if mask is not None: + # Wherever mask is True it becomes -infinity, otherwise 0 + mask = mask.to(q.dtype) * -torch.finfo(q.dtype).max + + x = memory_efficient_attention(q, k, v, attn_bias=mask) + x = x.reshape([B, N, C]) + + else: + qkv = qkv.permute(2, 0, 3, 1, 4) + q, k, v = qkv.unbind(0) # make torchscript happy (cannot use tensor as tuple) + + attn = (q @ k.transpose(-2, -1)) * self.scale + + if mask is not None: + mask = mask.unsqueeze(1) + attn = attn.masked_fill(mask, -torch.finfo(attn.dtype).max) + + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class CrossAttention(nn.Module): + def __init__(self, dim, dim_context=None, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0.): + super().__init__() + dim_context = dim_context or dim + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim ** -0.5 + + self.q = nn.Linear(dim, dim, bias=qkv_bias) + self.kv = nn.Linear(dim_context, dim * 2, bias=qkv_bias) + + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, context, mask=None): + B, N, C = x.shape + _, M, _ = context.shape + + q = self.q(x).reshape(B, N, self.num_heads, C // self.num_heads) + kv = self.kv(context).reshape(B, M, 2, self.num_heads, C // self.num_heads) + + if XFORMERS_AVAILABLE: + k, v = unbind(kv, 2) + + if mask is not None: + # Wherever mask is True it becomes -infinity, otherwise 0 + mask = mask.to(q.dtype) * -torch.finfo(q.dtype).max + + x = memory_efficient_attention(q, k, v, attn_bias=mask) + x = x.reshape([B, N, C]) + + else: + q = q.permute(0, 2, 1, 3) + kv = kv.permute(2, 0, 3, 1, 4) + k, v = kv[0], kv[1] + + attn = (q @ k.transpose(-2, -1)) * self.scale + if mask is not None: + mask = rearrange(mask, "b n m -> b 1 n m") + attn = attn.masked_fill(mask, -torch.finfo(attn.dtype).max) + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, -1) + + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class Block(nn.Module): + def __init__(self, dim, num_heads, temb_dim=None, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, skip=False, temb_in_mlp=False, temb_after_norm=True, temb_gate=True): + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, temb_dim=temb_dim if temb_in_mlp else None, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + if temb_after_norm and temb_dim is not None: + # adaLN modulation (see https://arxiv.org/abs/2212.09748) + self.adaLN_modulation = nn.Linear(temb_dim, 4 * dim) + if temb_gate and temb_dim is not None: + # adaLN-Zero gate (see https://arxiv.org/abs/2212.09748) + self.adaLN_gate = nn.Linear(temb_dim, 2 * dim) + nn.init.zeros_(self.adaLN_gate.weight) + nn.init.zeros_(self.adaLN_gate.bias) + self.skip_linear = nn.Linear(2*dim, dim) if skip else None + + def forward(self, x, temb=None, mask=None, skip_connection=None): + gate_msa, gate_mlp = self.adaLN_gate(F.silu(temb)).unsqueeze(1).chunk(2, dim=-1) if hasattr(self, 'adaLN_gate') else (1.0, 1.0) + shift_msa, scale_msa, shift_mlp, scale_mlp = self.adaLN_modulation(F.silu(temb)).chunk(4, dim=-1) if hasattr(self, 'adaLN_modulation') else 4*[0.0] + if self.skip_linear is not None: + x = self.skip_linear(torch.cat([x, skip_connection], dim=-1)) + x = x + gate_msa * self.drop_path(self.attn(modulate(self.norm1(x), shift_msa, scale_msa), mask)) + x = x + gate_mlp * self.drop_path(self.mlp(modulate(self.norm2(x), shift_mlp, scale_mlp), temb)) + return x + + +class DecoderBlock(nn.Module): + def __init__(self, dim, num_heads, temb_dim=None, dim_context=None, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, skip=False, temb_in_mlp=False, temb_after_norm=True, temb_gate=True): + super().__init__() + dim_context = dim_context or dim + self.norm1 = norm_layer(dim) + self.self_attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) + self.cross_attn = CrossAttention(dim, dim_context=dim_context, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) + self.query_norm = norm_layer(dim) + self.context_norm = norm_layer(dim_context) + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, temb_dim=temb_dim if temb_in_mlp else None, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + if temb_after_norm and temb_dim is not None: + # adaLN modulation (see https://arxiv.org/abs/2212.09748) + self.adaLN_modulation = nn.Linear(temb_dim, 6 * dim) + if temb_gate and temb_dim is not None: + # adaLN-Zero gate (see https://arxiv.org/abs/2212.09748) + self.adaLN_gate = nn.Linear(temb_dim, 3 * dim) + nn.init.zeros_(self.adaLN_gate.weight) + nn.init.zeros_(self.adaLN_gate.bias) + self.skip_linear = nn.Linear(2*dim, dim) if skip else None + + def forward(self, x, context, temb=None, sa_mask=None, xa_mask=None, skip_connection=None): + gate_msa, gate_mxa, gate_mlp = self.adaLN_gate(F.silu(temb)).unsqueeze(1).chunk(3, dim=-1) if hasattr(self, 'adaLN_gate') else (1.0, 1.0, 1.0) + shift_msa, scale_msa, shift_mxa, scale_mxa, shift_mlp, scale_mlp = self.adaLN_modulation(F.silu(temb)).chunk(6, dim=-1) if hasattr(self, 'adaLN_modulation') else 6*[0.0] + if self.skip_linear is not None: + x = self.skip_linear(torch.cat([x, skip_connection], dim=-1)) + x = x + gate_msa * self.drop_path(self.self_attn(modulate(self.norm1(x), shift_msa, scale_msa), sa_mask)) + x = x + gate_mxa * self.drop_path(self.cross_attn(modulate(self.query_norm(x), shift_mxa, scale_mxa), self.context_norm(context), xa_mask)) + x = x + gate_mlp * self.drop_path(self.mlp(modulate(self.norm2(x), shift_mlp, scale_mlp), temb)) + return x + + +class TransformerConcatCond(nn.Module): + """UViT Transformer bottleneck that concatenates the condition to the input. + + Args: + unet_dim: Number of channels in the last UNet down block. + cond_dim: Number of channels in the condition. + mid_layers: Number of Transformer layers. + mid_num_heads: Number of attention heads. + mid_dim: Transformer dimension. + mid_mlp_ratio: Ratio of MLP hidden dim to Transformer dim. + mid_qkv_bias: Whether to add bias to the query, key, and value projections. + mid_drop_rate: Dropout rate. + mid_attn_drop_rate: Attention dropout rate. + mid_drop_path_rate: Stochastic depth rate. + time_embed_dim: Dimension of the time embedding. + hw_posemb: Size (side) of the 2D positional embedding. + use_long_skip: Whether to use long skip connections. + See https://arxiv.org/abs/2209.12152 for more details. + """ + def __init__( + self, + unet_dim: int = 1024, + cond_dim: int = 32, + mid_layers: int = 12, + mid_num_heads: int = 12, + mid_dim: int = 768, + mid_mlp_ratio: int = 4, + mid_qkv_bias: bool = True, + mid_drop_rate: float = 0.0, + mid_attn_drop_rate: float = 0.0, + mid_drop_path_rate: float = 0.0, + time_embed_dim: int = 512, + hw_posemb: int = 16, + use_long_skip: bool = False, + ): + super().__init__() + + self.mid_pos_emb = build_2d_sincos_posemb(h=hw_posemb, w=hw_posemb, embed_dim=mid_dim) + self.mid_pos_emb = nn.Parameter(self.mid_pos_emb, requires_grad=False) + + self.use_long_skip = use_long_skip + if use_long_skip: + assert mid_layers % 2 == 1, 'mid_layers must be odd when using long skip connection' + + dpr = [x.item() for x in torch.linspace(0, mid_drop_path_rate, mid_layers)] # stochastic depth decay rule + self.mid_block = nn.ModuleList([ + Block(dim=mid_dim, temb_dim=time_embed_dim, num_heads=mid_num_heads, mlp_ratio=mid_mlp_ratio, qkv_bias=mid_qkv_bias, + drop=mid_drop_rate, attn_drop=mid_attn_drop_rate, drop_path=dpr[i], skip=i > mid_layers//2 and use_long_skip) + for i in range(mid_layers) + ]) + + self.mid_cond_proj = nn.Linear(cond_dim, mid_dim) + self.mid_proj_in = nn.Linear(unet_dim, mid_dim) + self.mid_proj_out = nn.Linear(mid_dim, unet_dim) + + self.mask_token = nn.Parameter(torch.zeros(mid_dim), requires_grad=True) + + + def forward(self, + x: torch.Tensor, + temb: torch.Tensor, + cond: torch.Tensor, + cond_mask: Optional[torch.BoolTensor] = None) -> torch.Tensor: + """TransformerConcatCond forward pass. + + Args: + x: UNet features from the last down block of shape [B, C_mid, H_mid, W_mid]. + temb: Time embedding of shape [B, temb_dim]. + cond: Condition of shape [B, cond_dim, H_cond, W_cond]. If H_cond and W_cond are + different from H_mid and W_mid, cond is interpolated to match the spatial size + of x. + cond_mask: Condition mask of shape [B, H_mid, W_mid]. If a mask is + defined, replaces masked-out tokens by a learnable mask-token. + Wherever cond_mask is True, the condition gets replaced by the mask token. + + Returns: + Features of shape [B, C_mid, H_mid, W_mid] to pass to the UNet up blocks. + """ + B, C_mid, H_mid, W_mid = x.shape + + # Rearrange and proj UNet features to sequence of tokens + x = rearrange(x, 'b d h w -> b (h w) d') + x = self.mid_proj_in(x) + + # Rearrange and proj conditioning to sequence of tokens + cond = F.interpolate(cond, (H_mid, W_mid)) # Interpolate if necessary + cond = rearrange(cond, 'b d h w -> b (h w) d') + cond = self.mid_cond_proj(cond) + + # If a mask is defined, replace masked-out tokens by a learnable mask-token + # Wherever cond_mask is True, the condition gets replaced by the mask token + if cond_mask is not None: + cond_mask = F.interpolate(cond_mask.unsqueeze(1).float(), (H_mid, W_mid), mode='nearest') > 0.5 + cond_mask = rearrange(cond_mask, 'b 1 h w -> b (h w)') + cond[cond_mask] = self.mask_token.to(dtype=cond.dtype) + + x = x + cond + + # Interpolate and rearrange positional embedding to sequence of tokens + mid_pos_emb = F.interpolate(self.mid_pos_emb, (H_mid, W_mid), mode='bicubic', align_corners=False) + mid_pos_emb = rearrange(mid_pos_emb, 'b d h w -> b (h w) d') + x = x + mid_pos_emb + + # Transformer forward pass with or without long skip connections + if not self.use_long_skip: + for blk in self.mid_block: + x = blk(x, temb) + else: + skip_connections = [] + num_skips = len(self.mid_block) // 2 + for blk in self.mid_block[:num_skips]: + x = blk(x, temb) + skip_connections.append(x) + x = self.mid_block[num_skips](x, temb) + for blk in self.mid_block[num_skips+1:]: + x = blk(x, temb, skip_connection=skip_connections.pop()) + + x = self.mid_proj_out(x) # Project Transformer output back to UNet channels + x = rearrange(x, 'b (h w) d -> b d h w', h=H_mid, w=W_mid) # Rearrange Transformer tokens to a spatial feature map for conv layers + + return x + +class TransformerXattnCond(nn.Module): + """UViT Transformer bottleneck that incroporates the condition via cross-attention. + + Args: + unet_dim: Number of channels in the last UNet down block. + cond_dim: Number of channels in the condition. + mid_layers: Number of Transformer layers. + mid_num_heads: Number of attention heads. + mid_dim: Transformer dimension. + mid_mlp_ratio: Ratio of MLP hidden dim to Transformer dim. + mid_qkv_bias: Whether to add bias to the query, key, and value projections. + mid_drop_rate: Dropout rate. + mid_attn_drop_rate: Attention dropout rate. + mid_drop_path_rate: Stochastic depth rate. + time_embed_dim: Dimension of the time embedding. + hw_posemb: Size (side) of the 2D positional embedding. + use_long_skip: Whether to use long skip connections. + See https://arxiv.org/abs/2209.12152 for more details. + """ + def __init__( + self, + unet_dim: int = 1024, + cond_dim: int = 32, + mid_layers: int = 12, + mid_num_heads: int = 12, + mid_dim: int = 768, + mid_mlp_ratio: int = 4, + mid_qkv_bias: bool = True, + mid_drop_rate: float = 0.0, + mid_attn_drop_rate: float = 0.0, + mid_drop_path_rate: float = 0.0, + time_embed_dim: int = 512, + hw_posemb: int = 16, + use_long_skip: bool = False, + ): + super().__init__() + + self.mid_pos_emb = build_2d_sincos_posemb(h=hw_posemb, w=hw_posemb, embed_dim=mid_dim) + self.mid_pos_emb = nn.Parameter(self.mid_pos_emb, requires_grad=False) + + self.use_long_skip = use_long_skip + if use_long_skip: + assert mid_layers % 2 == 1, 'mid_layers must be odd when using long skip connection' + + dpr = [x.item() for x in torch.linspace(0, mid_drop_path_rate, mid_layers)] # stochastic depth decay rule + self.mid_block = nn.ModuleList([ + DecoderBlock( + dim=mid_dim, temb_dim=time_embed_dim, num_heads=mid_num_heads, dim_context=cond_dim, + mlp_ratio=mid_mlp_ratio, qkv_bias=mid_qkv_bias, drop=mid_drop_rate, + attn_drop=mid_attn_drop_rate, drop_path=dpr[i], + skip=i > mid_layers//2 and use_long_skip + ) + for i in range(mid_layers) + ]) + + self.mid_proj_in = nn.Linear(unet_dim, mid_dim) + self.mid_proj_out = nn.Linear(mid_dim, unet_dim) + + def forward(self, + x: torch.Tensor, + temb: torch.Tensor, + cond: torch.Tensor, + cond_mask: Optional[torch.BoolTensor] = None) -> torch.Tensor: + """TransformerXattnCond forward pass. + + Args: + x: UNet features from the last down block of shape [B, C_mid, H_mid, W_mid]. + temb: Time embedding of shape [B, temb_dim]. + cond: Condition of shape [B, cond_dim, H_cond, W_cond]. + cond_mask: Condition cross-attention mask of shape [B, H_cond, W_cond]. + If a mask is defined, wherever cond_mask is True, the condition at that + spatial location is not cross-attended to. + + Returns: + Features of shape [B, C_mid, H_mid, W_mid] to pass to the UNet up blocks. + """ + B, C_mid, H_mid, W_mid = x.shape + + # Rearrange and proj UNet features to sequence of tokens + x = rearrange(x, 'b d h w -> b (h w) d') + x = self.mid_proj_in(x) + + # Rearrange conditioning to sequence of tokens + cond = rearrange(cond, 'b d h w -> b (h w) d') + + # Interpolate and rearrange positional embedding to sequence of tokens + mid_pos_emb = F.interpolate(self.mid_pos_emb, (H_mid, W_mid), mode='nearest') + mid_pos_emb = rearrange(mid_pos_emb, 'b d h w -> b (h w) d') + # Add UNet mid-block features and positional embedding + x = x + mid_pos_emb + + # Prepare the conditioning cross-attention mask + xa_mask = repeat(cond_mask, 'b h w -> b n (h w)', n=x.shape[1]) if cond_mask is not None else None + + # Transformer forward pass with or without long skip connections. + # In each layer, cross-attend to the conditioning. + if not self.use_long_skip: + for blk in self.mid_block: + x = blk(x, cond, temb, xa_mask=xa_mask) + else: + skip_connections = [] + num_skips = len(self.mid_block) // 2 + for blk in self.mid_block[:num_skips]: + x = blk(x, cond, temb, xa_mask=xa_mask) + skip_connections.append(x) + x = self.mid_block[num_skips](x, cond, temb, xa_mask=xa_mask) + for blk in self.mid_block[num_skips+1:]: + x = blk(x, cond, temb, xa_mask=xa_mask, skip_connection=skip_connections.pop()) + + x = self.mid_proj_out(x) # Project Transformer output back to UNet channels + x = rearrange(x, 'b (h w) d -> b d h w', h=H_mid, w=W_mid) # Rearrange Transformer tokens to a spatial feature map for conv layers + + return x + + +class UViT(ModelMixin, ConfigMixin): + """UViT model = Conditional UNet with Transformer bottleneck + blocks and optionalpatching. + See https://arxiv.org/abs/2301.11093 for more details. + + Args: + sample_size: Size of the input images. + in_channels: Number of input channels. + out_channels: Number of output channels. + patch_size: Size of the input patching operation. + See https://arxiv.org/abs/2207.04316 for more details. + block_out_channels: Number of output channels of each UNet ResNet-block. + layers_per_block: Number of ResNet blocks per UNet block. + downsample_before_mid: Whether to downsample before the Transformer bottleneck. + mid_layers: Number of Transformer blocks. + mid_num_heads: Number of attention heads. + mid_dim: Transformer dimension. + mid_mlp_ratio: Transformer MLP ratio. + mid_qkv_bias: Whether to use bias in the Transformer QKV projection. + mid_drop_rate: Dropout rate of the Transformer MLP and attention output projection. + mid_attn_drop_rate: Dropout rate of the Transformer attention. + mid_drop_path_rate: Stochastic depth rate of the Transformer blocks. + mid_hw_posemb: Size (side) of the Transformer positional embedding. + mid_use_long_skip: Whether to use long skip connections in the Transformer blocks. + See https://arxiv.org/abs/2209.12152 for more details. + cond_dim: Dimension of the conditioning vector. + cond_type: Type of conditioning. + 'concat' for concatenation, 'xattn' for cross-attention. + downsample_padding: Padding of the UNet downsampling convolutions. + act_fn: Activation function. + norm_num_groups: Number of groups in the UNet ResNet-block normalization. + norm_eps: Epsilon of the UNet ResNet-block normalization. + resnet_time_scale_shift: Time scale shift of the UNet ResNet-blocks. + resnet_out_scale_factor: Output scale factor of the UNet ResNet-blocks. + time_embedding_type: Type of the time embedding. + 'positional' for positional, 'fourier' for Fourier. + time_embedding_dim: Dimension of the time embedding. + time_embedding_act_fn: Activation function of the time embedding. + timestep_post_act: Activation function after the time embedding. + time_cond_proj_dim: Dimension of the optional conditioning projection. + flip_sin_to_cos: Whether to flip the sine to cosine in the time embedding. + freq_shift: Frequency shift of the time embedding. + res_embedding: Whether to perform original resolution conditioning. + See SDXL https://arxiv.org/abs/2307.01952 for more details. + """ + def __init__(self, + # UNet settings + sample_size: Optional[int] = None, + in_channels: int = 3, + out_channels: int = 3, + patch_size: int = 4, + block_out_channels: Tuple[int] = (128, 256, 512), + layers_per_block: Union[int, Tuple[int]] = 2, + downsample_before_mid: bool = False, + + # Mid-block Transformer settings + mid_layers: int = 12, + mid_num_heads: int = 12, + mid_dim: int = 768, + mid_mlp_ratio: int = 4, + mid_qkv_bias: bool = True, + mid_drop_rate: float = 0.0, + mid_attn_drop_rate: float = 0.0, + mid_drop_path_rate: float = 0.0, + mid_hw_posemb: int = 32, + mid_use_long_skip: bool = False, + + # Conditioning settings + cond_dim: int = 32, + cond_type: str = 'concat', + + # ResNet blocks settings + downsample_padding: int = 1, + act_fn: str = "silu", + norm_num_groups: Optional[int] = 32, + norm_eps: float = 1e-5, + resnet_time_scale_shift: str = "default", + resnet_out_scale_factor: int = 1.0, + + # Time embedding settings + time_embedding_type: str = "positional", + time_embedding_dim: Optional[int] = None, + time_embedding_act_fn: Optional[str] = None, + timestep_post_act: Optional[str] = None, + time_cond_proj_dim: Optional[int] = None, + flip_sin_to_cos: bool = True, + freq_shift: int = 0, + + # Original resolution embedding settings + res_embedding: bool = False): + + super().__init__() + + self.sample_size = sample_size + self.in_channels = in_channels + self.out_channels = out_channels + self.mid_dim = block_out_channels[-1] + self.res_embedding = res_embedding + + # input patching + self.conv_in = nn.Conv2d( + in_channels, block_out_channels[0], kernel_size=patch_size, padding=0, stride=patch_size + ) + + # time + if time_embedding_type == "fourier": + time_embed_dim = time_embedding_dim or block_out_channels[0] * 2 + if time_embed_dim % 2 != 0: + raise ValueError(f"`time_embed_dim` should be divisible by 2, but is {time_embed_dim}.") + self.time_proj = GaussianFourierProjection( + time_embed_dim // 2, set_W_to_weight=False, log=False, flip_sin_to_cos=flip_sin_to_cos + ) + timestep_input_dim = time_embed_dim + elif time_embedding_type == "positional": + time_embed_dim = time_embedding_dim or block_out_channels[0] * 4 + + self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift) + timestep_input_dim = block_out_channels[0] + else: + raise ValueError( + f"{time_embedding_type} does not exist. Please make sure to use one of `fourier` or `positional`." + ) + + self.time_embedding = TimestepEmbedding( + timestep_input_dim, + time_embed_dim, + act_fn=act_fn, + post_act_fn=timestep_post_act, + cond_proj_dim=time_cond_proj_dim, + ) + + if time_embedding_act_fn is None: + self.time_embed_act = None + elif time_embedding_act_fn == "swish": + self.time_embed_act = lambda x: F.silu(x) + elif time_embedding_act_fn == "mish": + self.time_embed_act = nn.Mish() + elif time_embedding_act_fn == "silu": + self.time_embed_act = nn.SiLU() + elif time_embedding_act_fn == "gelu": + self.time_embed_act = nn.GELU() + else: + raise ValueError(f"Unsupported activation function: {time_embedding_act_fn}") + + # original resolution embedding + if res_embedding: + if time_embedding_type == "fourier": + self.h_proj = GaussianFourierProjection( + time_embed_dim // 2, set_W_to_weight=False, log=False, flip_sin_to_cos=flip_sin_to_cos + ) + self.w_proj = GaussianFourierProjection( + time_embed_dim // 2, set_W_to_weight=False, log=False, flip_sin_to_cos=flip_sin_to_cos + ) + elif time_embedding_type == "positional": + self.height_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift) + self.width_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift) + + self.height_embedding = TimestepEmbedding( + timestep_input_dim, time_embed_dim, act_fn=act_fn, + post_act_fn=timestep_post_act, cond_proj_dim=time_cond_proj_dim, + ) + self.width_embedding = TimestepEmbedding( + timestep_input_dim, time_embed_dim, act_fn=act_fn, + post_act_fn=timestep_post_act, cond_proj_dim=time_cond_proj_dim, + ) + + self.down_blocks = nn.ModuleList([]) + self.up_blocks = nn.ModuleList([]) + + if isinstance(layers_per_block, int): + layers_per_block = [layers_per_block] * len(block_out_channels) + + # down + output_channel = block_out_channels[0] + for i in range(len(block_out_channels)): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = DownBlock2D( + num_layers=layers_per_block[i], + in_channels=input_channel, + out_channels=output_channel, + temb_channels=time_embed_dim, + add_downsample=not is_final_block, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + output_scale_factor=resnet_out_scale_factor, + ) + self.down_blocks.append(down_block) + if downsample_before_mid: + self.downsample_mid = Downsample2D(self.mid_dim, use_conv=True, out_channels=self.mid_dim) + + # mid + if cond_type == 'concat': + self.mid_block = TransformerConcatCond( + unet_dim=self.mid_dim, cond_dim=cond_dim, mid_layers=mid_layers, mid_num_heads=mid_num_heads, + mid_dim=mid_dim, mid_mlp_ratio=mid_mlp_ratio, mid_qkv_bias=mid_qkv_bias, + mid_drop_rate=mid_drop_rate, mid_attn_drop_rate=mid_attn_drop_rate, mid_drop_path_rate=mid_drop_path_rate, + time_embed_dim=time_embed_dim, hw_posemb=mid_hw_posemb, use_long_skip=mid_use_long_skip, + ) + elif cond_type == 'xattn': + self.mid_block = TransformerXattnCond( + unet_dim=self.mid_dim, cond_dim=cond_dim, mid_layers=mid_layers, mid_num_heads=mid_num_heads, + mid_dim=mid_dim, mid_mlp_ratio=mid_mlp_ratio, mid_qkv_bias=mid_qkv_bias, + mid_drop_rate=mid_drop_rate, mid_attn_drop_rate=mid_attn_drop_rate, mid_drop_path_rate=mid_drop_path_rate, + time_embed_dim=time_embed_dim, hw_posemb=mid_hw_posemb, use_long_skip=mid_use_long_skip, + ) + else: + raise ValueError(f"Unsupported cond_type: {cond_type}") + + # count how many layers upsample the images + self.num_upsamplers = 0 + + # up + if downsample_before_mid: + self.upsample_mid = Upsample2D(self.mid_dim, use_conv=True, out_channels=self.mid_dim) + + reversed_block_out_channels = list(reversed(block_out_channels)) + reversed_layers_per_block = list(reversed(layers_per_block)) + + output_channel = reversed_block_out_channels[0] + for i in range(len(reversed_block_out_channels)): + is_final_block = i == len(block_out_channels) - 1 + + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + input_channel = reversed_block_out_channels[min(i + 1, len(block_out_channels) - 1)] + + # add upsample block for all BUT final layer + if not is_final_block: + add_upsample = True + self.num_upsamplers += 1 + else: + add_upsample = False + + up_block = UpBlock2D( + num_layers=reversed_layers_per_block[i] + 1, + in_channels=input_channel, + out_channels=output_channel, + prev_output_channel=prev_output_channel, + temb_channels=time_embed_dim, + add_upsample=add_upsample, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + output_scale_factor=resnet_out_scale_factor, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + if norm_num_groups is not None: + self.conv_norm_out = nn.GroupNorm( + num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=norm_eps + ) + + if act_fn == "swish": + self.conv_act = lambda x: F.silu(x) + elif act_fn == "mish": + self.conv_act = nn.Mish() + elif act_fn == "silu": + self.conv_act = nn.SiLU() + elif act_fn == "gelu": + self.conv_act = nn.GELU() + else: + raise ValueError(f"Unsupported activation function: {act_fn}") + + else: + self.conv_norm_out = None + self.conv_act = None + + self.conv_out = nn.ConvTranspose2d( + block_out_channels[0], out_channels, kernel_size=patch_size, stride=patch_size + ) + + self.init_weights() + + def init_weights(self) -> None: + """Weight initialization following MAE's initialization scheme""" + + for name, m in self.named_modules(): + # Handle already zero-init gates + if "adaLN_gate" in name: + continue + # Handle ResNet gates that were not initialized by diffusers + if "conv2" in name: + nn.init.zeros_(m.weight) + nn.init.zeros_(m.bias) + + # Linear + elif isinstance(m, nn.Linear): + if 'qkv' in name: + # treat the weights of Q, K, V separately + val = math.sqrt(6. / float(m.weight.shape[0] // 3 + m.weight.shape[1])) + nn.init.uniform_(m.weight, -val, val) + elif 'kv' in name: + # treat the weights of K, V separately + val = math.sqrt(6. / float(m.weight.shape[0] // 2 + m.weight.shape[1])) + nn.init.uniform_(m.weight, -val, val) + else: + nn.init.xavier_uniform_(m.weight) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + # LayerNorm + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + # Embedding + elif isinstance(m, nn.Embedding): + nn.init.normal_(m.weight, std=self.init_std) + # Conv2d + elif isinstance(m, nn.Conv2d): + if '.conv_in' in name or '.conv_out' in name: + # From MAE, initialize projection like nn.Linear (instead of nn.Conv2d) + w = m.weight.data + nn.init.xavier_uniform_(w.view([w.shape[0], -1])) + + def forward( + self, + sample: torch.FloatTensor, + timestep: Union[torch.Tensor, float, int], + condition: torch.Tensor, + cond_mask: Optional[torch.Tensor] = None, + timestep_cond: Optional[torch.Tensor] = None, + orig_res: Optional[Union[torch.LongTensor, Tuple[int, int]]] = None, + **kwargs, + ) -> torch.Tensor: + """UViT forward pass. + + Args: + sample: Noisy image of shape (B, C, H, W). + timestep: Timestep(s) of the current batch. + condition: Conditioning tensor of shape (B, C_cond, H_cond, W_cond). When concatenating + the condition, it is interpolated to the resolution of the transformer (H_mid, W_mid). + cond_mask: Mask tensor of shape (B, H_mid, W_mid) when concatenating the condition + to the transformer, and (B, H_cond, W_cond) when using cross-attention. True for + masked out / ignored regions. + timestep_cond: Optional conditioning to add to the timestep embedding. + orig_res: The original resolution of the image to condition the diffusion on. Ignored if None. + See SDXL https://arxiv.org/abs/2307.01952 for more details. + + Returns: + Diffusion objective target image of shape (B, C, H, W). + """ + # By default samples have to be AT least a multiple of the overall upsampling factor. + # The overall upsampling factor is equal to 2 ** (# num of upsampling layers). + # However, the upsampling interpolation output size can be forced to fit any upsampling size + # on the fly if necessary. + default_overall_up_factor = 2**self.num_upsamplers + + # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor` + forward_upsample_size = False + upsample_size = None + + if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]): + forward_upsample_size = True + + # 1. time + timesteps = timestep + is_mps = sample.device.type == "mps" + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + # This would be a good case for the `match` statement (Python 3.10+) + if isinstance(timestep, float): + dtype = torch.float32 if is_mps else torch.float64 + else: + dtype = torch.int32 if is_mps else torch.int64 + timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device) + elif len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps.expand(sample.shape[0]) + + t_emb = self.time_proj(timesteps) + + # `Timesteps` does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=sample.dtype) + + emb = self.time_embedding(t_emb, timestep_cond) + + # 1.5 original resolution conditioning (see SDXL paper) + if orig_res is not None and self.res_embedding: + if not torch.is_tensor(orig_res): + h_orig, w_orig = orig_res + dtype = torch.int32 if is_mps else torch.int64 + h_orig = torch.tensor([h_orig], dtype=dtype, device=sample.device).expand(sample.shape[0]) + w_orig = torch.tensor([w_orig], dtype=dtype, device=sample.device).expand(sample.shape[0]) + else: + h_orig, w_orig = orig_res[:,0], orig_res[:,1] + + h_emb = self.height_proj(h_orig).to(dtype=sample.dtype) + w_emb = self.width_proj(w_orig).to(dtype=sample.dtype) + + emb = emb + self.height_embedding(h_emb) + emb = emb + self.width_embedding(w_emb) + + if self.time_embed_act is not None: + emb = self.time_embed_act(emb) + + # 2. pre-process + sample = self.conv_in(sample) + + # 3. down + down_block_res_samples = (sample,) + for downsample_block in self.down_blocks: + sample, res_samples = downsample_block(hidden_states=sample, temb=emb) + down_block_res_samples += res_samples + if hasattr(self, 'downsample_mid'): + sample = self.downsample_mid(sample) + + # 4. mid + sample = self.mid_block(sample, emb, condition, cond_mask) + + # 5. up + if hasattr(self, 'upsample_mid'): + sample = self.upsample_mid(sample) + for i, upsample_block in enumerate(self.up_blocks): + is_final_block = i == len(self.up_blocks) - 1 + + res_samples = down_block_res_samples[-len(upsample_block.resnets) :] + down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)] + + # if we have not reached the final block and need to forward the + # upsample size, we do it here + if not is_final_block and forward_upsample_size: + upsample_size = down_block_res_samples[-1].shape[2:] + + sample = upsample_block( + hidden_states=sample, temb=emb, res_hidden_states_tuple=res_samples, upsample_size=upsample_size + ) + + # 6. post-process + if self.conv_norm_out: + sample = self.conv_norm_out(sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample) + + return sample + + +def uvit_b_p4_f16(**kwargs): + return UViT( + patch_size=4, + block_out_channels=(128, 256), + layers_per_block=2, + downsample_before_mid=True, + mid_layers=12, + mid_num_heads=12, + mid_dim=768, + mid_mlp_ratio=4, + mid_qkv_bias=True, + **kwargs + ) + +def uvit_l_p4_f16(**kwargs): + return UViT( + patch_size=4, + block_out_channels=(128, 256), + layers_per_block=2, + downsample_before_mid=True, + mid_layers=24, + mid_num_heads=16, + mid_dim=1024, + mid_mlp_ratio=4, + mid_qkv_bias=True, + **kwargs + ) + + +def uvit_h_p4_f16(**kwargs): + return UViT( + patch_size=4, + block_out_channels=(128, 256), + layers_per_block=2, + downsample_before_mid=True, + mid_layers=32, + mid_num_heads=16, + mid_dim=1280, + mid_mlp_ratio=4, + mid_qkv_bias=True, + **kwargs + ) + +def uvit_b_p4_f16_longskip(**kwargs): + return UViT( + patch_size=4, + block_out_channels=(128, 256), + layers_per_block=2, + downsample_before_mid=True, + mid_layers=13, + mid_num_heads=12, + mid_dim=768, + mid_mlp_ratio=4, + mid_qkv_bias=True, + mid_use_long_skip=True, + **kwargs + ) + +def uvit_l_p4_f16_longskip(**kwargs): + return UViT( + patch_size=4, + block_out_channels=(128, 256), + layers_per_block=2, + downsample_before_mid=True, + mid_layers=25, + mid_num_heads=16, + mid_dim=1024, + mid_mlp_ratio=4, + mid_qkv_bias=True, + mid_use_long_skip=True, + **kwargs + ) + +def uvit_b_p4_f8(**kwargs): + return UViT( + patch_size=4, + block_out_channels=(128, 256), + layers_per_block=2, + downsample_before_mid=False, + mid_layers=12, + mid_num_heads=12, + mid_dim=768, + mid_mlp_ratio=4, + mid_qkv_bias=True, + **kwargs + ) + +def uvit_l_p4_f8(**kwargs): + return UViT( + patch_size=4, + block_out_channels=(128, 256), + layers_per_block=2, + downsample_before_mid=False, + mid_layers=24, + mid_num_heads=16, + mid_dim=1024, + mid_mlp_ratio=4, + mid_qkv_bias=True, + **kwargs + ) + +def uvit_b_p4_f16_extraconv(**kwargs): + return UViT( + patch_size=4, + block_out_channels=(128, 256, 512), + layers_per_block=2, + downsample_before_mid=False, + mid_layers=12, + mid_num_heads=12, + mid_dim=768, + mid_mlp_ratio=4, + mid_qkv_bias=True, + **kwargs + ) + +def uvit_l_p4_f16_extraconv(**kwargs): + return UViT( + patch_size=4, + block_out_channels=(128, 256, 512), + layers_per_block=2, + downsample_before_mid=False, + mid_layers=24, + mid_num_heads=16, + mid_dim=1024, + mid_mlp_ratio=4, + mid_qkv_bias=True, + **kwargs + ) + diff --git a/fourm/vq/models/vit_models.py b/fourm/vq/models/vit_models.py new file mode 100755 index 0000000000000000000000000000000000000000..e8cb363b95b50bca8bf24a3e8616bf037371db28 --- /dev/null +++ b/fourm/vq/models/vit_models.py @@ -0,0 +1,859 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +import warnings +from functools import partial +from typing import Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.cuda.amp import autocast +from einops import rearrange + +# xFormers imports +try: + from xformers.ops import memory_efficient_attention, unbind + XFORMERS_AVAILABLE = True +except ImportError: + print("xFormers not available") + XFORMERS_AVAILABLE = False + + +def pair(t): + return t if isinstance(t, tuple) else (t, t) + + +def build_2d_sincos_posemb(h, w, embed_dim=1024, temperature=10000.): + """Sine-cosine positional embeddings as used in MoCo-v3 + """ + grid_w = torch.arange(w, dtype=torch.float32) + grid_h = torch.arange(h, dtype=torch.float32) + grid_w, grid_h = torch.meshgrid(grid_w, grid_h, indexing='ij') + assert embed_dim % 4 == 0, 'Embed dimension must be divisible by 4 for 2D sin-cos position embedding' + pos_dim = embed_dim // 4 + omega = torch.arange(pos_dim, dtype=torch.float32) / pos_dim + omega = 1. / (temperature ** omega) + out_w = torch.einsum('m,d->md', [grid_w.flatten(), omega]) + out_h = torch.einsum('m,d->md', [grid_h.flatten(), omega]) + pos_emb = torch.cat([torch.sin(out_w), torch.cos(out_w), torch.sin(out_h), torch.cos(out_h)], dim=1)[None, :, :] + pos_emb = rearrange(pos_emb, 'b (h w) d -> b d h w', h=h, w=w, d=embed_dim) + return pos_emb + + +def _no_grad_trunc_normal_(tensor, mean, std, a, b): + # Cut & paste from PyTorch official master until it's in a few official releases - RW + # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf + def norm_cdf(x): + # Computes standard normal cumulative distribution function + return (1. + math.erf(x / math.sqrt(2.))) / 2. + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn("mean is more than 2 std from [a, b] in nn.init.trunc_normal_. " + "The distribution of values may be incorrect.", + stacklevel=2) + + with torch.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + l = norm_cdf((a - mean) / std) + u = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [l, u], then translate to + # [2l-1, 2u-1]. + tensor.uniform_(2 * l - 1, 2 * u - 1) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.): + # type: (Tensor, float, float, float, float) -> Tensor + r"""Fills the input Tensor with values drawn from a truncated + normal distribution. The values are effectively drawn from the + normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` + with values outside :math:`[a, b]` redrawn until they are within + the bounds. The method used for generating the random values works + best when :math:`a \leq \text{mean} \leq b`. + Args: + tensor: an n-dimensional `torch.Tensor` + mean: the mean of the normal distribution + std: the standard deviation of the normal distribution + a: the minimum cutoff value + b: the maximum cutoff value + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.trunc_normal_(w) + """ + return _no_grad_trunc_normal_(tensor, mean, std, a, b) + + +def drop_path(x, drop_prob: float = 0., training: bool = False): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + This is the same as the DropConnect impl I created for EfficientNet, etc networks, however, + the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper... + See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for + changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use + 'survival rate' as the argument. + """ + if drop_prob == 0. or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = keep_prob + torch.rand(shape, dtype=x.dtype, device=x.device) + random_tensor.floor_() # binarize + output = x.div(keep_prob) * random_tensor + return output + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + """ + + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + def extra_repr(self) -> str: + return 'p={}'.format(self.drop_prob) + + +class Mlp(nn.Module): + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + # x = self.drop(x) + # commit this for the orignal BERT implement + x = self.fc2(x) + x = self.drop(x) + return x + + +class Attention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0.): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim ** -0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x): + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads) + + if XFORMERS_AVAILABLE: + q, k, v = unbind(qkv, 2) # Each is of shape B x N x num_heads x C // num_heads + x = memory_efficient_attention(q, k, v) + x = x.reshape([B, N, C]) + else: + qkv = qkv.permute(2, 0, 3, 1, 4) + q, k, v = qkv.unbind(0) # make torchscript happy (cannot use tensor as tuple) + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class CrossAttention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0.): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim ** -0.5 + + self.q = nn.Linear(dim, dim, bias=qkv_bias) + self.kv = nn.Linear(dim, dim * 2, bias=qkv_bias) + + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, context): + B, N, C = x.shape + _, M, _ = context.shape + + q = self.q(x).reshape(B, N, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3) + kv = self.kv(context).reshape(B, M, 2, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + k, v = kv[0], kv[1] + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, -1) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class Block(nn.Module): + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm): + super().__init__() + self.norm1 = norm_layer(dim) + self.norm2 = norm_layer(dim) + self.attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + def forward(self, x, **kwargs): + x = x + self.drop_path(self.attn(self.norm1(x))) + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + + +class DecoderBlock(nn.Module): + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm): + super().__init__() + self.norm1 = norm_layer(dim) + self.self_attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) + self.cross_attn = CrossAttention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) + self.query_norm = norm_layer(dim) + self.context_norm = norm_layer(dim) + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + def forward(self, x, context, **kwargs): + x = x + self.drop_path(self.self_attn(self.norm1(x))) + x = x + self.drop_path(self.cross_attn(self.query_norm(x), self.context_norm(context))) + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + + +class LayerNorm(nn.Module): + r""" LayerNorm that supports two data formats: channels_last (default) or channels_first. + The ordering of the dimensions in the inputs. channels_last corresponds to inputs with + shape (batch_size, height, width, channels) while channels_first corresponds to inputs + with shape (batch_size, channels, height, width). + + From https://github.com/facebookresearch/ConvNeXt/blob/main/models/convnext.py + """ + def __init__(self, normalized_shape, eps=1e-6, data_format="channels_last"): + super().__init__() + self.weight = nn.Parameter(torch.ones(normalized_shape)) + self.bias = nn.Parameter(torch.zeros(normalized_shape)) + self.eps = eps + self.data_format = data_format + if self.data_format not in ["channels_last", "channels_first"]: + raise NotImplementedError + self.normalized_shape = (normalized_shape, ) + + def forward(self, x): + if self.data_format == "channels_last": + return F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps) + elif self.data_format == "channels_first": + u = x.mean(1, keepdim=True) + s = (x - u).pow(2).mean(1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.eps) + x = self.weight[:, None, None] * x + self.bias[:, None, None] + return x + +class ConvNeXtBlock(nn.Module): + r""" ConvNeXt Block. There are two equivalent implementations: + (1) DwConv -> LayerNorm (channels_first) -> 1x1 Conv -> GELU -> 1x1 Conv; all in (N, C, H, W) + (2) DwConv -> Permute to (N, H, W, C); LayerNorm (channels_last) -> Linear -> GELU -> Linear; Permute back + We use (2) as we find it slightly faster in PyTorch. + + From https://github.com/facebookresearch/ConvNeXt/blob/main/models/convnext.py + + Args: + dim (int): Number of input channels. + drop_path (float): Stochastic depth rate. Default: 0.0 + layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6. + """ + def __init__(self, dim, drop_path=0., layer_scale_init_value=1e-6): + super().__init__() + self.dwconv = nn.Conv2d(dim, dim, kernel_size=7, padding=3, groups=dim) # depthwise conv + self.norm = nn.LayerNorm(dim, eps=1e-6) + self.pwconv1 = nn.Linear(dim, 4 * dim) # pointwise/1x1 convs, implemented with linear layers + self.act = nn.GELU() + self.pwconv2 = nn.Linear(4 * dim, dim) + self.gamma = nn.Parameter(layer_scale_init_value * torch.ones((dim)), + requires_grad=True) if layer_scale_init_value > 0 else None + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + + def forward(self, x): + input = x + x = self.dwconv(x) + x = x.permute(0, 2, 3, 1) # (N, C, H, W) -> (N, H, W, C) + x = self.norm(x) + x = self.pwconv1(x) + x = self.act(x) + x = self.pwconv2(x) + if self.gamma is not None: + x = self.gamma * x + x = x.permute(0, 3, 1, 2) # (N, H, W, C) -> (N, C, H, W) + + x = input + self.drop_path(x) + return x + + +class ViTEncoder(nn.Module): + """Transformer to map images / feature maps to latent features. + + Args: + in_channels: Number of input channels. + patch_size: Patch size. + resolution: Image resolution. + dim_tokens: Transformer dimension. + depth: Number of transformer layers. + num_heads: Number of attention heads. + mlp_ratio: MLP ratio. + qkv_bias: If True, add bias to the qkv projection. + drop_rate: Dropout rate. + attn_drop_rate: Attention dropout rate. + drop_path_rate: Stochastic depth rate. + norm_layer: Normalization layer. + sincos_pos_emb: If True, use sine-cosine positional embedding. + learnable_pos_emb: If True, learn positional embedding. + patch_proj: If True, project image patches to tokens. + Consider disabling when encoding feature maps. + post_mlp: If True, add MLP after transformer. + See https://arxiv.org/abs/2110.04627. + ckpt_path: Path to checkpoint to load. + """ + def __init__(self, *, + in_channels: int = 3, + patch_size: int = 16, + resolution: int = 256, + dim_tokens: int = 768, + depth: int = 12, + num_heads: int = 12, + mlp_ratio: float = 4.0, + qkv_bias: bool = True, + drop_rate: float = 0.0, + attn_drop_rate: float = 0.0, + drop_path_rate: float = 0.0, + norm_layer: nn.Module = partial(nn.LayerNorm, eps=1e-6), + sincos_pos_emb: bool = True, + learnable_pos_emb: bool = False, + patch_proj: bool = True, + post_mlp: bool = False, + ckpt_path: Optional[str] = None, + **ignore_kwargs): + super().__init__() + self.in_channels = in_channels + self.P_H, self.P_W = pair(patch_size) + self.H, self.W = pair(resolution) + self.dim_tokens = dim_tokens + self.patch_proj = patch_proj + + assert (self.H % self.P_H == 0) and (self.W % self.P_W == 0), f'Image sizes {self.H}x{self.W} must be divisible by patch sizes {self.P_H}x{self.P_W}' + + N_H = self.H // self.P_H + N_W = self.W // self.P_W + + if sincos_pos_emb: + self.pos_emb = build_2d_sincos_posemb(h=N_H, w=N_W, embed_dim=self.dim_tokens) + self.pos_emb = nn.Parameter(self.pos_emb, requires_grad=learnable_pos_emb) + else: + self.pos_emb = nn.Parameter(torch.zeros(1, self.dim_tokens, N_H, N_W)) + trunc_normal_(self.pos_emb, std=0.02) + + # Image patches -> tokens projection + if patch_proj: + self.proj = nn.Conv2d( + in_channels=self.in_channels, out_channels=self.dim_tokens, + kernel_size=(self.P_H, self.P_W), stride=(self.P_H, self.P_W) + ) + else: + self.proj = nn.Conv2d( + in_channels=self.in_channels, out_channels=self.dim_tokens, + kernel_size=1, stride=1 + ) + + # Transformer blocks + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule + self.blocks = nn.Sequential(*[ + Block(dim=dim_tokens, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer) + for i in range(depth) + ]) + + if post_mlp: + self.norm_mlp = norm_layer(dim_tokens) + self.post_mlp = Mlp(dim_tokens, int(mlp_ratio*dim_tokens), act_layer=nn.Tanh) + + self.apply(self._init_weights) + for name, m in self.named_modules(): + if isinstance(m, nn.Linear): + if 'qkv' in name: + # treat the weights of Q, K, V separately + val = math.sqrt(6. / float(m.weight.shape[0] // 3 + m.weight.shape[1])) + nn.init.uniform_(m.weight, -val, val) + elif 'kv' in name: + # treat the weights of K, V separately + val = math.sqrt(6. / float(m.weight.shape[0] // 2 + m.weight.shape[1])) + nn.init.uniform_(m.weight, -val, val) + + if isinstance(m, nn.Conv2d): + if '.proj' in name: + # From MAE, initialize projection like nn.Linear (instead of nn.Conv2d) + w = m.weight.data + nn.init.xavier_uniform_(w.view([w.shape[0], -1])) + + if ckpt_path is not None: + print(f'Loading checkpoint from {ckpt_path}') + ckpt = torch.load(ckpt_path) + ckpt['model']['pos_emb'] = rearrange(ckpt['model']['pos_embed'][:,1:], 'b (nh nw) d -> b d nh nw', nh=N_H, nw=N_W) + ckpt['model']['proj.weight'] = ckpt['model']['patch_embed.proj.weight'] + ckpt['model']['proj.bias'] = ckpt['model']['patch_embed.proj.bias'] + msg = self.load_state_dict(ckpt['model'], strict=False) + print(msg) + + def _init_weights(self, m: nn.Module) -> None: + """Weight initialization""" + if isinstance(m, nn.Linear): + nn.init.xavier_uniform_(m.weight) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def get_num_layers(self) -> int: + """Get number of transformer layers.""" + return len(self.blocks) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ViT encoder forward pass. + + Args: + x: Input tensor of shape [B, C, H, W] or + [B, C, N_H, N_W] (patch projection disabled). + + Returns: + Output tensor of shape [B, dim_tokens, N_H, N_W]. + """ + # Create patches [B, C, H, W] -> [B, (H*W), C] + if self.patch_proj: + B, C, H, W = x.shape + assert (H % self.P_H == 0) and (W % self.P_W == 0), f'Image sizes {H}x{W} must be divisible by patch sizes {self.P_H}x{self.P_W}' + N_H, N_W = H // self.P_H, W // self.P_W # Number of patches in height and width + else: + B, C, N_H, N_W = x.shape + x = rearrange(self.proj(x), 'b d nh nw -> b (nh nw) d') + + if self.pos_emb is not None: + # Create positional embedding + x_pos_emb = F.interpolate(self.pos_emb, size=(N_H, N_W), mode='bicubic', align_corners=False) + x_pos_emb = rearrange(x_pos_emb, 'b d nh nw -> b (nh nw) d') + # Add positional embeddings to patches + x = x + x_pos_emb + + # Transformer forward pass + x = self.blocks(x) + + if hasattr(self, 'post_mlp'): + with autocast(enabled = False): + x = x.float() + self.post_mlp(self.norm_mlp(x.float())) + + # Reshape into 2D grid + x = rearrange(x, 'b (nh nw) d -> b d nh nw', nh=N_H, nw=N_W) + + return x + + +class ViTDecoder(nn.Module): + """Transformer to map latent features back to images / feature maps. + + Args: + out_channels: Number of output channels. + patch_size: Patch size. + resolution: Image resolution. + dim_tokens: Transformer dimension. + depth: Number of transformer layers. + num_heads: Number of attention heads. + mlp_ratio: MLP ratio. + qkv_bias: If True, add bias to the qkv projection. + drop_rate: Dropout rate. + attn_drop_rate: Attention dropout rate. + drop_path_rate: Stochastic depth rate. + norm_layer: Normalization layer. + sincos_pos_emb: If True, use sine-cosine positional embedding. + learnable_pos_emb: If True, learn positional embedding. + patch_proj: If True, reproject tokens back to images. + Consider disabling when encoding feature maps. + post_mlp: If True, add MLP before transformer. + See https://arxiv.org/abs/2110.04627. + out_conv: If True, add two ConvNeXt blocks after transformer + to deal with patch checkerboard artifacts. + """ + def __init__(self, *, + out_channels: int = 3, + patch_size: int = 16, + resolution: int = 256, + dim_tokens: int = 768, + depth: int = 12, + num_heads: int = 12, + mlp_ratio: float = 4.0, + qkv_bias: bool = True, + drop_rate: float = 0.0, + attn_drop_rate: float = 0.0, + drop_path_rate: float = 0.0, + norm_layer: nn.Module = partial(nn.LayerNorm, eps=1e-6), + sincos_pos_emb: bool = True, + learnable_pos_emb: bool = False, + patch_proj: bool = True, + post_mlp: bool = False, + out_conv: bool = False, + **ignore_kwargs): + super().__init__() + self.out_channels = out_channels + self.P_H, self.P_W = pair(patch_size) + self.H, self.W = pair(resolution) + self.dim_tokens = dim_tokens + self.patch_proj = patch_proj + + assert (self.H % self.P_H == 0) and (self.W % self.P_W == 0), f'Image sizes {self.H}x{self.W} must be divisible by patch sizes {self.P_H}x{self.P_W}' + + N_H = self.H // self.P_H + N_W = self.W // self.P_W + + if sincos_pos_emb: + self.pos_emb = build_2d_sincos_posemb(h=N_H, w=N_W, embed_dim=self.dim_tokens) + self.pos_emb = nn.Parameter(self.pos_emb, requires_grad=learnable_pos_emb) + else: + self.pos_emb = nn.Parameter(torch.zeros(1, self.dim_tokens, N_H, N_W)) + trunc_normal_(self.pos_emb, std=0.02) + + # Transformer blocks + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule + self.blocks = nn.Sequential(*[ + Block(dim=dim_tokens, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer) + for i in range(depth) + ]) + + # Tokens -> image output projection + if post_mlp: + self.norm_mlp = norm_layer(dim_tokens) + self.post_mlp = Mlp(dim_tokens, int(mlp_ratio*dim_tokens), act_layer=nn.Tanh) + if patch_proj: + self.out_proj = nn.Linear(dim_tokens, self.out_channels * self.P_H * self.P_W) + else: + self.out_proj = nn.Linear(dim_tokens, self.out_channels) + if out_conv: + self.out_conv = nn.Sequential(ConvNeXtBlock(dim=self.out_channels), ConvNeXtBlock(dim=self.out_channels)) + + self.apply(self._init_weights) + for name, m in self.named_modules(): + if isinstance(m, nn.Linear): + if 'qkv' in name: + # treat the weights of Q, K, V separately + val = math.sqrt(6. / float(m.weight.shape[0] // 3 + m.weight.shape[1])) + nn.init.uniform_(m.weight, -val, val) + elif 'kv' in name: + # treat the weights of K, V separately + val = math.sqrt(6. / float(m.weight.shape[0] // 2 + m.weight.shape[1])) + nn.init.uniform_(m.weight, -val, val) + + if isinstance(m, nn.Conv2d): + if '.proj' in name: + # From MAE, initialize projection like nn.Linear (instead of nn.Conv2d) + w = m.weight.data + nn.init.xavier_uniform_(w.view([w.shape[0], -1])) + + def _init_weights(self, m: nn.Module) -> None: + """Weight initialization""" + if isinstance(m, nn.Linear): + nn.init.xavier_uniform_(m.weight) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def get_num_layers(self) -> int: + """Get number of transformer layers.""" + return len(self.blocks) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ViT decoder forward pass. + + Args: + x: Input tensor of shape [B, dim_tokens, N_H, N_W]. + + Returns: + Output tensor of shape [B, C, H, W] or + [B, C, N_H, N_W] (patch projection disabled). + """ + B, D, N_H, N_W = x.shape + + # Reshape into 1D + x = rearrange(x, 'b d nh nw -> b (nh nw) d') + + if self.pos_emb is not None: + # Create positional embedding + x_pos_emb = F.interpolate(self.pos_emb, size=(N_H, N_W), mode='bicubic', align_corners=False) + x_pos_emb = rearrange(x_pos_emb, 'b d nh nw -> b (nh nw) d') + # Add positional embeddings to patches + x = x + x_pos_emb + + # Transformer forward pass + x = self.blocks(x) + + # Project each token to (C * P_H * P_W) + if hasattr(self, 'post_mlp'): + x = x + self.post_mlp(self.norm_mlp(x)) + x = self.out_proj(x) + + # Reshape sequence of patches into image or output features + ph, pw = (self.P_H, self.P_W) if self.patch_proj else (1, 1) + x = rearrange( + x, 'b (nh nw) (c ph pw) -> b c (nh ph) (nw pw)', + nh=N_H, nw=N_W, ph=ph, pw=pw, c=self.out_channels + ) + + # Optional conv layers to reduce patch artifacts + if hasattr(self, 'out_conv'): + x = self.out_conv(x) + + return x + + +# Encoder presets + +def vit_s_enc(in_channels, + patch_size, + resolution, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.0, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + sincos_pos_emb=True, + learnable_pos_emb=False, + patch_proj=True, + post_mlp=False): + model = ViTEncoder( + in_channels=in_channels, + patch_size=patch_size, + resolution=resolution, + dim_tokens=512, + depth=8, + num_heads=8, + mlp_ratio=4, + qkv_bias=True, + drop_rate=drop_rate, + attn_drop_rate=attn_drop_rate, + drop_path_rate=drop_path_rate, + norm_layer=norm_layer, + sincos_pos_emb=sincos_pos_emb, + learnable_pos_emb=learnable_pos_emb, + patch_proj=patch_proj, + post_mlp=post_mlp, + ) + return model + +def vit_b_enc(in_channels, + patch_size, + resolution, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.0, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + sincos_pos_emb=True, + learnable_pos_emb=False, + patch_proj=True, + post_mlp=False, + ckpt_path=None): + model = ViTEncoder( + in_channels=in_channels, + patch_size=patch_size, + resolution=resolution, + dim_tokens=768, + depth=12, + num_heads=12, + mlp_ratio=4, + qkv_bias=True, + drop_rate=drop_rate, + attn_drop_rate=attn_drop_rate, + drop_path_rate=drop_path_rate, + norm_layer=norm_layer, + sincos_pos_emb=sincos_pos_emb, + learnable_pos_emb=learnable_pos_emb, + patch_proj=patch_proj, + post_mlp=post_mlp, + ckpt_path=ckpt_path, + ) + return model + +def vit_l_enc(in_channels, + patch_size, + resolution, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.0, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + sincos_pos_emb=True, + learnable_pos_emb=False, + patch_proj=True, + post_mlp=False): + model = ViTEncoder( + in_channels=in_channels, + patch_size=patch_size, + resolution=resolution, + dim_tokens=1024, + depth=24, + num_heads=16, + mlp_ratio=4, + qkv_bias=True, + drop_rate=drop_rate, + attn_drop_rate=attn_drop_rate, + drop_path_rate=drop_path_rate, + norm_layer=norm_layer, + sincos_pos_emb=sincos_pos_emb, + learnable_pos_emb=learnable_pos_emb, + patch_proj=patch_proj, + post_mlp=post_mlp, + ) + return model + + +# Decoder presets + +def vit_s_dec(out_channels, + patch_size, + resolution, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.0, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + sincos_pos_emb=True, + learnable_pos_emb=False, + patch_proj=True, + post_mlp=False, + out_conv=False): + model = ViTDecoder( + out_channels=out_channels, + patch_size=patch_size, + resolution=resolution, + dim_tokens=512, + depth=8, + num_heads=8, + mlp_ratio=4, + qkv_bias=True, + drop_rate=drop_rate, + attn_drop_rate=attn_drop_rate, + drop_path_rate=drop_path_rate, + norm_layer=norm_layer, + sincos_pos_emb=sincos_pos_emb, + learnable_pos_emb=learnable_pos_emb, + patch_proj=patch_proj, + post_mlp=post_mlp, + out_conv=out_conv, + ) + return model + +def vit_b_dec(out_channels, + patch_size, + resolution, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.0, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + sincos_pos_emb=True, + learnable_pos_emb=False, + patch_proj=True, + post_mlp=False, + out_conv=False): + model = ViTDecoder( + out_channels=out_channels, + patch_size=patch_size, + resolution=resolution, + dim_tokens=768, + depth=12, + num_heads=12, + mlp_ratio=4, + qkv_bias=True, + drop_rate=drop_rate, + attn_drop_rate=attn_drop_rate, + drop_path_rate=drop_path_rate, + norm_layer=norm_layer, + sincos_pos_emb=sincos_pos_emb, + learnable_pos_emb=learnable_pos_emb, + patch_proj=patch_proj, + post_mlp=post_mlp, + out_conv=out_conv, + ) + return model + +def vit_l_dec(out_channels, + patch_size, + resolution, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.0, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + sincos_pos_emb=True, + learnable_pos_emb=False, + patch_proj=True, + post_mlp=False, + out_conv=False): + model = ViTDecoder( + out_channels=out_channels, + patch_size=patch_size, + resolution=resolution, + dim_tokens=1024, + depth=24, + num_heads=16, + mlp_ratio=4, + qkv_bias=True, + drop_rate=drop_rate, + attn_drop_rate=attn_drop_rate, + drop_path_rate=drop_path_rate, + norm_layer=norm_layer, + sincos_pos_emb=sincos_pos_emb, + learnable_pos_emb=learnable_pos_emb, + patch_proj=patch_proj, + post_mlp=post_mlp, + out_conv=out_conv, + ) + return model diff --git a/fourm/vq/percept_losses/__init__.py b/fourm/vq/percept_losses/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/fourm/vq/percept_losses/lpips.py b/fourm/vq/percept_losses/lpips.py new file mode 100644 index 0000000000000000000000000000000000000000..1ad4b0fc871babfd6a5946d4bc3cdba12686e6f4 --- /dev/null +++ b/fourm/vq/percept_losses/lpips.py @@ -0,0 +1,177 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +From: https://github.com/CompVis/taming-transformers +Stripped version of https://github.com/richzhang/PerceptualSimilarity/tree/master/models +""" + +import torch +import torch.nn as nn +from torchvision import models +from collections import namedtuple +import os, hashlib +import requests +from tqdm import tqdm + + +URL_MAP = { + "vgg_lpips": "https://heibox.uni-heidelberg.de/f/607503859c864bc1b30b/?dl=1" +} + +CKPT_MAP = { + "vgg_lpips": "vgg.pth" +} + +MD5_MAP = { + "vgg_lpips": "d507d7349b931f0638a25a48a722f98a" +} + +def download(url, local_path, chunk_size=1024): + os.makedirs(os.path.split(local_path)[0], exist_ok=True) + with requests.get(url, stream=True) as r: + total_size = int(r.headers.get("content-length", 0)) + with tqdm(total=total_size, unit="B", unit_scale=True) as pbar: + with open(local_path, "wb") as f: + for data in r.iter_content(chunk_size=chunk_size): + if data: + f.write(data) + pbar.update(chunk_size) + +def md5_hash(path): + with open(path, "rb") as f: + content = f.read() + return hashlib.md5(content).hexdigest() + +def get_ckpt_path(name, root, check=False): + assert name in URL_MAP + path = os.path.join(root, CKPT_MAP[name]) + if not os.path.exists(path) or (check and not md5_hash(path) == MD5_MAP[name]): + print("Downloading {} model from {} to {}".format(name, URL_MAP[name], path)) + download(URL_MAP[name], path) + md5 = md5_hash(path) + assert md5 == MD5_MAP[name], md5 + return path + +class LPIPS(nn.Module): + # Learned perceptual metric + def __init__(self, use_dropout=True): + super().__init__() + self.scaling_layer = ScalingLayer() + self.chns = [64, 128, 256, 512, 512] # vg16 features + self.net = vgg16(pretrained=True, requires_grad=False) + self.lin0 = NetLinLayer(self.chns[0], use_dropout=use_dropout) + self.lin1 = NetLinLayer(self.chns[1], use_dropout=use_dropout) + self.lin2 = NetLinLayer(self.chns[2], use_dropout=use_dropout) + self.lin3 = NetLinLayer(self.chns[3], use_dropout=use_dropout) + self.lin4 = NetLinLayer(self.chns[4], use_dropout=use_dropout) + self.load_from_pretrained() + for param in self.parameters(): + param.requires_grad = False + + def load_from_pretrained(self, name="vgg_lpips"): + ckpt = get_ckpt_path(name, "/tmp/lpips") + self.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False) + print("loaded pretrained LPIPS loss from {}".format(ckpt)) + + @classmethod + def from_pretrained(cls, name="vgg_lpips"): + if name != "vgg_lpips": + raise NotImplementedError + model = cls() + ckpt = get_ckpt_path(name) + model.load_state_dict(torch.load(ckpt, map_location=torch.device("cpu")), strict=False) + return model + + def forward(self, input, target): + in0_input, in1_input = (self.scaling_layer(input), self.scaling_layer(target)) + outs0, outs1 = self.net(in0_input), self.net(in1_input) + feats0, feats1, diffs = {}, {}, {} + lins = [self.lin0, self.lin1, self.lin2, self.lin3, self.lin4] + for kk in range(len(self.chns)): + feats0[kk], feats1[kk] = normalize_tensor(outs0[kk]), normalize_tensor(outs1[kk]) + diffs[kk] = (feats0[kk] - feats1[kk]) ** 2 + + res = [spatial_average(lins[kk].model(diffs[kk]), keepdim=True) for kk in range(len(self.chns))] + val = res[0] + for l in range(1, len(self.chns)): + val += res[l] + return val + + +class ScalingLayer(nn.Module): + def __init__(self): + super(ScalingLayer, self).__init__() + self.register_buffer('shift', torch.Tensor([-.030, -.088, -.188])[None, :, None, None]) + self.register_buffer('scale', torch.Tensor([.458, .448, .450])[None, :, None, None]) + + def forward(self, inp): + return (inp - self.shift) / self.scale + + +class NetLinLayer(nn.Module): + """ A single linear layer which does a 1x1 conv """ + def __init__(self, chn_in, chn_out=1, use_dropout=False): + super(NetLinLayer, self).__init__() + layers = [nn.Dropout(), ] if (use_dropout) else [] + layers += [nn.Conv2d(chn_in, chn_out, 1, stride=1, padding=0, bias=False), ] + self.model = nn.Sequential(*layers) + + +class vgg16(torch.nn.Module): + def __init__(self, requires_grad=False, pretrained=True): + super(vgg16, self).__init__() + vgg_pretrained_features = models.vgg16(pretrained=pretrained).features + self.slice1 = torch.nn.Sequential() + self.slice2 = torch.nn.Sequential() + self.slice3 = torch.nn.Sequential() + self.slice4 = torch.nn.Sequential() + self.slice5 = torch.nn.Sequential() + self.N_slices = 5 + for x in range(4): + self.slice1.add_module(str(x), vgg_pretrained_features[x]) + for x in range(4, 9): + self.slice2.add_module(str(x), vgg_pretrained_features[x]) + for x in range(9, 16): + self.slice3.add_module(str(x), vgg_pretrained_features[x]) + for x in range(16, 23): + self.slice4.add_module(str(x), vgg_pretrained_features[x]) + for x in range(23, 30): + self.slice5.add_module(str(x), vgg_pretrained_features[x]) + if not requires_grad: + for param in self.parameters(): + param.requires_grad = False + + def forward(self, X): + h = self.slice1(X) + h_relu1_2 = h + h = self.slice2(h) + h_relu2_2 = h + h = self.slice3(h) + h_relu3_3 = h + h = self.slice4(h) + h_relu4_3 = h + h = self.slice5(h) + h_relu5_3 = h + vgg_outputs = namedtuple("VggOutputs", ['relu1_2', 'relu2_2', 'relu3_3', 'relu4_3', 'relu5_3']) + out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3, h_relu5_3) + return out + + +def normalize_tensor(x,eps=1e-10): + norm_factor = torch.sqrt(torch.sum(x**2,dim=1,keepdim=True)) + return x/(norm_factor+eps) + + +def spatial_average(x, keepdim=True): + return x.mean([2,3],keepdim=keepdim) \ No newline at end of file diff --git a/fourm/vq/percept_losses/timm_perceptual_loss.py b/fourm/vq/percept_losses/timm_perceptual_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..b735cea871bc5a7e55ff97c8698cb21439555acd --- /dev/null +++ b/fourm/vq/percept_losses/timm_perceptual_loss.py @@ -0,0 +1,111 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List, Union +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision.transforms as transforms +import torchvision.transforms.functional as TF +from torchvision.models.feature_extraction import create_feature_extractor +from einops import rearrange +import timm + + +class TimmPerceptualLoss(nn.Module): + """Perceptual loss module using features from arbitrary timm models. + + Args: + model_id: timm model id. E.g. 'vit_base_patch14_dinov2.lvd142m' + feature_ids: List or hyphen-separated string of feature names to extract from the model. + For example, 'blocks.2-blocks.5-blocks.8-blocks.11'. To list all available features, use: + ```python + from torchvision.models.feature_extraction import get_graph_node_names + nodes, _ = get_graph_node_names(model) + ``` + feature_loss: Feature loss to use. One of ['cosine' or 'cos', 'l1' or 'mae']. Default: 'cosine'. + If 'l1' or 'mae' is used, the features will be normalized first. If 'cosine' or 'cos' is + used, the features will not be normalized, but the cosine similarity will be computed, + which is equivalent to normalization + MSE up to a factor of 2. + """ + def __init__(self, + model_id: str, + feature_ids: Union[str, List[str]], + feature_loss: str = 'cosine'): + super().__init__() + + feature_ids = feature_ids.split('-') if isinstance(feature_ids, str) else feature_ids + + self.feature_ids = feature_ids + self.feature_loss = feature_loss + + self.model = timm.create_model(model_id, pretrained=True) + self.feature_extractor = create_feature_extractor(self.model, return_nodes=self.feature_ids) + + # Transforms to preprocess inputs to the model + self.data_config = timm.data.resolve_model_data_config(self.model) + self.percept_transform = transforms.Compose([ + transforms.Normalize((-1.0, -1.0, -1.0), (2.0, 2.0, 2.0)), # [-1, 1] -> [0, 1] + transforms.Normalize(self.data_config['mean'], self.data_config['std']), # [0, 1] -> standardize with pre-computed statistics + transforms.Resize(self.data_config['input_size'][-2:], interpolation=TF.InterpolationMode.BILINEAR, antialias=True), + ]) + + def forward(self, preds: torch.Tensor, targets: torch.Tensor, preprocess_inputs=False) -> torch.Tensor: + """ + Compute perceptual loss between predictions and targets. If + preprocess_inputs is True, it is assumed that the targets are + scaled to the [-1, 1] range. Predictions will be scaled + assuming the same input range. + + Args: + preds: Predictions tensor of shape (B, C, H, W) + targets: Targets tensor of shape (B, C, H, W) + preprocess_inputs: If inputs are scaled to [-1, 1], enable + this to apply model specific preprocessing. Default: False. + + Returns: + Perceptual loss between predictions and targets. + """ + # Preprocess predictions and targets for the given feature extractor + if preprocess_inputs: + preds = self.percept_transform(preds) + targets = self.percept_transform(targets) + + # Extract features from predictions and targets + # Each is a dict of feature_name: feature_tensor + feats_preds = self.feature_extractor(preds) + feats_targets = self.feature_extractor(targets) + + loss = 0 + for feat_name in feats_preds.keys(): + + # Get individual feature map and reshape from (B, C, H, W) to (B, N, C) if needed + feat_preds = feats_preds[feat_name] + feat_targets = feats_targets[feat_name] + if feat_preds.ndim == 4: + feat_preds = rearrange(feat_preds, 'b c h w -> b (h w) c') + feat_targets = rearrange(feat_targets, 'b c h w -> b (h w) c') + + # Compute feature-wise loss + if self.feature_loss in ['l1', 'mae']: + feat_preds = F.normalize(feat_preds, dim=-1) + feat_targets = F.normalize(feat_targets, dim=-1) + loss += F.l1_loss(feat_preds, feat_targets, reduction='none').sum(-1).mean(-1) + elif self.feature_loss in ['cosine', 'cos']: + loss += 1 - F.cosine_similarity(feat_preds, feat_targets, dim=-1).mean(dim=-1) + else: + raise ValueError(f'Unknown feature loss: {self.feature_loss}') + + loss /= preds.shape[0] + + return loss \ No newline at end of file diff --git a/fourm/vq/quantizers/__init__.py b/fourm/vq/quantizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4f8daf8d498779d5339257384d729c0bf0694ba4 --- /dev/null +++ b/fourm/vq/quantizers/__init__.py @@ -0,0 +1,2 @@ +from .quantize_lucid import VectorQuantize as VectorQuantizerLucid +from .quantize_memcodes import Memcodes diff --git a/fourm/vq/quantizers/quantize_lucid.py b/fourm/vq/quantizers/quantize_lucid.py new file mode 100755 index 0000000000000000000000000000000000000000..5ca26e61fbde6611bf43aa4f127c342b03d38767 --- /dev/null +++ b/fourm/vq/quantizers/quantize_lucid.py @@ -0,0 +1,568 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This code is strongly influenced by https://github.com/lucidrains/vector-quantize-pytorch + +import torch +from torch import nn, einsum +import torch.nn.functional as F +import torch.distributed as distributed +from torch.cuda.amp import autocast + +from einops import rearrange, repeat +from contextlib import contextmanager + +def exists(val): + return val is not None + +def default(val, d): + return val if exists(val) else d + +def noop(*args, **kwargs): + pass + +def l2norm(t): + return F.normalize(t, p = 2, dim = -1) + +def log(t, eps = 1e-20): + return torch.log(t.clamp(min = eps)) + +def uniform_init(*shape): + t = torch.empty(shape) + nn.init.kaiming_uniform_(t) + return t + +def gumbel_noise(t): + noise = torch.zeros_like(t).uniform_(0, 1) + return -log(-log(noise)) + +def gumbel_sample(t, temperature = 1., dim = -1): + if temperature == 0: + return t.argmax(dim = dim) + + return ((t / temperature) + gumbel_noise(t)).argmax(dim = dim) + +def ema_inplace(moving_avg, new, decay): + moving_avg.data.mul_(decay).add_(new, alpha = (1 - decay)) + +def laplace_smoothing(x, n_categories, eps = 1e-5): + return (x + eps) / (x.sum() + n_categories * eps) + +def sample_vectors(samples, num): + num_samples, device = samples.shape[0], samples.device + + if num_samples >= num: + indices = torch.randperm(num_samples, device = device)[:num] + else: + indices = torch.randint(0, num_samples, (num,), device = device) + + return samples[indices] + +def pad_shape(shape, size, dim = 0): + return [size if i == dim else s for i, s in enumerate(shape)] + +def sample_multinomial(total_count, probs): + device = probs.device + probs = probs.cpu() + + total_count = probs.new_full((), total_count) + remainder = probs.new_ones(()) + sample = torch.empty_like(probs, dtype = torch.long) + + for i, p in enumerate(probs): + s = torch.binomial(total_count, p / remainder) + sample[i] = s + total_count -= s + remainder -= p + + return sample.to(device) + +def all_gather_sizes(x, dim): + size = torch.tensor(x.shape[dim], dtype = torch.long, device = x.device) + all_sizes = [torch.empty_like(size) for _ in range(distributed.get_world_size())] + distributed.all_gather(all_sizes, size) + + return torch.stack(all_sizes) + +def all_gather_variably_sized(x, sizes, dim = 0): + rank = distributed.get_rank() + all_x = [] + + for i, size in enumerate(sizes): + t = x if i == rank else x.new_empty(pad_shape(x.shape, size, dim)) + distributed.broadcast(t, src = i, async_op = True) + all_x.append(t) + + distributed.barrier() + return all_x + +def sample_vectors_distributed(local_samples, num): + rank = distributed.get_rank() + all_num_samples = all_gather_sizes(local_samples, dim = 0) + + if rank == 0: + samples_per_rank = sample_multinomial(num, all_num_samples / all_num_samples.sum()) + else: + samples_per_rank = torch.empty_like(all_num_samples) + + distributed.broadcast(samples_per_rank, src = 0) + samples_per_rank = samples_per_rank.tolist() + + local_samples = sample_vectors(local_samples, samples_per_rank[rank]) + all_samples = all_gather_variably_sized(local_samples, samples_per_rank, dim = 0) + return torch.cat(all_samples, dim = 0) + +def add_noise(x, eps=1e-10): + return x + torch.randn_like(x) * eps + +def add_noise_distributed(x, eps=1e-10): + if distributed.get_rank() == 0: + randn_noise = torch.randn_like(x) + else: + randn_noise = torch.empty_like(x) + distributed.broadcast(randn_noise, src = 0) + return x + randn_noise * eps + +def kmeans(samples, num_clusters, num_iters = 10, use_cosine_sim = False, + sample_fn = sample_vectors, all_reduce_fn = noop): + dim, dtype, device = samples.shape[-1], samples.dtype, samples.device + means = sample_fn(samples, num_clusters) + + for _ in range(num_iters): + if use_cosine_sim: + dists = samples @ means.t() + else: + diffs = rearrange(samples, 'n d -> n () d') \ + - rearrange(means, 'c d -> () c d') + dists = -(diffs ** 2).sum(dim = -1) + + buckets = torch.argmax(dists, dim = -1) + bins = torch.bincount(buckets, minlength = num_clusters) + all_reduce_fn(bins) + + zero_mask = bins == 0 + bins_min_clamped = bins.masked_fill(zero_mask, 1) + + new_means = buckets.new_zeros(num_clusters, dim, dtype = dtype) + new_means.scatter_add_(0, repeat(buckets, 'n -> n d', d = dim), samples) + new_means = new_means / bins_min_clamped[..., None] + all_reduce_fn(new_means) + + if use_cosine_sim: + new_means = l2norm(new_means) + + means = torch.where(zero_mask[..., None], means, new_means) + + return means, bins + +# regularization losses + +def orthgonal_loss_fn(t): + # eq (2) from https://arxiv.org/abs/2112.00384 + n = t.shape[0] + normed_codes = l2norm(t) + identity = torch.eye(n, device = t.device) + cosine_sim = einsum('i d, j d -> i j', normed_codes, normed_codes) + return ((cosine_sim - identity) ** 2).sum() / (n ** 2) + +# distance types + +class EuclideanCodebook(nn.Module): + def __init__( + self, + dim, + codebook_size, + kmeans_init = False, + kmeans_iters = 10, + decay = 0.8, + eps = 1e-5, + threshold_ema_dead_code = 2, + code_replacement_policy = 'batch_random', # batch_random or linde_buzo_gray + use_ddp = False, + learnable_codebook = False, + sample_codebook_temp = 0 + ): + super().__init__() + self.decay = decay + init_fn = uniform_init if not kmeans_init else torch.zeros + embed = init_fn(codebook_size, dim) + + self.codebook_size = codebook_size + self.kmeans_iters = kmeans_iters + self.eps = eps + self.threshold_ema_dead_code = threshold_ema_dead_code + self.code_replacement_policy = code_replacement_policy + self.sample_codebook_temp = sample_codebook_temp + + self.sample_fn = sample_vectors_distributed if use_ddp else sample_vectors + self.all_reduce_fn = distributed.all_reduce if use_ddp else noop + self.add_noise_fn = add_noise_distributed if use_ddp else add_noise + + self.register_buffer('initted', torch.Tensor([not kmeans_init])) + self.register_buffer('cluster_size', torch.zeros(codebook_size)) + self.register_buffer('embed_avg', embed.clone()) + + self.learnable_codebook = learnable_codebook + if learnable_codebook: + self.embed = nn.Parameter(embed) + else: + self.register_buffer('embed', embed) + + @torch.jit.ignore + def init_embed_(self, data): + if self.initted: + return + + embed, cluster_size = kmeans(data, self.codebook_size, self.kmeans_iters, + sample_fn = self.sample_fn, all_reduce_fn = self.all_reduce_fn) + + self.embed.data.copy_(embed) + self.embed_avg.data.copy_(embed.clone()) + self.cluster_size.data.copy_(cluster_size) + self.initted.data.copy_(torch.Tensor([True])) + + def replace_batch_random(self, samples, mask): + samples = l2norm(samples) + self.embed.data[mask] = self.sample_fn(samples, mask.sum().item()) + + def replace_linde_buzo_gray(self, mask): + num_unused = mask.sum() + most_used_idxs = self.cluster_size.argsort(descending=True)[:num_unused] + most_used_codes = self.embed.data[most_used_idxs] + self.embed.data[mask] = l2norm(self.add_noise_fn(most_used_codes)) + + def expire_codes_(self, batch_samples): + if self.threshold_ema_dead_code == 0: + return + + expired_codes = self.cluster_size < self.threshold_ema_dead_code + if not torch.any(expired_codes): + return + + if self.code_replacement_policy == 'batch_random': + # Replace dead codes by random latents from encoder + batch_samples = rearrange(batch_samples, '... d -> (...) d') + self.replace_batch_random(batch_samples, mask = expired_codes) + elif self.code_replacement_policy == 'linde_buzo_gray': + # Replace dead codes by most used codes + some noise (Linde-Buzo-Gray splitting algorithm) + self.replace_linde_buzo_gray(mask = expired_codes) + else: + raise ValueError(f'{self.code_replacement_policy} is not a valid dead code replacement strategy.') + + @autocast(enabled = False) + def forward(self, x): + x = x.float() + + shape, dtype = x.shape, x.dtype + flatten = rearrange(x, '... d -> (...) d') + + self.init_embed_(flatten) + + embed = self.embed if not self.learnable_codebook else self.embed.detach() + embed = self.embed.t() + + dist = -( + flatten.pow(2).sum(1, keepdim=True) + - 2 * flatten @ embed + + embed.pow(2).sum(0, keepdim=True) + ) + + embed_ind = gumbel_sample(dist, dim = -1, temperature = self.sample_codebook_temp) + embed_onehot = F.one_hot(embed_ind, self.codebook_size).type(dtype) + embed_ind = embed_ind.view(*shape[:-1]) + quantize = F.embedding(embed_ind, self.embed) + + if self.training: + cluster_size = embed_onehot.sum(0) + self.all_reduce_fn(cluster_size) + + ema_inplace(self.cluster_size, cluster_size, self.decay) + + embed_sum = flatten.t() @ embed_onehot + self.all_reduce_fn(embed_sum) + + ema_inplace(self.embed_avg, embed_sum.t(), self.decay) + cluster_size = laplace_smoothing(self.cluster_size, self.codebook_size, self.eps) * self.cluster_size.sum() + embed_normalized = self.embed_avg / cluster_size.unsqueeze(1) + self.embed.data.copy_(embed_normalized) + self.expire_codes_(x) + + return quantize, embed_ind + +class CosineSimCodebook(nn.Module): + def __init__( + self, + dim, + codebook_size, + kmeans_init = False, + kmeans_iters = 10, + decay = 0.8, + eps = 1e-5, + threshold_ema_dead_code = 2, + code_replacement_policy = 'batch_random', # batch_random or linde_buzo_gray + use_ddp = False, + learnable_codebook = False, + sample_codebook_temp = 0. + ): + super().__init__() + self.decay = decay + + if not kmeans_init: + embed = l2norm(uniform_init(codebook_size, dim)) + else: + embed = torch.zeros(codebook_size, dim) + + self.codebook_size = codebook_size + self.kmeans_iters = kmeans_iters + self.eps = eps + self.threshold_ema_dead_code = threshold_ema_dead_code + self.code_replacement_policy = code_replacement_policy + self.sample_codebook_temp = sample_codebook_temp + + self.sample_fn = sample_vectors_distributed if use_ddp else sample_vectors + self.all_reduce_fn = distributed.all_reduce if use_ddp else noop + self.add_noise_fn = add_noise_distributed if use_ddp else add_noise + + self.register_buffer('initted', torch.Tensor([not kmeans_init])) + self.register_buffer('cluster_size', torch.zeros(codebook_size)) + + self.learnable_codebook = learnable_codebook + if learnable_codebook: + self.embed = nn.Parameter(embed) + else: + self.register_buffer('embed', embed) + + self.counter = 0 + + @torch.jit.ignore + def init_embed_(self, data): + if self.initted: + return + + embed, cluster_size = kmeans(data, self.codebook_size, self.kmeans_iters, use_cosine_sim = True, + sample_fn = self.sample_fn, all_reduce_fn = self.all_reduce_fn) + + self.embed.data.copy_(embed) + self.cluster_size.data.copy_(cluster_size) + self.initted.data.copy_(torch.Tensor([True])) + + def replace_batch_random(self, samples, mask): + samples = l2norm(samples) + self.embed.data[mask] = self.sample_fn(samples, mask.sum().item()) + + def replace_linde_buzo_gray(self, mask): + num_unused = mask.sum() + most_used_idxs = self.cluster_size.argsort(descending=True)[:num_unused] + most_used_codes = self.embed.data[most_used_idxs] + self.embed.data[mask] = l2norm(self.add_noise_fn(most_used_codes)) + + def expire_codes_(self, batch_samples): + if self.threshold_ema_dead_code == 0: + return + + expired_codes = self.cluster_size < self.threshold_ema_dead_code + if not torch.any(expired_codes): + return + + if self.code_replacement_policy == 'batch_random': + # Replace dead codes by random latents from encoder + batch_samples = rearrange(batch_samples, '... d -> (...) d') + self.replace_batch_random(batch_samples, mask = expired_codes) + elif self.code_replacement_policy == 'linde_buzo_gray': + # Replace dead codes by most used codes + some noise (Linde-Buzo-Gray splitting algorithm) + self.replace_linde_buzo_gray(mask = expired_codes) + else: + raise ValueError(f'{self.code_replacement_policy} is not a valid dead code replacement strategy.') + + @autocast(enabled = False) + def forward(self, x): + x = x.float() + + shape, dtype = x.shape, x.dtype + + flatten = rearrange(x, '... d -> (...) d') + flatten = l2norm(flatten) + + self.init_embed_(flatten) + + embed = self.embed if not self.learnable_codebook else self.embed.detach() + embed = l2norm(embed) + + dist = flatten @ embed.t() + embed_ind = gumbel_sample(dist, dim = -1, temperature = self.sample_codebook_temp) + embed_onehot = F.one_hot(embed_ind, self.codebook_size).type(dtype) + embed_ind = embed_ind.view(*shape[:-1]) + + quantize = F.embedding(embed_ind, self.embed) + + if self.training: + bins = embed_onehot.sum(0) + self.all_reduce_fn(bins) + + ema_inplace(self.cluster_size, bins, self.decay) + + zero_mask = (bins == 0) + bins = bins.masked_fill(zero_mask, 1.) + + embed_sum = flatten.t() @ embed_onehot + self.all_reduce_fn(embed_sum) + + embed_normalized = (embed_sum / bins.unsqueeze(0)).t() + embed_normalized = l2norm(embed_normalized) + embed_normalized = torch.where(zero_mask[..., None], embed, + embed_normalized) + ema_inplace(self.embed, embed_normalized, self.decay) + self.expire_codes_(x) + + return quantize, embed_ind + +# main class + +class VectorQuantize(nn.Module): + def __init__( + self, + dim, + codebook_size, + codebook_dim = None, + heads = 1, + decay = 0.8, + eps = 1e-5, + kmeans_init = False, + kmeans_iters = 10, + use_cosine_sim = False, + threshold_ema_dead_code = 0, + code_replacement_policy = 'batch_random', # batch_random or linde_buzo_gray + channel_last = False, + accept_image_fmap = True, + commitment_weight = 1., + orthogonal_reg_weight = 0., + orthogonal_reg_active_codes_only = False, + orthogonal_reg_max_codes = None, + sample_codebook_temp = 0., + sync_codebook = False, + norm_latents = False, + ): + super().__init__() + self.heads = heads + codebook_dim = default(codebook_dim, dim) + codebook_input_dim = codebook_dim * heads + + requires_projection = codebook_input_dim != dim + self.project_in = nn.Linear(dim, codebook_input_dim) if requires_projection else nn.Identity() + self.project_out = nn.Linear(codebook_input_dim, dim) if requires_projection else nn.Identity() + + self.eps = eps + self.commitment_weight = commitment_weight + self.norm_latents = norm_latents + + has_codebook_orthogonal_loss = orthogonal_reg_weight > 0 + self.orthogonal_reg_weight = orthogonal_reg_weight + self.orthogonal_reg_active_codes_only = orthogonal_reg_active_codes_only + self.orthogonal_reg_max_codes = orthogonal_reg_max_codes + + codebook_class = EuclideanCodebook if not use_cosine_sim else CosineSimCodebook + + self._codebook = codebook_class( + dim = codebook_dim, + codebook_size = codebook_size, + kmeans_init = kmeans_init, + kmeans_iters = kmeans_iters, + decay = decay, + eps = eps, + threshold_ema_dead_code = threshold_ema_dead_code, + code_replacement_policy = code_replacement_policy, + use_ddp = sync_codebook, + learnable_codebook = has_codebook_orthogonal_loss, + sample_codebook_temp = sample_codebook_temp + ) + + self.codebook_size = codebook_size + + self.accept_image_fmap = accept_image_fmap + self.channel_last = channel_last + + @property + def codebook(self): + return self._codebook.embed + + def indices_to_embedding(self, indices): + embedding = F.embedding(indices, self.codebook) + embedding = rearrange(embedding, 'b h w c -> b c h w') + return embedding + + def forward(self, x): + shape, device, heads, is_multiheaded, codebook_size = x.shape, x.device, self.heads, self.heads > 1, self.codebook_size + + need_transpose = not self.channel_last and not self.accept_image_fmap + + if self.accept_image_fmap: + height, width = x.shape[-2:] + x = rearrange(x, 'b c h w -> b (h w) c') + + if need_transpose: + x = rearrange(x, 'b d n -> b n d') + + x = self.project_in(x) + + if is_multiheaded: + x = rearrange(x, 'b n (h d) -> (b h) n d', h = heads) + + if self.norm_latents: + # If specified, normalize encoder latents for computing commitment loss + x = l2norm(x) + + quantize, embed_ind = self._codebook(x) + + if self.training: + quantize = x + (quantize - x).detach() + + loss = torch.tensor([0.], device = device, requires_grad = self.training) + + if self.training: + if self.commitment_weight > 0: + commit_loss = F.mse_loss(quantize.detach(), x) + loss = loss + commit_loss * self.commitment_weight + + if self.orthogonal_reg_weight > 0: + codebook = self.codebook + + if self.orthogonal_reg_active_codes_only: + # only calculate orthogonal loss for the activated codes for this batch + unique_code_ids = torch.unique(embed_ind) + codebook = codebook[unique_code_ids] + + num_codes = codebook.shape[0] + if exists(self.orthogonal_reg_max_codes) and num_codes > self.orthogonal_reg_max_codes: + rand_ids = torch.randperm(num_codes, device = device)[:self.orthogonal_reg_max_codes] + codebook = codebook[rand_ids] + + orthogonal_reg_loss = orthgonal_loss_fn(codebook) + loss = loss + orthogonal_reg_loss * self.orthogonal_reg_weight + + if is_multiheaded: + quantize = rearrange(quantize, '(b h) n d -> b n (h d)', h = heads) + embed_ind = rearrange(embed_ind, '(b h) n -> b n h', h = heads) + + quantize = self.project_out(quantize) + + if need_transpose: + quantize = rearrange(quantize, 'b n d -> b d n') + + if self.accept_image_fmap: + quantize = rearrange(quantize, 'b (h w) c -> b c h w', h = height, w = width) + embed_ind = rearrange(embed_ind, 'b (h w) ... -> b h w ...', h = height, w = width) + if is_multiheaded: + embed_ind = rearrange(embed_ind, 'b h w ... -> b ... h w') + + return quantize, loss, embed_ind \ No newline at end of file diff --git a/fourm/vq/quantizers/quantize_memcodes.py b/fourm/vq/quantizers/quantize_memcodes.py new file mode 100644 index 0000000000000000000000000000000000000000..8defdbca0406dbf4e0ab6ab8a5633f38b9779985 --- /dev/null +++ b/fourm/vq/quantizers/quantize_memcodes.py @@ -0,0 +1,124 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This code is strongly influenced by https://github.com/lucidrains/NWT-pytorch/ + +import torch +import torch.nn.functional as F +from torch import nn, einsum + +from einops import rearrange, repeat +from einops.layers.torch import EinMix as Mix + + +class Memcodes(nn.Module): + def __init__( + self, + *, + dim, + codebook_size, + heads = 1, + temperature = 1., + channel_last = False, + accept_image_fmap = True, + **kwargs, + ): + super().__init__() + assert (dim % heads) == 0, 'dimension must be divisible by number of heads' + self.heads = heads + self.dim = dim + self.scale = (dim // heads) ** -0.5 + self.temperature = temperature + self.codebook_size = codebook_size + self.accept_image_fmap = accept_image_fmap + self.channel_last = channel_last + + num_codebooks = heads + codebook_dim = dim // heads + + self.codes = nn.Parameter(torch.randn(num_codebooks, codebook_size, codebook_dim)) + self.to_k = Mix('h n d -> h n c', weight_shape = 'h d c', h = heads, d = codebook_dim, c = codebook_dim) + self.to_v = Mix('h n d -> h n c', weight_shape = 'h d c', h = heads, d = codebook_dim, c = codebook_dim) + + def indices_to_embedding(self, indices): + batch = indices.shape[0] + + values = self.to_v(self.codes) + values = repeat(values, 'h n d -> b h n d', b = batch) + + indices = repeat(indices, '... -> ... d', d = values.shape[-1]).squeeze(2) + + if self.accept_image_fmap and len(indices.size())==4: + out = values.gather(2, indices) + out = rearrange(out, 'b h n d -> b (h d) n 1') + return out + else: + out = values.gather(2, indices.unsqueeze(2)) + return rearrange(out, 'b h n d -> b n (h d)') + + def forward(self, x): + need_transpose = not self.channel_last and not self.accept_image_fmap + + if self.accept_image_fmap: + height, width = x.shape[-2:] + x = rearrange(x, 'b c h w -> b (h w) c') + + if need_transpose: + x = rearrange(x, 'b d n -> b n d') + + assert x.shape[-1] == self.dim + + # split out heads + + q = rearrange(x, 'b n (h d) -> b h n d', h = self.heads) + + q = q * self.scale + + # get key / values of codes + + k, v = self.to_k(self.codes), self.to_v(self.codes) + + # straight through gumbel softmax + + logits = einsum('b h i d, h j d -> b h i j', q, k) + + if self.training: + attn = F.gumbel_softmax(logits, tau = self.temperature, dim = -1, hard = True) + codebook_indices = attn.argmax(dim = -1) + else: + codebook_indices = logits.argmax(dim = -1) + attn = F.one_hot(codebook_indices, num_classes = self.codebook_size).float() + + if self.heads == 1: + codebook_indices = codebook_indices.squeeze(1) + + out = einsum('b h i j, h j d -> b h i d', attn, v) + + # merge heads + out = rearrange(out, 'b h n d -> b n (h d)') + + if need_transpose: + out = rearrange(out, 'b n d -> b d n') + + if self.accept_image_fmap: + out = rearrange(out, 'b (h w) c -> b c h w', h = height, w = width) + if self.heads == 1: + codebook_indices = rearrange(codebook_indices, 'b (h w) -> b h w', h = height, w = width) + else: + codebook_indices = rearrange(codebook_indices, 'b n (h w) -> b n h w', h = height, w = width) + + # Dummy codebook loss for compatibility with other types of quantizers + codebook_loss = torch.tensor([0.], device=x.device) + + return out, codebook_loss, codebook_indices diff --git a/fourm/vq/scheduling/__init__.py b/fourm/vq/scheduling/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..71cbe8e0c7709fccae41547d44faae7c0ddce1d7 --- /dev/null +++ b/fourm/vq/scheduling/__init__.py @@ -0,0 +1,5 @@ +from .diffusion_pipeline import PipelineCond +from .scheduling_ddpm import DDPMScheduler +from .scheduling_ddim import DDIMScheduler +from .scheduling_pndm import PNDMScheduler +from .scheduling_utils import * \ No newline at end of file diff --git a/fourm/vq/scheduling/diffusion_pipeline.py b/fourm/vq/scheduling/diffusion_pipeline.py new file mode 100755 index 0000000000000000000000000000000000000000..664b449d4d9b7169fce851ad24f835dc1725ace6 --- /dev/null +++ b/fourm/vq/scheduling/diffusion_pipeline.py @@ -0,0 +1,133 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional, Tuple, Union +import torch +from diffusers import DiffusionPipeline +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from tqdm import tqdm + +from fourm.utils import to_2tuple + + +def rescale_noise_cfg(noise_cfg, noise_pred_conditional, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_conditional.std(dim=list(range(1, noise_pred_conditional.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +class PipelineCond(DiffusionPipeline): + """Pipeline for conditional image generation. + + This model inherits from `DiffusionPipeline`. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + model: The conditional diffusion model. + scheduler: A diffusion scheduler, e.g. see scheduling_ddpm.py + """ + def __init__(self, model: torch.nn.Module, scheduler: SchedulerMixin): + super().__init__() + self.register_modules(model=model, scheduler=scheduler) + + @torch.no_grad() + def __call__(self, + cond: torch.Tensor, + generator: Optional[torch.Generator] = None, + timesteps: Optional[int] = None, + guidance_scale: float = 0.0, + guidance_rescale: float = 0.0, + image_size: Optional[Union[Tuple[int, int], int]] = None, + verbose: bool = True, + scheduler_timesteps_mode: str = 'trailing', + orig_res: Optional[Union[torch.LongTensor, Tuple[int, int]]] = None, + **kwargs) -> torch.Tensor: + """The call function to the pipeline for conditional image generation. + + Args: + cond: The conditional input to the model. + generator: A torch.Generator to make generation deterministic. + timesteps: The number of denoising steps. More denoising steps usually lead to a higher + quality image at the expense of slower inference. Defaults to the number of training + timesteps if not given. + guidance_scale: The scale of the classifier-free guidance. If set to 0.0, no guidance is used. + guidance_rescale: Rescaling factor to fix the variance when using guidance scaling. + image_size: The size of the image to generate. If not given, the default training size + of the model is used. + verbose: Whether to show a progress bar. + scheduler_timesteps_mode: The mode to use for DDIMScheduler. One of `trailing`, `linspace`, + `leading`. See https://arxiv.org/abs/2305.08891 for more details. + orig_res: The original resolution of the image to condition the diffusion on. Ignored if None. + See SDXL https://arxiv.org/abs/2307.01952 for more details. + + Returns: + The generated image. + """ + + timesteps = self.scheduler.config.num_train_timesteps if timesteps is None else timesteps + batch_size, _, _, _ = cond.shape + + # Sample gaussian noise to begin loop + image_size = self.model.sample_size if image_size is None else image_size + image_size = to_2tuple(image_size) + image = torch.randn( + (batch_size, self.model.in_channels, image_size[0], image_size[1]), + generator=generator, + ) + image = image.to(self.model.device) + + do_cfg = callable(guidance_scale) or guidance_scale > 1.0 + + # Set step values + self.scheduler.set_timesteps(timesteps, mode=scheduler_timesteps_mode) + + if verbose: + pbar = tqdm(total=len(self.scheduler.timesteps)) + + for t in self.scheduler.timesteps: + # 1. Predict noise model_output + model_output = self.model(image, t, cond, orig_res=orig_res, **kwargs) + + if do_cfg: + model_output_uncond = self.model(image, t, cond, unconditional=True, **kwargs) # TODO: is there a better way to get unconditional output? + + if callable(guidance_scale): + guidance_scale_value = guidance_scale(t/self.scheduler.config.num_train_timesteps) + else: + guidance_scale_value = guidance_scale + model_output_cfg = model_output_uncond + guidance_scale_value * (model_output - model_output_uncond) + + if guidance_rescale > 0.0: + model_output = rescale_noise_cfg(model_output_cfg, model_output, guidance_rescale=guidance_rescale) + else: + model_output = model_output_cfg + + # 2. Compute previous image: x_t -> t_t-1 + with torch.cuda.amp.autocast(enabled=False): + image = self.scheduler.step(model_output.float(), t, image, generator=generator).prev_sample + + if verbose: + pbar.update() + if verbose: + pbar.close() + + return image + diff --git a/fourm/vq/scheduling/scheduling_ddim.py b/fourm/vq/scheduling/scheduling_ddim.py new file mode 100644 index 0000000000000000000000000000000000000000..bce43c7e2bf27de80517280f1b13970ce0729b92 --- /dev/null +++ b/fourm/vq/scheduling/scheduling_ddim.py @@ -0,0 +1,412 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This code is strongly influenced by https://github.com/huggingface/diffusers, +# https://github.com/pesser/pytorch_diffusion and https://github.com/hojonathanho/diffusion + +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.utils import BaseOutput, randn_tensor +from diffusers.schedulers.scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin + +from .scheduling_utils import enforce_zero_terminal_snr, betas_for_alpha_bar, scaled_cosine_alphas + + +@dataclass +# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->DDIM +class DDIMSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's step function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample (x_{t-1}) of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample (x_{0}) based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.FloatTensor + pred_original_sample: Optional[torch.FloatTensor] = None + + +class DDIMScheduler(SchedulerMixin, ConfigMixin): + """ + Denoising diffusion implicit models is a scheduler that extends the denoising procedure introduced in denoising + diffusion probabilistic models (DDPMs) with non-Markovian guidance. + + [`~ConfigMixin`] takes care of storing all config attributes that are passed in the scheduler's `__init__` + function, such as `num_train_timesteps`. They can be accessed via `scheduler.config.num_train_timesteps`. + [`SchedulerMixin`] provides general loading and saving functionality via the [`SchedulerMixin.save_pretrained`] and + [`~SchedulerMixin.from_pretrained`] functions. + + For more details, see the original paper: https://arxiv.org/abs/2010.02502 + + Args: + num_train_timesteps (`int`): number of diffusion steps used to train the model. + beta_start (`float`): the starting `beta` value of inference. + beta_end (`float`): the final `beta` value. + beta_schedule (`str`): + the beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, optional): + option to pass an array of betas directly to the constructor to bypass `beta_start`, `beta_end` etc. + clip_sample (`bool`, default `True`): + option to clip predicted sample for numerical stability. + clip_sample_range (`float`, default `1.0`): + the maximum magnitude for sample clipping. Valid only when `clip_sample=True`. + set_alpha_to_one (`bool`, default `True`): + each diffusion step uses the value of alphas product at that step and at the previous one. For the final + step there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`, + otherwise it uses the value of alpha at step 0. + steps_offset (`int`, default `0`): + an offset added to the inference steps. You can use a combination of `offset=1` and + `set_alpha_to_one=False`, to make the last step use step 0 for the previous alpha product, as done in + stable diffusion. + prediction_type (`str`, default `epsilon`, optional): + prediction type of the scheduler function, one of `epsilon` (predicting the noise of the diffusion + process), `sample` (directly predicting the noisy sample`) or `v_prediction` (see section 2.4 + https://imagen.research.google/video/paper.pdf) + thresholding (`bool`, default `False`): + whether to use the "dynamic thresholding" method (introduced by Imagen, https://arxiv.org/abs/2205.11487). + Note that the thresholding method is unsuitable for latent-space diffusion models (such as + stable-diffusion). + dynamic_thresholding_ratio (`float`, default `0.995`): + the ratio for the dynamic thresholding method. Default is `0.995`, the same as Imagen + (https://arxiv.org/abs/2205.11487). Valid only when `thresholding=True`. + sample_max_value (`float`, default `1.0`): + the threshold value for dynamic thresholding. Valid only when `thresholding=True`. + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + clip_sample: bool = True, + set_alpha_to_one: bool = True, + steps_offset: int = 0, + prediction_type: str = "v_prediction", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + clip_sample_range: float = 1.0, + sample_max_value: float = 1.0, + zero_terminal_snr: bool = True, + ): + if 'shifted_cosine:' in beta_schedule: + # Syntax is "shifted_cosine:{noise_shift}" + noise_shift = float(beta_schedule.split(':')[1]) + self.alphas_cumprod = scaled_cosine_alphas(num_train_timesteps, noise_shift) + else: + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = ( + torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + ) + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + if zero_terminal_snr: + self.betas = enforce_zero_terminal_snr(self.betas) + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + # At every step in ddim, we are looking into the previous alphas_cumprod + # For the final step, there is no previous alphas_cumprod because we are already at 0 + # `set_alpha_to_one` decides whether we set this parameter simply to one or + # whether we use the final alpha of the "non-previous" one. + self.final_alpha_cumprod = torch.tensor(1.0) if set_alpha_to_one else self.alphas_cumprod[0] + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # setable values + self.num_inference_steps = None + self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy().astype(np.int64)) + + def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): input sample + timestep (`int`, optional): current timestep + + Returns: + `torch.FloatTensor`: scaled input sample + """ + return sample + + def _get_variance(self, timestep, prev_timestep): + alpha_prod_t = self.alphas_cumprod[timestep] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + variance = (beta_prod_t_prev / beta_prod_t) * (1 - alpha_prod_t / alpha_prod_t_prev) + + return variance + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, height, width = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * height * width) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, height, width) + sample = sample.to(dtype) + + return sample + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None, mode='trailing'): + """ + Sets the discrete timesteps used for the diffusion chain. Supporting function to be run before inference. + + Args: + num_inference_steps (`int`): + the number of diffusion steps used when generating samples with a pre-trained model. + mode (`str`): + how to set the timesteps. One of trailing, linspace, leading. See https://arxiv.org/abs/2305.08891 + """ + + if num_inference_steps > self.config.num_train_timesteps: + raise ValueError( + f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle" + f" maximal {self.config.num_train_timesteps} timesteps." + ) + + self.num_inference_steps = num_inference_steps + step_ratio = self.config.num_train_timesteps // self.num_inference_steps + if mode == 'leading': + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.int64) + elif mode == 'trailing': + timesteps = np.arange(self.config.num_train_timesteps, 0, -step_ratio).round().astype(np.int64) - 1 + elif mode == 'linspace': + timesteps = np.linspace(self.config.num_train_timesteps, 1, self.num_inference_steps).round().astype(np.int64) - 1 + else: + raise NotImplementedError + self.timesteps = torch.from_numpy(timesteps).to(device) + self.timesteps += self.config.steps_offset + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + eta: float = 0.0, + use_clipped_model_output: bool = False, + generator=None, + variance_noise: Optional[torch.FloatTensor] = None, + return_dict: bool = True, + ) -> Union[DDIMSchedulerOutput, Tuple]: + """ + Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + current instance of sample being created by diffusion process. + eta (`float`): weight of noise for added noise in diffusion step. + use_clipped_model_output (`bool`): if `True`, compute "corrected" `model_output` from the clipped + predicted original sample. Necessary because predicted original sample is clipped to [-1, 1] when + `self.config.clip_sample` is `True`. If no clipping has happened, "corrected" `model_output` would + coincide with the one provided as input and `use_clipped_model_output` will have not effect. + generator: random number generator. + variance_noise (`torch.FloatTensor`): instead of generating noise for the variance using `generator`, we + can directly provide the noise for the variance itself. This is useful for methods such as + CycleDiffusion. (https://arxiv.org/abs/2210.05559) + return_dict (`bool`): option for returning tuple rather than DDIMSchedulerOutput class + + Returns: + [`~schedulers.scheduling_utils.DDIMSchedulerOutput`] or `tuple`: + [`~schedulers.scheduling_utils.DDIMSchedulerOutput`] if `return_dict` is True, otherwise a `tuple`. When + returning a tuple, the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + # See formulas (12) and (16) of DDIM paper https://arxiv.org/pdf/2010.02502.pdf + # Ideally, read DDIM paper in-detail understanding + + # Notation ( -> + # - pred_noise_t -> e_theta(x_t, t) + # - pred_original_sample -> f_theta(x_t, t) or x_0 + # - std_dev_t -> sigma_t + # - eta -> η + # - pred_sample_direction -> "direction pointing to x_t" + # - pred_prev_sample -> "x_t-1" + + # 1. get previous step value (=t-1) + prev_timestep = timestep - self.config.num_train_timesteps // self.num_inference_steps + + # 2. compute alphas, betas + alpha_prod_t = self.alphas_cumprod[timestep] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + + beta_prod_t = 1 - alpha_prod_t + + # 3. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + if self.config.prediction_type == "epsilon": + pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5) + pred_epsilon = model_output + elif self.config.prediction_type == "sample": + pred_original_sample = model_output + pred_epsilon = (sample - alpha_prod_t ** (0.5) * pred_original_sample) / beta_prod_t ** (0.5) + elif self.config.prediction_type == "v_prediction": + pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output + pred_epsilon = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction`" + ) + + # 4. Clip or threshold "predicted x_0" + if self.config.thresholding: + pred_original_sample = self._threshold_sample(pred_original_sample) + elif self.config.clip_sample: + pred_original_sample = pred_original_sample.clamp( + -self.config.clip_sample_range, self.config.clip_sample_range + ) + + # 5. compute variance: "sigma_t(η)" -> see formula (16) + # σ_t = sqrt((1 − α_t−1)/(1 − α_t)) * sqrt(1 − α_t/α_t−1) + variance = self._get_variance(timestep, prev_timestep) + std_dev_t = eta * variance ** (0.5) + + if use_clipped_model_output: + # the pred_epsilon is always re-derived from the clipped x_0 in Glide + pred_epsilon = (sample - alpha_prod_t ** (0.5) * pred_original_sample) / beta_prod_t ** (0.5) + + # 6. compute "direction pointing to x_t" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + pred_sample_direction = (1 - alpha_prod_t_prev - std_dev_t**2) ** (0.5) * pred_epsilon + + # 7. compute x_t without "random noise" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + prev_sample = alpha_prod_t_prev ** (0.5) * pred_original_sample + pred_sample_direction + + if eta > 0: + if variance_noise is not None and generator is not None: + raise ValueError( + "Cannot pass both generator and variance_noise. Please make sure that either `generator` or" + " `variance_noise` stays `None`." + ) + + if variance_noise is None: + variance_noise = randn_tensor( + model_output.shape, generator=generator, device=model_output.device, dtype=model_output.dtype + ) + variance = std_dev_t * variance_noise + + prev_sample = prev_sample + variance + + if not return_dict: + return (prev_sample,) + + return DDIMSchedulerOutput(prev_sample=prev_sample, pred_original_sample=pred_original_sample) + + def get_alpha_sigma_sqrts(self, timesteps, device, dtype, shape) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as original_samples + alphas_cumprod = self.alphas_cumprod.to(device=device, dtype=dtype) + timesteps = timesteps.to(device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + return sqrt_alpha_prod, sqrt_one_minus_alpha_prod + + def add_noise( + self, original_samples: torch.FloatTensor, noise: torch.FloatTensor, timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + sqrt_alpha_prod, sqrt_one_minus_alpha_prod = self.get_alpha_sigma_sqrts(timesteps, original_samples.device, original_samples.dtype, original_samples.shape) + noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise + return noisy_samples + + def get_velocity( + self, sample: torch.FloatTensor, noise: torch.FloatTensor, timesteps: torch.IntTensor + ) -> torch.FloatTensor: + sqrt_alpha_prod, sqrt_one_minus_alpha_prod = self.get_alpha_sigma_sqrts(timesteps, sample.device, sample.dtype, sample.shape) + velocity = sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample + return velocity + + def get_noise( + self, sample: torch.FloatTensor, velocity: torch.FloatTensor, timesteps: torch.IntTensor + ) -> torch.FloatTensor: + sqrt_alpha_prod, sqrt_one_minus_alpha_prod = self.get_alpha_sigma_sqrts(timesteps, sample.device, sample.dtype, sample.shape) + noise = sqrt_alpha_prod * velocity + sqrt_one_minus_alpha_prod * sample + return noise + + def __len__(self): + return self.config.num_train_timesteps \ No newline at end of file diff --git a/fourm/vq/scheduling/scheduling_ddpm.py b/fourm/vq/scheduling/scheduling_ddpm.py new file mode 100644 index 0000000000000000000000000000000000000000..e2b7e6ee18133b15afab1b6f8e4021414816b5d4 --- /dev/null +++ b/fourm/vq/scheduling/scheduling_ddpm.py @@ -0,0 +1,446 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This code is strongly influenced by https://github.com/huggingface/diffusers +# and https://github.com/ermongroup/ddim + +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.utils import BaseOutput, randn_tensor +from diffusers.schedulers.scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin + +from .scheduling_utils import enforce_zero_terminal_snr, betas_for_alpha_bar, scaled_cosine_alphas + + +@dataclass +class DDPMSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's step function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample (x_{t-1}) of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample (x_{0}) based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.FloatTensor + pred_original_sample: Optional[torch.FloatTensor] = None + +class DDPMScheduler(SchedulerMixin, ConfigMixin): + """ + Denoising diffusion probabilistic models (DDPMs) explores the connections between denoising score matching and + Langevin dynamics sampling. + + [`~ConfigMixin`] takes care of storing all config attributes that are passed in the scheduler's `__init__` + function, such as `num_train_timesteps`. They can be accessed via `scheduler.config.num_train_timesteps`. + [`SchedulerMixin`] provides general loading and saving functionality via the [`SchedulerMixin.save_pretrained`] and + [`~SchedulerMixin.from_pretrained`] functions. + + For more details, see the original paper: https://arxiv.org/abs/2006.11239 + + Args: + num_train_timesteps (`int`): number of diffusion steps used to train the model. + beta_start (`float`): the starting `beta` value of inference. + beta_end (`float`): the final `beta` value. + beta_schedule (`str`): + the beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, `squaredcos_cap_v2` or `sigmoid`. + trained_betas (`np.ndarray`, optional): + option to pass an array of betas directly to the constructor to bypass `beta_start`, `beta_end` etc. + variance_type (`str`): + options to clip the variance used when adding noise to the denoised sample. Choose from `fixed_small`, + `fixed_small_log`, `fixed_large`, `fixed_large_log`, `learned` or `learned_range`. + clip_sample (`bool`, default `True`): + option to clip predicted sample for numerical stability. + clip_sample_range (`float`, default `1.0`): + the maximum magnitude for sample clipping. Valid only when `clip_sample=True`. + prediction_type (`str`, default `epsilon`, optional): + prediction type of the scheduler function, one of `epsilon` (predicting the noise of the diffusion + process), `sample` (directly predicting the noisy sample`) or `v_prediction` (see section 2.4 + https://imagen.research.google/video/paper.pdf) + thresholding (`bool`, default `False`): + whether to use the "dynamic thresholding" method (introduced by Imagen, https://arxiv.org/abs/2205.11487). + Note that the thresholding method is unsuitable for latent-space diffusion models (such as + stable-diffusion). + dynamic_thresholding_ratio (`float`, default `0.995`): + the ratio for the dynamic thresholding method. Default is `0.995`, the same as Imagen + (https://arxiv.org/abs/2205.11487). Valid only when `thresholding=True`. + sample_max_value (`float`, default `1.0`): + the threshold value for dynamic thresholding. Valid only when `thresholding=True`. + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + variance_type: str = "fixed_small", + clip_sample: bool = True, + prediction_type: str = "v_prediction", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + clip_sample_range: float = 1.0, + sample_max_value: float = 1.0, + zero_terminal_snr: bool = True, + ): + if 'shifted_cosine:' in beta_schedule: + # Syntax is "shifted_cosine:{noise_shift}" + noise_shift = float(beta_schedule.split(':')[1]) + self.alphas_cumprod = scaled_cosine_alphas(num_train_timesteps, noise_shift) + else: + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = ( + torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + ) + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + elif beta_schedule == "sigmoid": + # GeoDiff sigmoid schedule + betas = torch.linspace(-6, 6, num_train_timesteps) + self.betas = torch.sigmoid(betas) * (beta_end - beta_start) + beta_start + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + if zero_terminal_snr: + self.betas = enforce_zero_terminal_snr(self.betas) + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + self.one = torch.tensor(1.0) + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # setable values + self.custom_timesteps = False + self.num_inference_steps = None + self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy()) + + self.variance_type = variance_type + + def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): input sample + timestep (`int`, optional): current timestep + + Returns: + `torch.FloatTensor`: scaled input sample + """ + return sample + + def set_timesteps( + self, + num_inference_steps: Optional[int] = None, + device: Union[str, torch.device] = None, + timesteps: Optional[List[int]] = None, + **kwargs, + ): + """ + Sets the discrete timesteps used for the diffusion chain. Supporting function to be run before inference. + + Args: + num_inference_steps (`Optional[int]`): + the number of diffusion steps used when generating samples with a pre-trained model. If passed, then + `timesteps` must be `None`. + device (`str` or `torch.device`, optional): + the device to which the timesteps are moved to. + custom_timesteps (`List[int]`, optional): + custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of equal spacing between timesteps is used. If passed, `num_inference_steps` + must be `None`. + + """ + if num_inference_steps is not None and timesteps is not None: + raise ValueError("Can only pass one of `num_inference_steps` or `custom_timesteps`.") + + if timesteps is not None: + for i in range(1, len(timesteps)): + if timesteps[i] >= timesteps[i - 1]: + raise ValueError("`custom_timesteps` must be in descending order.") + + if timesteps[0] >= self.config.num_train_timesteps: + raise ValueError( + f"`timesteps` must start before `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps}." + ) + + timesteps = np.array(timesteps, dtype=np.int64) + self.custom_timesteps = True + else: + if num_inference_steps > self.config.num_train_timesteps: + raise ValueError( + f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle" + f" maximal {self.config.num_train_timesteps} timesteps." + ) + + self.num_inference_steps = num_inference_steps + + step_ratio = self.config.num_train_timesteps // self.num_inference_steps + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.int64) + self.custom_timesteps = False + + self.timesteps = torch.from_numpy(timesteps).to(device) + + def _get_variance(self, t, predicted_variance=None, variance_type=None): + prev_t = self.previous_timestep(t) + + alpha_prod_t = self.alphas_cumprod[t] + alpha_prod_t_prev = self.alphas_cumprod[prev_t] if prev_t >= 0 else self.one + current_beta_t = 1 - alpha_prod_t / alpha_prod_t_prev + + # For t > 0, compute predicted variance βt (see formula (6) and (7) from https://arxiv.org/pdf/2006.11239.pdf) + # and sample from it to get previous sample + # x_{t-1} ~ N(pred_prev_sample, variance) == add variance to pred_sample + variance = (1 - alpha_prod_t_prev) / (1 - alpha_prod_t) * current_beta_t + + # we always take the log of variance, so clamp it to ensure it's not 0 + variance = torch.clamp(variance, min=1e-20) + + if variance_type is None: + variance_type = self.config.variance_type + + # hacks - were probably added for training stability + if variance_type == "fixed_small": + variance = variance + # for rl-diffuser https://arxiv.org/abs/2205.09991 + elif variance_type == "fixed_small_log": + variance = torch.log(variance) + variance = torch.exp(0.5 * variance) + elif variance_type == "fixed_large": + variance = current_beta_t + elif variance_type == "fixed_large_log": + # Glide max_log + variance = torch.log(current_beta_t) + elif variance_type == "learned": + return predicted_variance + elif variance_type == "learned_range": + min_log = torch.log(variance) + max_log = torch.log(current_beta_t) + frac = (predicted_variance + 1) / 2 + variance = frac * max_log + (1 - frac) * min_log + + return variance + + def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, height, width = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * height * width) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, height, width) + sample = sample.to(dtype) + + return sample + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + generator=None, + return_dict: bool = True, + ) -> Union[DDPMSchedulerOutput, Tuple]: + """ + Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + current instance of sample being created by diffusion process. + generator: random number generator. + return_dict (`bool`): option for returning tuple rather than DDPMSchedulerOutput class + + Returns: + [`~schedulers.scheduling_utils.DDPMSchedulerOutput`] or `tuple`: + [`~schedulers.scheduling_utils.DDPMSchedulerOutput`] if `return_dict` is True, otherwise a `tuple`. When + returning a tuple, the first element is the sample tensor. + + """ + t = timestep + + prev_t = self.previous_timestep(t) + + if model_output.shape[1] == sample.shape[1] * 2 and self.variance_type in ["learned", "learned_range"]: + model_output, predicted_variance = torch.split(model_output, sample.shape[1], dim=1) + else: + predicted_variance = None + + # 1. compute alphas, betas + alpha_prod_t = self.alphas_cumprod[t] + alpha_prod_t_prev = self.alphas_cumprod[prev_t] if prev_t >= 0 else self.one + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + current_alpha_t = alpha_prod_t / alpha_prod_t_prev + current_beta_t = 1 - current_alpha_t + + # 2. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (15) from https://arxiv.org/pdf/2006.11239.pdf + if self.config.prediction_type == "epsilon": + pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5) + elif self.config.prediction_type == "sample": + pred_original_sample = model_output + elif self.config.prediction_type == "v_prediction": + pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample` or" + " `v_prediction` for the DDPMScheduler." + ) + + # 3. Clip or threshold "predicted x_0" + if self.config.thresholding: + pred_original_sample = self._threshold_sample(pred_original_sample) + elif self.config.clip_sample: + pred_original_sample = pred_original_sample.clamp( + -self.config.clip_sample_range, self.config.clip_sample_range + ) + + # 4. Compute coefficients for pred_original_sample x_0 and current sample x_t + # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf + pred_original_sample_coeff = (alpha_prod_t_prev ** (0.5) * current_beta_t) / beta_prod_t + current_sample_coeff = current_alpha_t ** (0.5) * beta_prod_t_prev / beta_prod_t + + # 5. Compute predicted previous sample µ_t + # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf + pred_prev_sample = pred_original_sample_coeff * pred_original_sample + current_sample_coeff * sample + + # 6. Add noise + variance = 0 + if t > 0: + device = model_output.device + variance_noise = randn_tensor( + model_output.shape, generator=generator, device=device, dtype=model_output.dtype + ) + if self.variance_type == "fixed_small_log": + variance = self._get_variance(t, predicted_variance=predicted_variance) * variance_noise + elif self.variance_type == "learned_range": + variance = self._get_variance(t, predicted_variance=predicted_variance) + variance = torch.exp(0.5 * variance) * variance_noise + else: + variance = (self._get_variance(t, predicted_variance=predicted_variance) ** 0.5) * variance_noise + + pred_prev_sample = pred_prev_sample + variance + + if not return_dict: + return (pred_prev_sample,) + + return DDPMSchedulerOutput(prev_sample=pred_prev_sample, pred_original_sample=pred_original_sample) + + def get_alpha_sigma_sqrts(self, timesteps, device, dtype, shape) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as original_samples + alphas_cumprod = self.alphas_cumprod.to(device=device, dtype=dtype) + timesteps = timesteps.to(device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + return sqrt_alpha_prod, sqrt_one_minus_alpha_prod + + def add_noise( + self, original_samples: torch.FloatTensor, noise: torch.FloatTensor, timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + sqrt_alpha_prod, sqrt_one_minus_alpha_prod = self.get_alpha_sigma_sqrts(timesteps, original_samples.device, original_samples.dtype, original_samples.shape) + noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise + return noisy_samples + + def get_velocity( + self, sample: torch.FloatTensor, noise: torch.FloatTensor, timesteps: torch.IntTensor + ) -> torch.FloatTensor: + sqrt_alpha_prod, sqrt_one_minus_alpha_prod = self.get_alpha_sigma_sqrts(timesteps, sample.device, sample.dtype, sample.shape) + velocity = sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample + return velocity + + def get_noise( + self, sample: torch.FloatTensor, velocity: torch.FloatTensor, timesteps: torch.IntTensor + ) -> torch.FloatTensor: + sqrt_alpha_prod, sqrt_one_minus_alpha_prod = self.get_alpha_sigma_sqrts(timesteps, sample.device, sample.dtype, sample.shape) + noise = sqrt_alpha_prod * velocity + sqrt_one_minus_alpha_prod * sample + return noise + + def __len__(self): + return self.config.num_train_timesteps + + def previous_timestep(self, timestep): + if self.custom_timesteps: + index = (self.timesteps == timestep).nonzero(as_tuple=True)[0][0] + if index == self.timesteps.shape[0] - 1: + prev_t = torch.tensor(-1) + else: + prev_t = self.timesteps[index + 1] + else: + num_inference_steps = ( + self.num_inference_steps if self.num_inference_steps else self.config.num_train_timesteps + ) + prev_t = timestep - self.config.num_train_timesteps // num_inference_steps + + return prev_t \ No newline at end of file diff --git a/fourm/vq/scheduling/scheduling_pndm.py b/fourm/vq/scheduling/scheduling_pndm.py new file mode 100644 index 0000000000000000000000000000000000000000..fb95f7b761bb83be1ea9ec73f921a7db97145012 --- /dev/null +++ b/fourm/vq/scheduling/scheduling_pndm.py @@ -0,0 +1,469 @@ +# Copyright 2023 Zhejiang University Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This file is strongly influenced by https://github.com/ermongroup/ddim + +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.utils import BaseOutput, randn_tensor +from diffusers.schedulers.scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +class PNDMScheduler(SchedulerMixin, ConfigMixin): + """ + Pseudo numerical methods for diffusion models (PNDM) proposes using more advanced ODE integration techniques, + namely Runge-Kutta method and a linear multi-step method. + + [`~ConfigMixin`] takes care of storing all config attributes that are passed in the scheduler's `__init__` + function, such as `num_train_timesteps`. They can be accessed via `scheduler.config.num_train_timesteps`. + [`SchedulerMixin`] provides general loading and saving functionality via the [`SchedulerMixin.save_pretrained`] and + [`~SchedulerMixin.from_pretrained`] functions. + + For more details, see the original paper: https://arxiv.org/abs/2202.09778 + + Args: + num_train_timesteps (`int`): number of diffusion steps used to train the model. + beta_start (`float`): the starting `beta` value of inference. + beta_end (`float`): the final `beta` value. + beta_schedule (`str`): + the beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, optional): + option to pass an array of betas directly to the constructor to bypass `beta_start`, `beta_end` etc. + skip_prk_steps (`bool`): + allows the scheduler to skip the Runge-Kutta steps that are defined in the original paper as being required + before plms steps; defaults to `False`. + set_alpha_to_one (`bool`, default `False`): + each diffusion step uses the value of alphas product at that step and at the previous one. For the final + step there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`, + otherwise it uses the value of alpha at step 0. + prediction_type (`str`, default `epsilon`, optional): + prediction type of the scheduler function, one of `epsilon` (predicting the noise of the diffusion process) + or `v_prediction` (see section 2.4 https://imagen.research.google/video/paper.pdf) + timestep_spacing (`str`, default `"leading"`): + The way the timesteps should be scaled. Refer to Table 2. of [Common Diffusion Noise Schedules and Sample + Steps are Flawed](https://arxiv.org/abs/2305.08891) for more information. + steps_offset (`int`, default `0`): + an offset added to the inference steps. You can use a combination of `offset=1` and + `set_alpha_to_one=False`, to make the last step use step 0 for the previous alpha product, as done in + stable diffusion. + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + skip_prk_steps: bool = False, + set_alpha_to_one: bool = False, + prediction_type: str = "epsilon", + timestep_spacing: str = "leading", + steps_offset: int = 0, + **kwargs, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = ( + torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + ) + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + self.final_alpha_cumprod = torch.tensor(1.0) if set_alpha_to_one else self.alphas_cumprod[0] + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # For now we only support F-PNDM, i.e. the runge-kutta method + # For more information on the algorithm please take a look at the paper: https://arxiv.org/pdf/2202.09778.pdf + # mainly at formula (9), (12), (13) and the Algorithm 2. + self.pndm_order = 4 + + # running values + self.cur_model_output = 0 + self.counter = 0 + self.cur_sample = None + self.ets = [] + + # setable values + self.num_inference_steps = None + self._timesteps = np.arange(0, num_train_timesteps)[::-1].copy() + self.prk_timesteps = None + self.plms_timesteps = None + self.timesteps = None + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None, mode='leading'): + """ + Sets the discrete timesteps used for the diffusion chain. Supporting function to be run before inference. + + Args: + num_inference_steps (`int`): + the number of diffusion steps used when generating samples with a pre-trained model. + """ + + self.num_inference_steps = num_inference_steps + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + + assert mode == self.config.timestep_spacing, f"Timestep Spacing mode should be \'{self.config.timestep_spacing}\'" + + if mode == "linspace": + self._timesteps = ( + np.linspace(0, self.config.num_train_timesteps - 1, num_inference_steps).round().astype(np.int64) + ) + elif mode == "leading": + step_ratio = self.config.num_train_timesteps // self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + self._timesteps = (np.arange(0, num_inference_steps) * step_ratio).round() + self._timesteps += self.config.steps_offset + elif mode == "trailing": + step_ratio = self.config.num_train_timesteps / self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + self._timesteps = np.round(np.arange(self.config.num_train_timesteps, 0, -step_ratio))[::-1].astype( + np.int64 + ) + self._timesteps -= 1 + else: + raise ValueError( + f"{mode} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + if self.config.skip_prk_steps: + # for some models like stable diffusion the prk steps can/should be skipped to + # produce better results. When using PNDM with `self.config.skip_prk_steps` the implementation + # is based on crowsonkb's PLMS sampler implementation: https://github.com/CompVis/latent-diffusion/pull/51 + self.prk_timesteps = np.array([]) + self.plms_timesteps = np.concatenate([self._timesteps[:-1], self._timesteps[-2:-1], self._timesteps[-1:]])[ + ::-1 + ].copy() + else: + prk_timesteps = np.array(self._timesteps[-self.pndm_order :]).repeat(2) + np.tile( + np.array([0, self.config.num_train_timesteps // num_inference_steps // 2]), self.pndm_order + ) + self.prk_timesteps = (prk_timesteps[:-1].repeat(2)[1:-1])[::-1].copy() + self.plms_timesteps = self._timesteps[:-3][ + ::-1 + ].copy() # we copy to avoid having negative strides which are not supported by torch.from_numpy + + timesteps = np.concatenate([self.prk_timesteps, self.plms_timesteps]).astype(np.int64) + self.timesteps = torch.from_numpy(timesteps).to(device) + + self.ets = [] + self.counter = 0 + self.cur_model_output = 0 + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + return_dict: bool = True, + **kwargs, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion + process from the learned model outputs (most often the predicted noise). + + This function calls `step_prk()` or `step_plms()` depending on the internal variable `counter`. + + Args: + model_output (`torch.FloatTensor`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + current instance of sample being created by diffusion process. + return_dict (`bool`): option for returning tuple rather than SchedulerOutput class + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + [`~schedulers.scheduling_utils.SchedulerOutput`] if `return_dict` is True, otherwise a `tuple`. When + returning a tuple, the first element is the sample tensor. + + """ + if self.counter < len(self.prk_timesteps) and not self.config.skip_prk_steps: + return self.step_prk(model_output=model_output, timestep=timestep, sample=sample, return_dict=return_dict) + else: + return self.step_plms(model_output=model_output, timestep=timestep, sample=sample, return_dict=return_dict) + + def step_prk( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Step function propagating the sample with the Runge-Kutta method. RK takes 4 forward passes to approximate the + solution to the differential equation. + + Args: + model_output (`torch.FloatTensor`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + current instance of sample being created by diffusion process. + return_dict (`bool`): option for returning tuple rather than SchedulerOutput class + + Returns: + [`~scheduling_utils.SchedulerOutput`] or `tuple`: [`~scheduling_utils.SchedulerOutput`] if `return_dict` is + True, otherwise a `tuple`. When returning a tuple, the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + diff_to_prev = 0 if self.counter % 2 else self.config.num_train_timesteps // self.num_inference_steps // 2 + prev_timestep = timestep - diff_to_prev + timestep = self.prk_timesteps[self.counter // 4 * 4] + + if self.counter % 4 == 0: + self.cur_model_output += 1 / 6 * model_output + self.ets.append(model_output) + self.cur_sample = sample + elif (self.counter - 1) % 4 == 0: + self.cur_model_output += 1 / 3 * model_output + elif (self.counter - 2) % 4 == 0: + self.cur_model_output += 1 / 3 * model_output + elif (self.counter - 3) % 4 == 0: + model_output = self.cur_model_output + 1 / 6 * model_output + self.cur_model_output = 0 + + # cur_sample should not be `None` + cur_sample = self.cur_sample if self.cur_sample is not None else sample + + prev_sample = self._get_prev_sample(cur_sample, timestep, prev_timestep, model_output) + self.counter += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + def step_plms( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Step function propagating the sample with the linear multi-step method. This has one forward pass with multiple + times to approximate the solution. + + Args: + model_output (`torch.FloatTensor`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + current instance of sample being created by diffusion process. + return_dict (`bool`): option for returning tuple rather than SchedulerOutput class + + Returns: + [`~scheduling_utils.SchedulerOutput`] or `tuple`: [`~scheduling_utils.SchedulerOutput`] if `return_dict` is + True, otherwise a `tuple`. When returning a tuple, the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if not self.config.skip_prk_steps and len(self.ets) < 3: + raise ValueError( + f"{self.__class__} can only be run AFTER scheduler has been run " + "in 'prk' mode for at least 12 iterations " + "See: https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/pipeline_pndm.py " + "for more information." + ) + + prev_timestep = timestep - self.config.num_train_timesteps // self.num_inference_steps + + if self.counter != 1: + self.ets = self.ets[-3:] + self.ets.append(model_output) + else: + prev_timestep = timestep + timestep = timestep + self.config.num_train_timesteps // self.num_inference_steps + + if len(self.ets) == 1 and self.counter == 0: + model_output = model_output + self.cur_sample = sample + elif len(self.ets) == 1 and self.counter == 1: + model_output = (model_output + self.ets[-1]) / 2 + sample = self.cur_sample + self.cur_sample = None + elif len(self.ets) == 2: + model_output = (3 * self.ets[-1] - self.ets[-2]) / 2 + elif len(self.ets) == 3: + model_output = (23 * self.ets[-1] - 16 * self.ets[-2] + 5 * self.ets[-3]) / 12 + else: + model_output = (1 / 24) * (55 * self.ets[-1] - 59 * self.ets[-2] + 37 * self.ets[-3] - 9 * self.ets[-4]) + + prev_sample = self._get_prev_sample(sample, timestep, prev_timestep, model_output) + self.counter += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + def scale_model_input(self, sample: torch.FloatTensor, *args, **kwargs) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): input sample + + Returns: + `torch.FloatTensor`: scaled input sample + """ + return sample + + def _get_prev_sample(self, sample, timestep, prev_timestep, model_output): + # See formula (9) of PNDM paper https://arxiv.org/pdf/2202.09778.pdf + # this function computes x_(t−δ) using the formula of (9) + # Note that x_t needs to be added to both sides of the equation + + # Notation ( -> + # alpha_prod_t -> α_t + # alpha_prod_t_prev -> α_(t−δ) + # beta_prod_t -> (1 - α_t) + # beta_prod_t_prev -> (1 - α_(t−δ)) + # sample -> x_t + # model_output -> e_θ(x_t, t) + # prev_sample -> x_(t−δ) + alpha_prod_t = self.alphas_cumprod[timestep] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + if self.config.prediction_type == "v_prediction": + model_output = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample + elif self.config.prediction_type != "epsilon": + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon` or `v_prediction`" + ) + + # corresponds to (α_(t−δ) - α_t) divided by + # denominator of x_t in formula (9) and plus 1 + # Note: (α_(t−δ) - α_t) / (sqrt(α_t) * (sqrt(α_(t−δ)) + sqr(α_t))) = + # sqrt(α_(t−δ)) / sqrt(α_t)) + sample_coeff = (alpha_prod_t_prev / alpha_prod_t) ** (0.5) + + # corresponds to denominator of e_θ(x_t, t) in formula (9) + model_output_denom_coeff = alpha_prod_t * beta_prod_t_prev ** (0.5) + ( + alpha_prod_t * beta_prod_t * alpha_prod_t_prev + ) ** (0.5) + + # full formula (9) + prev_sample = ( + sample_coeff * sample - (alpha_prod_t_prev - alpha_prod_t) * model_output / model_output_denom_coeff + ) + + return prev_sample + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as original_samples + alphas_cumprod = self.alphas_cumprod.to(device=original_samples.device, dtype=original_samples.dtype) + timesteps = timesteps.to(original_samples.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(original_samples.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/fourm/vq/scheduling/scheduling_utils.py b/fourm/vq/scheduling/scheduling_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..631a4d8b3d9127f54048713f156e53d6e35d83b3 --- /dev/null +++ b/fourm/vq/scheduling/scheduling_utils.py @@ -0,0 +1,102 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +import numpy as np +import torch + + +def enforce_zero_terminal_snr(betas: torch.Tensor) -> torch.Tensor: + """Scales the noise schedule betas so that last time step has zero SNR. + See https://arxiv.org/abs/2305.08891 + + Args: + betas: the initial diffusion noise schedule betas + + Returns: + The diffusion noise schedule betas with the last time step having zero SNR + """ + + # Convert betas to alphas_bar_sqrt + alphas = 1 - betas + alphas_bar = alphas.cumprod(0) + alphas_bar_sqrt = alphas_bar.sqrt() + + # Store old values. + alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone() + alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone() + # Shift so last timestep is zero. + alphas_bar_sqrt -= alphas_bar_sqrt_T + # Scale so first timestep is back to old value. + alphas_bar_sqrt *= alphas_bar_sqrt_0 / ( + alphas_bar_sqrt_0 - alphas_bar_sqrt_T) + + # Convert alphas_bar_sqrt to betas + alphas_bar = alphas_bar_sqrt ** 2 + alphas = alphas_bar[1:] / alphas_bar[:-1] + alphas = torch.cat([alphas_bar[0:1], alphas]) + betas = 1 - alphas + return betas + + +def betas_for_alpha_bar(num_diffusion_timesteps: int, max_beta: float = 0.999) -> torch.Tensor: + """Create a beta schedule that discretizes the given alpha_t_bar function, + which defines the cumulative product of (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to + the cumulative product of (1-beta) up to that part of the diffusion process. + + Args: + num_diffusion_timesteps: the number of betas to produce. + max_beta: the maximum beta to use; use values lower than 1 to + prevent singularities. + + Returns: + The betas used by the scheduler to step the model outputs + """ + + def alpha_bar(time_step): + return math.cos((time_step + 0.008) / 1.008 * math.pi / 2) ** 2 + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar(t2) / alpha_bar(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +def scaled_cosine_alphas(num_diffusion_timesteps: int, noise_shift: float = 1.0) -> torch.Tensor: + """Shifts a cosine noise schedule by a specified amount in log-SNR space. + + noise_shift = 1.0 corresponds to the standard cosine noise schedule. + 0 < noise_shift < 1.0 corresponds to a less noisy schedule (better + suited if the conditioning is highly informative, e.g. low-res images). + noise_shift > 1.0 corresponds to a more noisy schedule (better suited + if the conditioning is not as informative, e.g. captions). + + See https://arxiv.org/abs/2305.18231 + + Args: + num_diffusion_timesteps: the number of diffusion timesteps. + noise_shift: the amount to shift the noise schedule by in log-SNR space. + + Returns: + The alphas_cumprod used by the diffusion noise scheduler + """ + t = torch.linspace(0, 1, num_diffusion_timesteps).to(torch.float64) + log_snr = -2 * (torch.tan(torch.pi * t / 2).log() + np.log(noise_shift)) + log_snr = log_snr.clamp(-15,15).float() + alphas_cumprod = log_snr.sigmoid() + alphas_cumprod[-1] = 0.0 + return alphas_cumprod \ No newline at end of file diff --git a/fourm/vq/vq_utils.py b/fourm/vq/vq_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8e9bcf27910d8e6a7413149dc282c371a230185d --- /dev/null +++ b/fourm/vq/vq_utils.py @@ -0,0 +1,46 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +import numpy as np + + +def compute_codebook_usage( + all_tokens: torch.LongTensor, + codebook_size: int = 16_384, + window_size: int = 65_536) -> float: + """Computes the codebook usage for a given set of encoded tokens, by computing the + percentage of unique tokens in windows of a given size. The window size should be + chosen as batch_size * sequence_length, where batch_size is recommended to be set + to 256, and the sequence_length is the number of tokens per image. We follow + ViT-VQGAN's approach of using batch_size 256. (https://arxiv.org/abs/2110.04627) + + Args: + all_tokens: A tensor of shape (n_tokens, ) containing all the encoded tokens. + codebook_size: The size of the codebook. + window_size: The size of the window to compute the codebook usage in. + + Returns: + The average codebook usage. + """ + n_full_windows = all_tokens.shape[0] // window_size + + percentages = [] + for i, token_window in enumerate(torch.split(all_tokens, window_size)): + if i < n_full_windows: + usage_perc = len(np.unique(token_window)) / codebook_size + percentages.append(usage_perc) + else: + break + + return np.mean(percentages) \ No newline at end of file diff --git a/fourm/vq/vqvae.py b/fourm/vq/vqvae.py new file mode 100755 index 0000000000000000000000000000000000000000..7b29a730359048431c961c22e21c935e5c0677df --- /dev/null +++ b/fourm/vq/vqvae.py @@ -0,0 +1,1047 @@ +# Copyright 2024 EPFL and Apple Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List, Tuple, Dict, Optional, Union, Any +from contextlib import nullcontext +import copy +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange, repeat +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers import StableDiffusionPipeline +from huggingface_hub import PyTorchModelHubMixin + +from fourm.vq.quantizers import VectorQuantizerLucid, Memcodes +import fourm.vq.models.vit_models as vit_models +import fourm.vq.models.unet.unet as unet +import fourm.vq.models.uvit as uvit +import fourm.vq.models.controlnet as controlnet +from fourm.vq.models.mlp_models import build_mlp +from fourm.vq.scheduling import DDPMScheduler, DDIMScheduler, PNDMScheduler, PipelineCond + +from fourm.utils import denormalize + + +# If freeze_enc is True, the following modules will be frozen +FREEZE_MODULES = ['encoder', 'quant_proj', 'quantize', 'cls_emb'] + +class VQ(nn.Module, PyTorchModelHubMixin): + """Base class for VQVAE and DiVAE models. Implements the encoder and quantizer, and can be used as such without a decoder + after training. + + Args: + image_size: Input and target image size. + image_size_enc: Input image size for the encoder. Defaults to image_size. Change this when loading weights + from a tokenizer trained on a different image size. + n_channels: Number of input channels. + n_labels: Number of classes for semantic segmentation. + enc_type: String identifier specifying the encoder architecture. See vq/vit_models.py and vq/mlp_models.py + for available architectures. + patch_proj: Whether or not to use a ViT-style patch-wise linear projection in the encoder. + post_mlp: Whether or not to add a small point-wise MLP before the quantizer. + patch_size: Patch size for the encoder. + quant_type: String identifier specifying the quantizer implementation. Can be 'lucid', or 'memcodes'. + codebook_size: Number of codebook entries. + num_codebooks: Number of "parallel" codebooks to use. Only relevant for 'lucid' and 'memcodes' quantizers. + When using this, the tokens will be of shape B N_C H_Q W_Q, where N_C is the number of codebooks. + latent_dim: Dimensionality of the latent code. Can be small when using norm_codes=True, + see ViT-VQGAN (https://arxiv.org/abs/2110.04627) paper for details. + norm_codes: Whether or not to normalize the codebook entries to the unit sphere. + See ViT-VQGAN (https://arxiv.org/abs/2110.04627) paper for details. + norm_latents: Whether or not to normalize the latent codes to the unit sphere for computing commitment loss. + sync_codebook: Enable this when training on multiple GPUs, and disable for single GPUs, e.g. at inference. + ema_decay: Decay rate for the exponential moving average of the codebook entries. + threshold_ema_dead_code: Threshold for replacing stale codes that are used less than the + indicated exponential moving average of the codebook entries. + code_replacement_policy: Policy for replacing stale codes. Can be 'batch_random' or 'linde_buzo_gray'. + commitment_weight: Weight for the quantizer commitment loss. + kmeans_init: Whether or not to initialize the codebook entries with k-means clustering. + ckpt_path: Path to a checkpoint to load the model weights from. + ignore_keys: List of keys to ignore when loading the state_dict from the above checkpoint. + freeze_enc: Whether or not to freeze the encoder weights. See FREEZE_MODULES for the list of modules. + undo_std: Whether or not to undo any ImageNet standardization and transform the images to [-1,1] + before feeding the input to the encoder. + config: Dictionary containing the model configuration. Only used when loading + from Huggingface Hub. Ignore otherwise. + """ + def __init__(self, + image_size: int = 224, + image_size_enc: Optional[int] = None, + n_channels: str = 3, + n_labels: Optional[int] = None, + enc_type: str = 'vit_b_enc', + patch_proj: bool = True, + post_mlp: bool = False, + patch_size: int = 16, + quant_type: str = 'lucid', + codebook_size: Union[int, str] = 16384, + num_codebooks: int = 1, + latent_dim: int = 32, + norm_codes: bool = True, + norm_latents: bool = False, + sync_codebook: bool = True, + ema_decay: float = 0.99, + threshold_ema_dead_code: float = 0.25, + code_replacement_policy: str = 'batch_random', + commitment_weight: float = 1.0, + kmeans_init: bool = False, + ckpt_path: Optional[str] = None, + ignore_keys: List[str] = [ + 'decoder', 'loss', + 'post_quant_conv', 'post_quant_proj', + 'encoder.pos_emb', + ], + freeze_enc: bool = False, + undo_std: bool = False, + config: Optional[Dict[str, Any]] = None, + **kwargs): + if config is not None: + config = copy.deepcopy(config) + self.__init__(**config) + return + + super().__init__() + + self.image_size = image_size + self.n_channels = n_channels + self.n_labels = n_labels + self.enc_type = enc_type + self.patch_proj = patch_proj + self.post_mlp = post_mlp + self.patch_size = patch_size + self.quant_type = quant_type + self.codebook_size = codebook_size + self.num_codebooks = num_codebooks + self.latent_dim = latent_dim + self.norm_codes = norm_codes + self.norm_latents = norm_latents + self.sync_codebook = sync_codebook + self.ema_decay = ema_decay + self.threshold_ema_dead_code = threshold_ema_dead_code + self.code_replacement_policy = code_replacement_policy + self.commitment_weight = commitment_weight + self.kmeans_init = kmeans_init + self.ckpt_path = ckpt_path + self.ignore_keys = ignore_keys + self.freeze_enc = freeze_enc + self.undo_std = undo_std + + # For semantic segmentation + if n_labels is not None: + self.cls_emb = nn.Embedding(num_embeddings=n_labels, embedding_dim=n_channels) + self.colorize = torch.randn(3, n_labels, 1, 1) + else: + self.cls_emb = None + + # Init encoder + image_size_enc = image_size_enc or image_size + if 'vit' in enc_type: + self.encoder = getattr(vit_models, enc_type)( + in_channels=n_channels, patch_size=patch_size, + resolution=image_size_enc, patch_proj=patch_proj, post_mlp=post_mlp + ) + self.enc_dim = self.encoder.dim_tokens + elif 'MLP' in enc_type: + self.encoder = build_mlp(model_id=enc_type, dim_in=n_channels, dim_out=None) + self.enc_dim = self.encoder.dim_out + else: + raise NotImplementedError(f'{enc_type} not implemented.') + + # Encoder -> quantizer projection + self.quant_proj = torch.nn.Conv2d(self.enc_dim, self.latent_dim, 1) + + # Init quantizer + if quant_type == 'lucid': + self.quantize = VectorQuantizerLucid( + dim=latent_dim, + codebook_size=codebook_size, + codebook_dim=latent_dim, + heads=num_codebooks, + use_cosine_sim = norm_codes, + threshold_ema_dead_code = threshold_ema_dead_code, + code_replacement_policy=code_replacement_policy, + sync_codebook = sync_codebook, + decay = ema_decay, + commitment_weight=self.commitment_weight, + norm_latents = norm_latents, + kmeans_init=kmeans_init, + ) + elif quant_type == 'memcodes': + self.quantize = Memcodes( + dim=latent_dim, codebook_size=codebook_size, + heads=num_codebooks, temperature=1., + ) + else: + raise ValueError(f'{quant_type} not a valid quant_type.') + + # Load checkpoint + if ckpt_path is not None: + self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys) + + # Freeze encoder + if freeze_enc: + for module_name, module in self.named_children(): + if module_name not in FREEZE_MODULES: + continue + for param in module.parameters(): + param.requires_grad = False + module.eval() + + def train(self, mode: bool = True) -> 'VQ': + """Override the default train() to set the training mode to all modules + except the encoder if freeze_enc is True. + + Args: + mode: Whether to set the model to training mode (True) or evaluation mode (False). + """ + if not isinstance(mode, bool): + raise ValueError("training mode is expected to be boolean") + self.training = mode + for module_name, module in self.named_children(): + if self.freeze_enc and module_name in FREEZE_MODULES: + continue + module.train(mode) + return self + + def init_from_ckpt(self, path: str, ignore_keys: List[str] = list()) -> 'VQ': + """Loads the state_dict from a checkpoint file and initializes the model with it. + Renames the keys in the state_dict if necessary (e.g. when loading VQ-GAN weights). + + Args: + path: Path to the checkpoint file. + ignore_keys: List of keys to ignore when loading the state_dict. + + Returns: + self + """ + ckpt = torch.load(path, map_location="cpu") + sd = ckpt['model'] if 'model' in ckpt else ckpt['state_dict'] + + # Compatibility with ViT-VQGAN weights + if 'quant_conv.0.weight' in sd and 'quant_conv.0.bias' in sd: + print("Renaming quant_conv.0 to quant_proj") + sd['quant_proj.weight'] = sd['quant_conv.0.weight'] + sd['quant_proj.bias'] = sd['quant_conv.0.bias'] + del sd['quant_conv.0.weight'] + del sd['quant_conv.0.bias'] + elif 'quant_conv.weight' in sd and 'quant_conv.bias' in sd: + print("Renaming quant_conv to quant_proj") + sd['quant_proj.weight'] = sd['quant_conv.weight'] + sd['quant_proj.bias'] = sd['quant_conv.bias'] + del sd['quant_conv.weight'] + del sd['quant_conv.bias'] + if 'post_quant_conv.0.weight' in sd and 'post_quant_conv.0.bias' in sd: + print("Renaming post_quant_conv.0 to post_quant_proj") + sd['post_quant_proj.weight'] = sd['post_quant_conv.0.weight'] + sd['post_quant_proj.bias'] = sd['post_quant_conv.0.bias'] + del sd['post_quant_conv.0.weight'] + del sd['post_quant_conv.0.bias'] + elif 'post_quant_conv.weight' in sd and 'post_quant_conv.bias' in sd: + print("Renaming post_quant_conv to post_quant_proj") + sd['post_quant_proj.weight'] = sd['post_quant_conv.weight'] + sd['post_quant_proj.bias'] = sd['post_quant_conv.bias'] + del sd['post_quant_conv.weight'] + del sd['post_quant_conv.bias'] + + keys = list(sd.keys()) + for k in keys: + for ik in ignore_keys: + if k.startswith(ik): + print("Deleting key {} from state_dict.".format(k)) + del sd[k] + msg = self.load_state_dict(sd, strict=False) + print(msg) + print(f"Restored from {path}") + + return self + + def prepare_input(self, x: torch.Tensor) -> torch.Tensor: + """Preprocesses the input image tensor before feeding it to the encoder. + If self.undo_std, the input is first denormalized from the ImageNet + standardization to [-1, 1]. If semantic segmentation is performed, the + class indices are embedded. + + Args: + x: Input image tensor of shape B C H W + or B H W in case of semantic segmentation + + Returns: + Preprocessed input tensor of shape B C H W + """ + if self.undo_std: + x = 2.0 * denormalize(x) - 1.0 + if self.cls_emb is not None: + x = rearrange(self.cls_emb(x), 'b h w c -> b c h w') + return x + + def to_rgb(self, x: torch.Tensor) -> torch.Tensor: + """When semantic segmentation is performed, this function converts the + class embeddings to RGB. + + Args: + x: Input tensor of shape B C H W + + Returns: + RGB tensor of shape B C H W + """ + x = F.conv2d(x, weight=self.colorize) + x = (x-x.min())/(x.max()-x.min()) + return x + + def encode(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.LongTensor]: + """Encodes an input image tensor and quantizes the latent code. + + Args: + x: Input image tensor of shape B C H W + or B H W in case of semantic segmentation + + Returns: + quant: Quantized latent code of shape B D_Q H_Q W_Q + code_loss: Codebook loss + tokens: Quantized indices of shape B H_Q W_Q + """ + x = self.prepare_input(x) + h = self.encoder(x) + h = self.quant_proj(h) + quant, code_loss, tokens = self.quantize(h) + return quant, code_loss, tokens + + def tokenize(self, x: torch.Tensor) -> torch.LongTensor: + """Tokenizes an input image tensor. + + Args: + x: Input image tensor of shape B C H W + or B H W in case of semantic segmentation + + Returns: + Quantized indices of shape B H_Q W_Q + """ + _, _, tokens = self.encode(x) + return tokens + + def autoencode(self, x: torch.Tensor, **kwargs) -> torch.Tensor: + """Autoencodes an input image tensor by encoding it, quantizing the latent code, + and decoding it back to an image. + + Args: + x: Input image tensor of shape B C H W + or B H W in case of semantic segmentation + + Returns: + Reconstructed image tensor of shape B C H W + """ + pass + + def decode_quant(self, quant: torch.Tensor, **kwargs) -> torch.Tensor: + """Decodes quantized latent codes back to an image. + + Args: + quant: Quantized latent code of shape B D_Q H_Q W_Q + + Returns: + Decoded image tensor of shape B C H W + """ + pass + + def tokens_to_embedding(self, tokens: torch.LongTensor) -> torch.Tensor: + """Look up the codebook entries corresponding the discrete tokens. + + Args: + tokens: Quantized indices of shape B H_Q W_Q + + Returns: + Quantized latent code of shape B D_Q H_Q W_Q + """ + return self.quantize.indices_to_embedding(tokens) + + def decode_tokens(self, tokens: torch.LongTensor, **kwargs) -> torch.Tensor: + """Decodes discrete tokens back to an image. + + Args: + tokens: Quantized indices of shape B H_Q W_Q + + Returns: + Decoded image tensor of shape B C H W + """ + quant = self.tokens_to_embedding(tokens) + dec = self.decode_quant(quant, **kwargs) + return dec + + def forward(self, x: torch.Tensor, **kwargs) -> Tuple[torch.Tensor, torch.Tensor]: + """Forward pass of the encoder and quantizer. + + Args: + x: Input image tensor of shape B C H W + or B H W in case of semantic segmentation + + Returns: + quant: Quantized latent code of shape B D_Q H_Q W_Q + code_loss: Codebook loss + """ + quant, code_loss, _ = self.encode(x) + return quant, code_loss + + +class VQVAE(VQ): + """VQ-VAE model = simple encoder + decoder with a discrete bottleneck and + basic reconstruction loss (optionall with perceptual loss), i.e. no diffusion, + nor GAN discriminator. + + Args: + dec_type: String identifier specifying the decoder architecture. + See vq/vit_models.py and vq/mlp_models.py for available architectures. + out_conv: Whether or not to add final conv layers to the ViT decoder. + image_size_dec: Image size for the decoder. Defaults to self.image_size. + Change this when loading weights from a tokenizer decoder trained on a + different image size. + patch_size_dec: Patch size for the decoder. Defaults to self.patch_size. + config: Dictionary containing the model configuration. Only used when loading + from Huggingface Hub. Ignore otherwise. + """ + def __init__(self, + dec_type: str = 'vit_b_dec', + out_conv: bool = False, + image_size_dec: int = None, + patch_size_dec: int = None, + config: Optional[Dict[str, Any]] = None, + *args, + **kwargs): + if config is not None: + config = copy.deepcopy(config) + self.__init__(**config) + return + # Don't want to load the weights just yet + self.original_ckpt_path = kwargs.get('ckpt_path', None) + kwargs['ckpt_path'] = None + super().__init__(*args, **kwargs) + self.ckpt_path = self.original_ckpt_path + + # Init decoder + out_channels = self.n_channels if self.n_labels is None else self.n_labels + image_size_dec = image_size_dec or self.image_size + patch_size = patch_size_dec or self.patch_size + if 'vit' in dec_type: + self.decoder = getattr(vit_models, dec_type)( + out_channels=out_channels, patch_size=patch_size, + resolution=image_size_dec, out_conv=out_conv, post_mlp=self.post_mlp, + patch_proj=self.patch_proj + ) + self.dec_dim = self.decoder.dim_tokens + elif 'MLP' in dec_type: + self.decoder = build_mlp(model_id=dec_type, dim_in=None, dim_out=out_channels) + self.dec_dim = self.decoder.dim_in + else: + raise NotImplementedError(f'{dec_type} not implemented.') + + # Quantizer -> decoder projection + self.post_quant_proj = torch.nn.Conv2d(self.latent_dim, self.dec_dim, 1) + + # Load checkpoint + if self.ckpt_path is not None: + self.init_from_ckpt(self.ckpt_path, ignore_keys=self.ignore_keys) + + def decode_quant(self, quant: torch.Tensor, **kwargs) -> torch.Tensor: + """Decodes quantized latent codes back to an image. + + Args: + quant: Quantized latent code of shape B D_Q H_Q W_Q + + Returns: + Decoded image tensor of shape B C H W + """ + quant = self.post_quant_proj(quant) + dec = self.decoder(quant) + return dec + + def forward(self, x: torch.Tensor, **kwargs) -> Tuple[torch.Tensor, torch.Tensor]: + """Forward pass of the encoder, quantizer, and decoder. + + Args: + x: Input image tensor of shape B C H W + or B H W in case of semantic segmentation + + Returns: + dec: Decoded image tensor of shape B C H W + code_loss: Codebook loss + """ + with torch.no_grad() if self.freeze_enc else nullcontext(): + quant, code_loss, _ = self.encode(x) + dec = self.decode_quant(quant) + return dec, code_loss + + def autoencode(self, x: torch.Tensor, **kwargs) -> torch.Tensor: + """Autoencodes an input image tensor by encoding it, quantizing the + latent code, and decoding it back to an image. + + Args: + x: Input image tensor of shape B C H W + or B H W in case of semantic segmentation + + Returns: + Reconstructed image tensor of shape B C H W + """ + dec, _ = self.forward(x) + return dec + + +class DiVAE(VQ): + """DiVAE ("Diffusion VQ-VAE") model = simple encoder + diffusion decoder with + a discrete bottleneck, inspired by https://arxiv.org/abs/2206.00386. + + Args: + dec_type: String identifier specifying the decoder architecture. + See vq/models/unet/unet.py and vq/models/uvit.py for available architectures. + num_train_timesteps: Number of diffusion timesteps to use for training. + cls_free_guidance_dropout: Dropout probability for classifier-free guidance. + masked_cfg: Whether or not to randomly mask out conditioning tokens. + cls_free_guidance_dropout must be > 0.0 for this to have any effect, and + decides how often masking is performed. E.g. with 0.5, half of the time + the conditioning tokens will be randomly masked, and half the time they + will be kept as is. + masked_cfg_low: Lower bound of number of tokens to mask out. + masked_cfg_high: Upper bound of number of tokens to mask out (inclusive). + Defaults to total number of tokens (H_Q * W_Q) if it is set to None. + scheduler: String identifier specifying the diffusion scheduler to use. + Can be 'ddpm' or 'ddim'. + beta_schedule: String identifier specifying the beta schedule to use for + the diffusion process. Can be 'linear', 'squaredcos_cap_v2' (cosine), + 'shifted_cosine:{shift_amount}'; see vq/scheduling for details. + prediction_type: String identifier specifying the type of prediction to use. + Can be 'sample', 'epsilon', or 'v_prediction'; see vq/scheduling for details. + clip_sample: Whether or not to clip the samples to [-1, 1], at inference only. + thresholding: Whether or not to use dynamic thresholding (introduced by Imagen, + https://arxiv.org/abs/2205.11487) for the diffusion process, at inference only. + conditioning: String identifier specifying the way to condition the diffusion + decoder. Can be 'concat' or 'xattn'. See models for details (only relevant to UViT). + dec_transformer_dropout: Dropout rate for the transformer layers in the + diffusion decoder (only relevant to UViT models). + zero_terminal_snr: Whether or not to enforce zero terminal SNR, i.e. the SNR + at the last timestep is set to zero. This is useful for preventing the model + from "cheating" by using information in the last timestep to reconstruct the image. + See https://arxiv.org/abs/2305.08891. + image_size_dec: Image size for the decoder. Defaults to image_size. + Change this when loading weights from a tokenizer decoder trained on a + different image size. + config: Dictionary containing the model configuration. Only used when loading + from Huggingface Hub. Ignore otherwise. + """ + def __init__(self, + dec_type: str = 'unet_patched', + num_train_timesteps: int = 1000, + cls_free_guidance_dropout: float = 0.0, + masked_cfg: bool = False, + masked_cfg_low: int = 0, + masked_cfg_high: Optional[int] = None, + scheduler: str = 'ddpm', + beta_schedule: str = 'squaredcos_cap_v2', + prediction_type: str = 'v_prediction', + clip_sample: bool = False, + thresholding: bool = True, + conditioning: str = 'concat', + dec_transformer_dropout: float = 0.2, + zero_terminal_snr: bool = True, + image_size_dec: Optional[int] = None, + config: Optional[Dict[str, Any]] = None, + *args, **kwargs): + if config is not None: + config = copy.deepcopy(config) + self.__init__(**config) + return + # Don't want to load the weights just yet + self.original_ckpt_path = kwargs.get('ckpt_path', None) + kwargs['ckpt_path'] = None + super().__init__(*args, **kwargs) + self.ckpt_path = self.original_ckpt_path + self.num_train_timesteps = num_train_timesteps + self.beta_schedule = beta_schedule + self.prediction_type = prediction_type + self.clip_sample = clip_sample + self.thresholding = thresholding + self.zero_terminal_snr = zero_terminal_snr + + if cls_free_guidance_dropout > 0.0: + self.cfg_dist = torch.distributions.Bernoulli(probs=cls_free_guidance_dropout) + else: + self.cfg_dist = None + self.masked_cfg = masked_cfg + self.masked_cfg_low = masked_cfg_low + self.masked_cfg_high = masked_cfg_high + + # Init diffusion decoder + image_size_dec = image_size_dec or self.image_size + if 'unet_' in dec_type: + self.decoder = getattr(unet, dec_type)( + in_channels=self.n_channels, + out_channels=self.n_channels, + cond_channels=self.latent_dim, + image_size=image_size_dec, + ) + elif 'uvit_' in dec_type: + self.decoder = getattr(uvit, dec_type)( + sample_size=image_size_dec, + in_channels=self.n_channels, + out_channels=self.n_channels, + cond_dim=self.latent_dim, + cond_type=conditioning, + mid_drop_rate=dec_transformer_dropout, + ) + else: + raise NotImplementedError(f'dec_type {dec_type} not implemented.') + + # Init training diffusion scheduler / default pipeline for generation + scheduler_cls = DDPMScheduler if scheduler == 'ddpm' else DDIMScheduler + self.noise_scheduler = scheduler_cls( + num_train_timesteps=num_train_timesteps, + thresholding=thresholding, + clip_sample=clip_sample, + beta_schedule=beta_schedule, + prediction_type=prediction_type, + zero_terminal_snr=zero_terminal_snr, + ) + self.pipeline = PipelineCond(model=self.decoder, scheduler=self.noise_scheduler) + + # Load checkpoint + if self.ckpt_path is not None: + self.init_from_ckpt(self.ckpt_path, ignore_keys=self.ignore_keys) + + def sample_mask(self, quant: torch.Tensor, low: int = 0, high: Optional[int] = None) -> torch.BoolTensor: + """Returns a mask of shape B H_Q W_Q, where True = masked-out, False = keep. + + Args: + quant: Dequantized latent tensor of shape B D_Q H_Q W_Q + low: Lower bound of number of tokens to mask out + high: Upper bound of number of tokens to mask out (inclusive). + Defaults to total number of tokens (H_Q * W_Q) if it is set to None. + + Returns: + Boolean mask of shape B H_Q W_Q + """ + B, _, H_Q, W_Q = quant.shape + num_tokens = H_Q * W_Q + high = high if high is not None else num_tokens + + zero_idxs = torch.randint(low=low, high=high+1, size=(B,), device=quant.device) + noise = torch.rand(B, num_tokens, device=quant.device) + ids_arange_shuffle = torch.argsort(noise, dim=1) # ascend: small is keep, large is remove + mask = torch.where(ids_arange_shuffle < zero_idxs.unsqueeze(1), 0, 1) + mask = rearrange(mask, 'b (h w) -> b h w', h=H_Q, w=W_Q).bool() + + return mask + + def _get_pipeline(self, scheduler: Optional[SchedulerMixin] = None) -> PipelineCond: + """Creates a conditional diffusion pipeline with the given scheduler. + + Args: + scheduler: Scheduler to use for the diffusion pipeline. + If None, the default scheduler will be used. + + Returns: + Conditional diffusion pipeline. + """ + return PipelineCond(model=self.decoder, scheduler=scheduler) if scheduler is not None else self.pipeline + + def decode_quant(self, + quant: torch.Tensor, + timesteps: Optional[int] = None, + scheduler: Optional[SchedulerMixin] = None, + generator: Optional[torch.Generator] = None, + image_size: Optional[Union[Tuple[int, int], int]] = None, + verbose: bool = False, + scheduler_timesteps_mode: str = 'trailing', + orig_res: Optional[Union[torch.LongTensor, Tuple[int, int]]] = None) -> torch.Tensor: + """Decodes quantized latent codes back to an image. + + Args: + quant: Quantized latent code of shape B D_Q H_Q W_Q + timesteps: Number of diffusion timesteps to use. Defaults to self.num_train_timesteps. + scheduler: Scheduler to use for the diffusion pipeline. Defaults to the training scheduler. + generator: Random number generator to use for sampling. By default generations are stochastic. + image_size: Image size to use for the diffusion pipeline. Defaults to decoder image size. + verbose: Whether or not to print progress bar. + scheduler_timesteps_mode: The mode to use for DDIMScheduler. One of `trailing`, `linspace`, + `leading`. See https://arxiv.org/abs/2305.08891 for more details. + orig_res: The original resolution of the image to condition the diffusion on. Ignored if None. + See SDXL https://arxiv.org/abs/2307.01952 for more details. + + Returns: + Decoded image tensor of shape B C H W + """ + pipeline = self._get_pipeline(scheduler) + dec = pipeline( + quant, timesteps=timesteps, generator=generator, image_size=image_size, + verbose=verbose, scheduler_timesteps_mode=scheduler_timesteps_mode, orig_res=orig_res + ) + return dec + + def decode_tokens(self, tokens: torch.LongTensor, **kwargs) -> torch.Tensor: + """See `decode_quant` for details on the optional args.""" + return super().decode_tokens(tokens, **kwargs) + + def autoencode(self, + input_clean: torch.Tensor, + timesteps: Optional[int] = None, + scheduler: Optional[SchedulerMixin] = None, + generator: Optional[torch.Generator] = None, + verbose: bool = True, + scheduler_timesteps_mode: str = 'trailing', + orig_res: Optional[Union[torch.LongTensor, Tuple[int, int]]] = None, + **kwargs) -> torch.Tensor: + """Autoencodes an input image tensor by encoding it, quantizing the latent code, + and decoding it back to an image. + + Args: + input_clean: Input image tensor of shape B C H W + or B H W in case of semantic segmentation + timesteps: Number of diffusion timesteps to use. Defaults to self.num_train_timesteps. + scheduler: Scheduler to use for the diffusion pipeline. Defaults to the training scheduler. + generator: Random number generator to use for sampling. By default generations are stochastic. + verbose: Whether or not to print progress bar. + scheduler_timesteps_mode: The mode to use for DDIMScheduler. One of `trailing`, `linspace`, + `leading`. See https://arxiv.org/abs/2305.08891 for more details. + orig_res: The original resolution of the image to condition the diffusion on. Ignored if None. + See SDXL https://arxiv.org/abs/2307.01952 for more details. + + Returns: + Reconstructed image tensor of shape B C H W + """ + pipeline = self._get_pipeline(scheduler) + quant, _, _ = self.encode(input_clean) + image_size = input_clean.shape[-1] + dec = pipeline( + quant, timesteps=timesteps, generator=generator, image_size=image_size, + verbose=verbose, scheduler_timesteps_mode=scheduler_timesteps_mode, orig_res=orig_res + ) + return dec + + def forward(self, + input_clean: torch.Tensor, + input_noised: torch.Tensor, + timesteps: Union[torch.Tensor, float, int], + cond_mask: Optional[torch.Tensor] = None, + orig_res: Optional[Union[torch.LongTensor, Tuple[int, int]]] = None) -> Tuple[torch.Tensor, torch.Tensor]: + """Forward pass of the encoder, quantizer, and decoder. + + Args: + input_clean: Clean input image tensor of shape B C H W + or B H W in case of semantic segmentation. Used for encoding. + input_noised: Noised input image tensor of shape B C H W. Used as + input to the diffusion decoder. + timesteps: Timesteps for conditioning the diffusion decoder on. + cond_mask: Optional mask for the diffusion conditioning. + True = masked-out, False = keep. + orig_res: The original resolution of the image to condition the diffusion on. Ignored if None. + See SDXL https://arxiv.org/abs/2307.01952 for more details. + + Returns: + dec: Decoded image tensor of shape B C H W + code_loss: Codebook loss + """ + with torch.no_grad() if self.freeze_enc else nullcontext(): + quant, code_loss, _ = self.encode(input_clean) + + if cond_mask is None and self.cfg_dist is not None and self.training: + # Create a random mask for each batch element. True = masked-out, False = keep + B, _, H_Q, W_Q = quant.shape + cond_mask = self.cfg_dist.sample((B,)).to(quant.device, dtype=torch.bool) + cond_mask = repeat(cond_mask, 'b -> b h w', h=H_Q, w=W_Q) + if self.masked_cfg: + mask = self.sample_mask(quant, low=self.masked_cfg_low, high=self.masked_cfg_high) + cond_mask = (mask * cond_mask) + + dec = self.decoder(input_noised, timesteps, quant, cond_mask=cond_mask, orig_res=orig_res) + return dec, code_loss + + +class VQControlNet(VQ): + """VQControlNet model = simple pertrained encoder + a ControlNet decoder conditioned on tokens. + + Args: + sd_path: Path to the Stable Diffusion weights for training the ControlNet. + image_size_sd: Stable diffusion input image size. Defaults to image_size. + Change this to the image size that Stable Diffusion is trained on. + pretrained_cn: Whether to use pretrained Stable Diffusion weights for the control model. + cls_free_guidance_dropout: Dropout probability for classifier-free guidance. + masked_cfg: Whether or not to randomly mask out conditioning tokens. + cls_free_guidance_dropout must be > 0.0 for this to have any effect, and + decides how often masking is performed. E.g. with 0.5, half of the time + the conditioning tokens will be randomly masked, and half the time they + will be kept as is. + masked_cfg_low: Lower bound of number of tokens to mask out. + masked_cfg_high: Upper bound of number of tokens to mask out (inclusive). + Defaults to total number of tokens (H_Q * W_Q) if it is set to None. + enable_xformer: Enables xFormers. + adapter: Path to the adapter model weights. The adapter model is initialy trained to map + the tokens to a VAE latent-like representation. Then the output of the adapter model + is passed as the condition to train the ControlNet. By default there is no adapter usage. + config: Dictionary containing the model configuration. Only used when loading + from Huggingface Hub. Ignore otherwise. + """ + def __init__(self, + sd_path: str = "runwayml/stable-diffusion-v1-5", + image_size_sd: Optional[int] = None, + pretrained_cn: bool = False, + cls_free_guidance_dropout: float = 0.0, + masked_cfg: bool = False, + masked_cfg_low: int = 0, + masked_cfg_high: Optional[int] = None, + enable_xformer: bool = False, + adapter: Optional[str] = None, + config: Optional[Dict[str, Any]] = None, + *args, **kwargs): + if config is not None: + config = copy.deepcopy(config) + self.__init__(**config) + return + # Don't want to load the weights just yet + self.original_ckpt_path = kwargs.get('ckpt_path', None) + kwargs['ckpt_path'] = None + super().__init__(*args, **kwargs) + self.ckpt_path = self.original_ckpt_path + + if cls_free_guidance_dropout > 0.0: + self.cfg_dist = torch.distributions.Bernoulli(probs=cls_free_guidance_dropout) + else: + self.cfg_dist = None + self.masked_cfg = masked_cfg + self.masked_cfg_low = masked_cfg_low + self.masked_cfg_high = masked_cfg_high + self.image_size_sd = self.image_size if image_size_sd is None else image_size_sd + + sd_pipeline = StableDiffusionPipeline.from_pretrained(sd_path) + try: + import xformers + XFORMERS_AVAILABLE = True + except ImportError: + print("xFormers not available") + XFORMERS_AVAILABLE = False + enable_xformer = enable_xformer and XFORMERS_AVAILABLE + if enable_xformer: + print('Enabling xFormer for Stable Diffusion') + sd_pipeline.enable_xformers_memory_efficient_attention() + + self.decoder = getattr(controlnet, 'controlnet')( + in_channels=4, + cond_channels=self.latent_dim, + sd_pipeline=sd_pipeline, + image_size=self.image_size_sd, + pretrained_cn=pretrained_cn, + enable_xformer=enable_xformer, + adapter=adapter, + ) + + # Use the defualt controlnet pipeline both for training and generation + self.noise_scheduler = PNDMScheduler(**sd_pipeline.scheduler.config) + self.vae = sd_pipeline.vae + self._freeze_vae() + + self.pipeline = PipelineCond(model=self.decoder, scheduler=self.noise_scheduler) + + # Load checkpoint + if self.ckpt_path is not None: + self.init_from_ckpt(self.ckpt_path, ignore_keys=self.ignore_keys) + + def sample_mask(self, quant: torch.Tensor, low: int = 0, high: Optional[int] = None) -> torch.BoolTensor: + """Returns a mask of shape B H_Q W_Q, where True = masked-out, False = keep. + + Args: + quant: Dequantized latent tensor of shape B D_Q H_Q W_Q + low: Lower bound of number of tokens to mask out + high: Upper bound of number of tokens to mask out (inclusive). + Defaults to total number of tokens (H_Q * W_Q) if it is set to None. + + Returns: + Boolean mask of shape B H_Q W_Q + """ + B, _, H_Q, W_Q = quant.shape + num_tokens = H_Q * W_Q + high = high if high is not None else num_tokens + + zero_idxs = torch.randint(low=low, high=high+1, size=(B,), device=quant.device) + noise = torch.rand(B, num_tokens, device=quant.device) + ids_arange_shuffle = torch.argsort(noise, dim=1) # ascend: small is keep, large is remove + mask = torch.where(ids_arange_shuffle < zero_idxs.unsqueeze(1), 0, 1) + mask = rearrange(mask, 'b (h w) -> b h w', h=H_Q, w=W_Q).bool() + + return mask + + def decode_quant(self, + quant: torch.Tensor, + timesteps: Optional[int] = None, + generator: Optional[torch.Generator] = None, + image_size: Optional[Union[Tuple[int, int], int]] = None, + verbose: bool = False, + vae_decode: bool = False, + scheduler_timesteps_mode: str = 'leading', + prompt: Optional[Union[List[str], str]]= None, + orig_res: Optional[Union[torch.LongTensor, Tuple[int, int]]] = None, + guidance_scale: int = 0.0, + cond_scale: int = 1.0) -> torch.Tensor: + """Decodes quantized latent codes back to an image. + + Args: + quant: Quantized latent code of shape B D_Q H_Q W_Q + timesteps: Number of diffusion timesteps to use. Defaults to self.num_train_timesteps. + generator: Random number generator to use for sampling. By default generations are stochastic. + image_size: Image size to use for the diffusion pipeline. Defaults to decoder image size. + verbose: Whether or not to print progress bar. + vae_decode: If set to True decodes the latent output of stable diffusion + scheduler_timesteps_mode: The mode to use for DDIMScheduler. One of `trailing`, `linspace`, + `leading`. See https://arxiv.org/abs/2305.08891 for more details. + prompt: the input prompts for controlnet. + orig_res: The original resolution of the image to condition the diffusion on. Ignored if None. + See SDXL https://arxiv.org/abs/2307.01952 for more details. + guidance_scale: Classifier free guidance scale. + cond_scale: Scale that is multiplied by the output of control model before being added + to stable diffusion layers in controlnet. + + Returns: + Decoded tensor of shape B C H W + """ + dec = self.pipeline( + quant, timesteps=timesteps, generator=generator, image_size=image_size, + verbose=verbose, scheduler_timesteps_mode=scheduler_timesteps_mode, prompt=prompt, + guidance_scale=guidance_scale, cond_scale=cond_scale, + ) + + if vae_decode: + return self.vae_decode(dec) + + return dec + + def decode_tokens(self, tokens: torch.LongTensor, **kwargs) -> torch.Tensor: + """See `decode_quant` for details on the optional args.""" + return super().decode_tokens(tokens, **kwargs) + + @torch.no_grad() + def vae_encode(self, x: torch.Tensor): + """Encodes the input image into vae latent representaiton. + + Args: + x: Input images + + Returns: + Encoded latent tensor of shape B C H W + """ + z = self.vae.encode(x).latent_dist.sample() + z = z * self.vae.config.scaling_factor + return z + + @torch.no_grad() + def vae_decode(self, x: torch.Tensor, clip: bool = True) -> torch.Tensor: + """Decodes the vae latent representation into vae latent representaiton. + + Args: + x: VAE latent representation + clip: If set True clips the decoded image between -1 and 1. + + Returns: + Decoded image of shape B C H W + """ + x = self.vae.decode(x / self.vae.config.scaling_factor).sample + if clip: + x = torch.clip(x, min=-1, max=1) + return x + + def autoencode(self, + input_clean: torch.Tensor, + timesteps: Optional[int] = None, + generator: Optional[torch.Generator] = None, + image_size: Optional[Union[Tuple[int, int], int]] = None, + verbose: bool = False, + vae_decode: bool = False, + scheduler_timesteps_mode: str = 'leading', + prompt: Optional[Union[List[str], str]]= None, + orig_res: Optional[Union[torch.LongTensor, Tuple[int, int]]] = None, + guidance_scale: int = 0.0, + cond_scale: int = 1.0) -> torch.Tensor: + """Autoencodes an input image tensor by encoding it, quantizing the latent code, + and decoding it back to an image. + + Args: + input_clean: Input image tensor of shape B C H W + or B H W in case of semantic segmentation + timesteps: Number of diffusion timesteps to use. Defaults to self.num_train_timesteps. + scheduler: Scheduler to use for the diffusion pipeline. Defaults to the training scheduler. + generator: Random number generator to use for sampling. By default generations are stochastic. + image_size: Image size to use for the diffusion pipeline. Defaults to decoder image size. + verbose: Whether or not to print progress bar. + vae_decode: If set to True, decodes the latent output of stable diffusion + scheduler_timesteps_mode: The mode to use for DDIMScheduler. One of `trailing`, `linspace`, + `leading`. See https://arxiv.org/abs/2305.08891 for more details. + prompt: the input prompts for controlnet. + orig_res: The original resolution of the image to condition the diffusion on. Ignored if None. + See SDXL https://arxiv.org/abs/2307.01952 for more details. + guidance_scale: Classifier free guidance scale. + cond_scale: Scale that is multiplied by the output of control model before being added + to stable diffusion layers in controlnet. + + Returns: + Reconstructed tensor of shape B C H W + """ + quant, _, _ = self.encode(input_clean) + dec = self.pipeline( + quant, timesteps=timesteps, generator=generator, + verbose=verbose, scheduler_timesteps_mode=scheduler_timesteps_mode, prompt=prompt, + guidance_scale=guidance_scale, cond_scale=cond_scale, + ) + + if vae_decode: + return self.vae_decode(dec) + + return dec + + def forward(self, + input_clean: torch.Tensor, + input_noised: torch.Tensor, + timesteps: Union[torch.Tensor, float, int], + cond_mask: Optional[torch.Tensor] = None, + prompt: Optional[Union[List[str], str]] = None, + orig_res: Optional[Union[torch.LongTensor, Tuple[int, int]]] = None) -> Tuple[torch.Tensor, torch.Tensor]: + """Forward pass of the encoder, quantizer, and decoder. + + Args: + input_clean: Clean input image tensor of shape B C H W + or B H W in case of semantic segmentation. Used for encoding. + input_noised: Noised input image tensor of shape B C H W. Used as + input to the diffusion decoder. + timesteps: Timesteps for conditioning the diffusion decoder on. + cond_mask: Optional mask for the diffusion conditioning. + True = masked-out, False = keep. + prompt: ControlNet input prompt. Defaults to an empty string. + orig_res: The original resolution of the image to condition the diffusion on. Ignored if None. + See SDXL https://arxiv.org/abs/2307.01952 for more details. + + Returns: + dec: Decoded image tensor of shape B C H W + code_loss: Codebook loss + """ + with torch.no_grad() if self.freeze_enc else nullcontext(): + quant, code_loss, _ = self.encode(input_clean) + + if cond_mask is None and self.cfg_dist is not None and self.training: + # Create a random mask for each batch element. True = masked-out, False = keep + B, _, H_Q, W_Q = quant.shape + cond_mask = self.cfg_dist.sample((B,)).to(quant.device, dtype=torch.bool) + cond_mask = repeat(cond_mask, 'b -> b h w', h=H_Q, w=W_Q) + if self.masked_cfg: + mask = self.sample_mask(quant, low=self.masked_cfg_low, high=self.masked_cfg_high) + cond_mask = (mask * cond_mask) + + dec = self.decoder(input_noised, timesteps, quant, cond_mask=cond_mask, orig_res=orig_res, prompt=prompt) + return dec, code_loss + + def _freeze_vae(self): + """Freezes VAE""" + for param in self.vae.parameters(): + param.requires_grad = False \ No newline at end of file